From fgsch at lodoss.net Mon Apr 2 14:09:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 2 Apr 2018 14:09:11 +0000 (UTC) Subject: [master] cd6b026 Enable coredump with ASAN Message-ID: <20180402140911.9F9E199CE7@lists.varnish-cache.org> commit cd6b026ad8bb8495d127b2f4e6fc2f5172a69951 Author: Federico G. Schwindt Date: Mon Apr 2 15:06:45 2018 +0100 Enable coredump with ASAN Mostly adding it here for documentation purposes. diff --git a/.travis.yml b/.travis.yml index f716744..f3d5a2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,7 @@ before_install: sudo apt-get install -y clang-$CLANG llvm-$CLANG; export CC=clang-$CLANG; export CONFIGURE_ARGS="--enable-developer-warnings --enable-debugging-symbols --disable-stack-protector --with-persistent-storage ${SAN_FLAGS}"; - export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0; + export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0; export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr; export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr; export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr; From geoff at uplex.de Mon Apr 2 15:24:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 2 Apr 2018 15:24:10 +0000 (UTC) Subject: [master] dfcb080 Improved vtc test for vmod_proxy Message-ID: <20180402152410.5039F9B452@lists.varnish-cache.org> commit dfcb080d3a514324f314fa325e4b46152861a230 Author: Emmanuel Hocdet Date: Thu Mar 15 15:15:16 2018 +0100 Improved vtc test for vmod_proxy diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc index 49cb0ef..d40e353 100644 --- a/bin/varnishtest/tests/o00005.vtc +++ b/bin/varnishtest/tests/o00005.vtc @@ -9,18 +9,30 @@ varnish v1 -proto "PROXY" -vcl+backend { import proxy; sub vcl_deliver { + if (proxy.is_ssl()) { + set resp.http.is_ssl = "yes"; + } + if (!proxy.client_has_cert_sess()) { + set resp.http.client_has_cert_sess = "no"; + } + if (!proxy.client_has_cert_conn()) { + set resp.http.client_has_cert_conn = "no"; + } if (proxy.ssl_verify_result() == 0) { set resp.http.verify = "ok"; } set resp.http.alpn = proxy.alpn(); + set resp.http.authority = proxy.authority(); set resp.http.ssl-version = proxy.ssl_version(); set resp.http.cipher = proxy.ssl_cipher(); set resp.http.key = proxy.cert_key(); set resp.http.sign = proxy.cert_sign(); + set resp.http.cn = proxy.client_cert_cn(); } } -start client c1 { + # PROXY2 with CRC32C TLV sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a 21 11 00 65" sendhex "d9 46 b5 21 5f 8e a8 22 ed 96 01 bb 03 00 04 95" sendhex "03 ee 75 01 00 02 68 32 02 00 0a 68 6f 63 64 65" @@ -32,10 +44,29 @@ client c1 { txreq rxresp expect resp.status == 200 + expect resp.http.is_ssl == yes + expect resp.http.client_has_cert_sess == no + expect resp.http.client_has_cert_conn == no expect resp.http.verify == ok expect resp.http.alpn == h2 + expect resp.http.authority == hocdet.net expect resp.http.ssl-version == TLSv1.3 expect resp.http.cipher == AEAD-AES128-GCM-SHA256 expect resp.http.key == EC256 expect resp.http.sign == RSA-SHA256 + expect resp.http.cn == "" +} -run + +client c1 { + # PROXY2 with CRC32C TLV and bad checksum + sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a 21 11 00 65" + sendhex "d9 46 b5 21 5f 8e a8 22 ed 96 01 bb 03 00 04 95" + sendhex "03 ee 75 01 00 02 68 32 02 00 0a 68 6f 63 64 65" + sendhex "74 2e 6e 65 74 20 00 3d 01 00 00 00 00 21 00 07" + sendhex "54 4c 53 76 31 2e 33 25 00 05 45 43 32 35 36 24" + sendhex "00 0a 52 53 41 2d 53 48 41 32 35 36 23 00 16 41" + sendhex "45 41 44 2d 41 45 53 31 32 38 2d 47 43 4d 2d 53" + sendhex "48 41 32 35 35" + txreq + expect_close } -run From fgsch at lodoss.net Mon Apr 2 15:47:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 2 Apr 2018 15:47:08 +0000 (UTC) Subject: [master] e5555da Run make distcheck on PRs Message-ID: <20180402154708.C36929BB23@lists.varnish-cache.org> commit e5555da4b39574d8568429b372ecc128541c2cee Author: Federico G. Schwindt Date: Mon Apr 2 16:39:24 2018 +0100 Run make distcheck on PRs Also give irc notifications a try. Discussed with phk and geoff. diff --git a/.travis.yml b/.travis.yml index f3d5a2b..9eb8e4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,13 @@ addons: apt: packages: - python-docutils +notifications: + irc: + channels: + - "irc.linpro.no#varnish-hacking" + on_success: change + use_notice: true + skip_join: true before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; @@ -50,4 +57,5 @@ before_install: script: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PYTHONPATH=`brew --prefix`/lib/python2.7/site-packages; fi - - make -j3 check VERBOSE=1 + - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then make -j3 distcheck; fi + - if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then make -j3 check VERBOSE=1; fi From nils.goroll at uplex.de Tue Apr 3 09:33:13 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 3 Apr 2018 09:33:13 +0000 (UTC) Subject: [master] b40aa2c disambiguate VTLA variable names - vcp is now connection pooling Message-ID: <20180403093313.BAA69B592C@lists.varnish-cache.org> commit b40aa2cea0ec1100abc58ae9555db3e35922d708 Author: Nils Goroll Date: Sat Mar 31 15:41:37 2018 +0200 disambiguate VTLA variable names - vcp is now connection pooling diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 5ac0a00..6001d32 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -686,12 +686,12 @@ VRT_VSM_Cluster_New(VRT_CTX, size_t sz) } void v_matchproto_() -VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsmw_cluster **vcp) +VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsmw_cluster **vsmcp) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(vcp); - VSMW_DestroyCluster(heritage.proc_vsmw, vcp); + AN(vsmcp); + VSMW_DestroyCluster(heritage.proc_vsmw, vsmcp); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 8501cc4..0aa5477 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -275,15 +275,15 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx) } void -VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp) +VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp) { struct vsmw_cluster *vc; vsmw_lock(); CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); - AN(vcp); - vc = *vcp; - *vcp = NULL; + AN(vsmcp); + vc = *vsmcp; + *vsmcp = NULL; CHECK_OBJ_NOTNULL(vc, VSMW_CLUSTER_MAGIC); if (vc->cseg != NULL) { From nils.goroll at uplex.de Tue Apr 3 09:33:13 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 3 Apr 2018 09:33:13 +0000 (UTC) Subject: [master] aab5dda more details to fetcherror Message-ID: <20180403093313.CA800B592E@lists.varnish-cache.org> commit aab5dda67d83bf0f4999e1718aaedbb810c1ead7 Author: Nils Goroll Date: Sun Apr 1 14:45:34 2018 +0200 more details to fetcherror Ref #2622 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 392baba..acef648 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -33,6 +33,7 @@ #include "config.h" #include +#include #include "cache_varnishd.h" @@ -116,7 +117,8 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, pfd = VTP_Get(bp->tcp_pool, tmod, wrk, force_fresh); if (pfd == NULL) { VSLb(bo->vsl, SLT_FetchError, - "backend %s: fail", bp->director->display_name); + "backend %s: fail errno %d (%s)", + bp->director->display_name, errno, strerror(errno)); // XXX: Per backend stats ? VSC_C_main->backend_fail++; bo->htc = NULL; From fgsch at lodoss.net Tue Apr 3 09:48:09 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 3 Apr 2018 09:48:09 +0000 (UTC) Subject: [master] 18920d6 Also install python-sphinx Message-ID: <20180403094809.472A6B5E85@lists.varnish-cache.org> commit 18920d641c6cdf5c401f7084db13575d289aad02 Author: Federico G. Schwindt Date: Tue Apr 3 10:46:47 2018 +0100 Also install python-sphinx Required for make distcheck. Spotted by geoff. diff --git a/.travis.yml b/.travis.yml index 9eb8e4a..858d079 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ addons: apt: packages: - python-docutils + - python-sphinx notifications: irc: channels: From fgsch at lodoss.net Tue Apr 3 13:39:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 3 Apr 2018 13:39:11 +0000 (UTC) Subject: [master] 78866f7 Make "make distcheck" happy under osx Message-ID: <20180403133911.7B3EE630F8@lists.varnish-cache.org> commit 78866f7b4f2ddac2b72d9e30c93c6368cfdd88bf Author: Federico G. Schwindt Date: Tue Apr 3 14:31:10 2018 +0100 Make "make distcheck" happy under osx Also report success/failure earlier. diff --git a/.travis.yml b/.travis.yml index 858d079..2ff305c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ sudo: required language: c matrix: + fast_finish: true include: - os: linux dist: trusty @@ -34,7 +35,7 @@ notifications: before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; - brew install docutils nghttp2; + brew install docutils sphinx-doc nghttp2; fi - if [[ -n "$CLANG" ]]; then wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | From phk at phk.freebsd.dk Tue Apr 3 15:27:51 2018 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 03 Apr 2018 15:27:51 +0000 Subject: [master] 78866f7 Make "make distcheck" happy under osx In-Reply-To: <20180403133911.7B3EE630F8@lists.varnish-cache.org> References: <20180403133911.7B3EE630F8@lists.varnish-cache.org> Message-ID: <63530.1522769271@critter.freebsd.dk> -------- In message <20180403133911.7B3EE630F8 at lists.varnish-cache.org>, Federico G. Sch windt writes: Strictly speaking we now also have tests which can use haproxy (tests/h*.vtc), but both those and the tests which use nghttp2 are conditional on finding the respective binaries. Poul-Henning >commit 78866f7b4f2ddac2b72d9e30c93c6368cfdd88bf >Author: Federico G. Schwindt >Date: Tue Apr 3 14:31:10 2018 +0100 > > Make "make distcheck" happy under osx > > Also report success/failure earlier. > >diff --git a/.travis.yml b/.travis.yml >index 858d079..2ff305c 100644 >--- a/.travis.yml >+++ b/.travis.yml >@@ -2,6 +2,7 @@ > sudo: required > language: c > matrix: >+ fast_finish: true > include: > - os: linux > dist: trusty >@@ -34,7 +35,7 @@ notifications: > before_install: > - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then > brew update; >- brew install docutils nghttp2; >+ brew install docutils sphinx-doc nghttp2; > fi > - if [[ -n "$CLANG" ]]; then > wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | >_______________________________________________ >varnish-commit mailing list >varnish-commit at varnish-cache.org >https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Wed Apr 4 06:55:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Apr 2018 06:55:14 +0000 (UTC) Subject: [master] 7a881b1 Change how gcov is run Message-ID: <20180404065514.CCFF0A63B4@lists.varnish-cache.org> commit 7a881b177c5b77597398b333f557a8846337fc88 Author: Poul-Henning Kamp Date: Wed Apr 4 06:53:50 2018 +0000 Change how gcov is run diff --git a/tools/vtest.sh b/tools/vtest.sh index 9202039..1508c2f 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -149,14 +149,6 @@ makegcov () ( set -x cd "${SRCDIR}" - export CFLAGS="-fprofile-arcs -ftest-coverage -fstack-protector -DDONT_DLCLOSE_VMODS" CC=gcc49 - export MAKEFLAGS=-j1 - umask 0 - - find . -name '*.gc??' -print | xargs rm -f - - sh autogen.des || exit 1 - make || exit 1 if [ `id -u` -eq 0 ] ; then @@ -180,14 +172,19 @@ failedtests () ( cd "${SRCDIR}" + VTCDIR=bin/varnishtest/tests + VERSION=`./configure --version | awk 'NR == 1 {print $NF}'` LOGDIR="varnish-$VERSION/_build/sub/bin/varnishtest/tests" - VTCDIR=bin/varnishtest/tests # cope with older automake, remove the sub directory test ! -d $LOGDIR && LOGDIR="varnish-$VERSION/_build/bin/varnishtest/tests" + # gcov situation + test ! -d $LOGDIR && + LOGDIR="bin/varnishtest/tests" + grep -l ':test-result: FAIL' "$LOGDIR"/*.trs | while read trs do @@ -201,14 +198,16 @@ failedtests () ( done ) - +if $enable_gcov ; then + export CC=gcc6 + export CFLAGS="-fprofile-arcs -ftest-coverage -fstack-protector -DDONT_DLCLOSE_VMODS" + export MAKEFLAGS=-j1 +fi orev=000 waitnext=${WAITBAD} i=0 -last_day=`date +%d` - while [ $MAXRUNS -eq 0 ] || [ $i -lt $MAXRUNS ] do i=$((i + 1)) @@ -231,19 +230,6 @@ do rm -rf "${REPORTDIR}" mkdir "${REPORTDIR}" - if ! $enable_gcov ; then - do_gcov=false - elif [ -f _force_gcov ] ; then - do_gcov=true - rm -f _force_gcov - elif [ `date +%d` == $last_day ] ; then - do_gcov=false - elif [ `date +%H` -lt 3 ] ; then - do_gcov=false - else - do_gcov=true - fi - echo "VTEST 1.04" > ${VTEST_REPORT} echo "DATE `date +%s`" >> ${VTEST_REPORT} echo "BRANCH trunk" >> ${VTEST_REPORT} @@ -257,20 +243,24 @@ do fi echo "MESSAGE ${MESSAGE}" >> ${VTEST_REPORT} echo "GITREV $rev" >> ${VTEST_REPORT} + + find . -name '*.gc??' -print | xargs rm -f + if ! autogen >> ${REPORTDIR}/_autogen 2>&1 ; then echo "AUTOGEN BAD" >> ${VTEST_REPORT} echo "MANIFEST _autogen" >> ${VTEST_REPORT} else echo "AUTOGEN GOOD" >> ${VTEST_REPORT} - if $do_gcov ; then - last_day=`date +%d` + if $enable_gcov ; then if makegcov >> ${REPORTDIR}/_makegcov 2>&1 ; then mv ${SRCDIR}/_gcov ${REPORTDIR}/ echo "MAKEGCOV GOOD" >> ${VTEST_REPORT} echo "MANIFEST _gcov" >> ${VTEST_REPORT} + waitnext=${WAITGOOD} else echo "MAKEGCOV BAD" >> ${VTEST_REPORT} echo "MANIFEST _makegcov" >> ${VTEST_REPORT} + failedtests >> ${VTEST_REPORT} fi elif ! makedistcheck >> ${REPORTDIR}/_makedistcheck 2>&1 ; then echo "MAKEDISTCHECK BAD" >> ${VTEST_REPORT} From hermunn at varnish-software.com Wed Apr 4 09:47:09 2018 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 4 Apr 2018 09:47:09 +0000 (UTC) Subject: [master] c78555b Go back to trunk after release of 6.0.0 Message-ID: <20180404094709.8F6B2A927A@lists.varnish-cache.org> commit c78555b62eb200634282f510968fe1fcbcd19cd7 Author: P?l Hermunn Johansen Date: Wed Apr 4 11:30:11 2018 +0200 Go back to trunk after release of 6.0.0 diff --git a/configure.ac b/configure.ac index e16e2d5..db46168 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2018 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [6.0.0], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [trunk], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index bb85dc2..68bfe51 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,6 +1,6 @@ -==================== -Varnish Cache master -==================== +============================= +Varnish Cache Trunk (ongoing) +============================= bundled tools ------------- From phk at FreeBSD.org Wed Apr 4 10:12:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Apr 2018 10:12:09 +0000 (UTC) Subject: [master] c430f13 Update to gcc6 Message-ID: <20180404101209.B64BEA996A@lists.varnish-cache.org> commit c430f13179599acd7fa767f5a12cecbf4ebe6f6e Author: Poul-Henning Kamp Date: Wed Apr 4 10:10:29 2018 +0000 Update to gcc6 diff --git a/tools/gcov_digest.py b/tools/gcov_digest.py index 4b24187..fd595c9 100644 --- a/tools/gcov_digest.py +++ b/tools/gcov_digest.py @@ -33,7 +33,7 @@ found in a subdirectory tree. Options: -g gcov-program - default:gcov49 + default:gcov6 -o output-filename default: stdout @@ -180,7 +180,7 @@ if __name__ == "__main__": optlist, args = getopt.getopt(sys.argv[1:], "g:o:x:") fo = sys.stdout - gcovprog = "gcov49" + gcovprog = "gcov6 for f, v in optlist: if f == '-o' and v == '-': From phk at FreeBSD.org Wed Apr 4 11:57:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Apr 2018 11:57:10 +0000 (UTC) Subject: [master] 13bd9d0 Typo Message-ID: <20180404115710.92CDCAD8A9@lists.varnish-cache.org> commit 13bd9d05b7346ee45ca79ee4fca79e3ae983c393 Author: Poul-Henning Kamp Date: Wed Apr 4 11:56:33 2018 +0000 Typo diff --git a/tools/gcov_digest.py b/tools/gcov_digest.py index fd595c9..138f7e8 100644 --- a/tools/gcov_digest.py +++ b/tools/gcov_digest.py @@ -180,7 +180,7 @@ if __name__ == "__main__": optlist, args = getopt.getopt(sys.argv[1:], "g:o:x:") fo = sys.stdout - gcovprog = "gcov6 + gcovprog = "gcov6" for f, v in optlist: if f == '-o' and v == '-': From nils.goroll at uplex.de Wed Apr 4 12:58:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 4 Apr 2018 12:58:10 +0000 (UTC) Subject: [master] 3b941aa improve do_stream documentation Message-ID: <20180404125810.7CA53AEADB@lists.varnish-cache.org> commit 3b941aa6d1888a5fa175b4085e4d2707442eca89 Author: Nils Goroll Date: Wed Apr 4 14:56:59 2018 +0200 improve do_stream documentation diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index c4ded7c..85c11fb 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -719,13 +719,17 @@ beresp.do_stream Writable from: vcl_backend_response, vcl_backend_error + Default: true + Deliver the object to the client while fetching the whole object into varnish. For uncacheable objects, storage for parts of the body which have been sent to the client may get freed early, depending on the storage engine used. - + + This variable has no effect if do_esi is true and when the + response body is empty. beresp.do_gzip From nils.goroll at uplex.de Wed Apr 4 13:31:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 4 Apr 2018 13:31:10 +0000 (UTC) Subject: [master] cfd77a6 common sense "and" == boolean logic "or" Message-ID: <20180404133110.4FA19AF45F@lists.varnish-cache.org> commit cfd77a6f6c2ff4219312cf057f70d7a59ec84988 Author: Nils Goroll Date: Wed Apr 4 15:30:14 2018 +0200 common sense "and" == boolean logic "or" diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 85c11fb..6df3635 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -728,7 +728,7 @@ beresp.do_stream have been sent to the client may get freed early, depending on the storage engine used. - This variable has no effect if do_esi is true and when the + This variable has no effect if do_esi is true or when the response body is empty. beresp.do_gzip From nils.goroll at uplex.de Wed Apr 4 14:49:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 4 Apr 2018 14:49:11 +0000 (UTC) Subject: [master] 2c4e481 Use libumem only as a storage allocator, not to override malloc etc. Message-ID: <20180404144911.09C88B0D3B@lists.varnish-cache.org> commit 2c4e481ed1e7ddfc8cc04233cea53b0942340ab2 Author: Geoff Simmons Date: Mon Apr 2 22:05:05 2018 +0200 Use libumem only as a storage allocator, not to override malloc etc. We do this by not linking with libumem, but rather using dlopen/dlsym to load the slab allocator interface, which is used by the stevedore. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 739b16c..1e8640e 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -169,7 +169,7 @@ varnishd_LDADD = \ @SAN_LDFLAGS@ \ @JEMALLOC_LDADD@ \ @PCRE_LIBS@ \ - ${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${RT_LIBS} ${LIBM} ${UMEM_LIBS} + ${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${RT_LIBS} ${LIBM} noinst_PROGRAMS = vhp_gen_hufdec vhp_gen_hufdec_SOURCES = hpack/vhp_gen_hufdec.c diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 05b4c61..237a45b 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -136,7 +136,7 @@ usage(void) printf(FMT, "-s [name=]kind[,options]", "Storage specification"); printf(FMT, "", "Can be specified multiple times."); -#ifdef HAVE_LIBUMEM +#ifdef HAVE_UMEM_H printf(FMT, "", " -s default (=umem)"); printf(FMT, "", " -s umem"); #else diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index ba7e5b1..a37ddf3 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -126,7 +126,7 @@ static const struct choice STV_choice[] = { { "deprecated_persistent", &smp_stevedore }, { "persistent", &smp_fake_stevedore }, #endif -#if defined(HAVE_LIBUMEM) +#if defined(HAVE_UMEM_H) { "umem", &smu_stevedore }, { "default", &smu_stevedore }, #else diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index afc2771..acb291a 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -33,13 +33,14 @@ #include "config.h" -#if defined(HAVE_LIBUMEM) +#if defined(HAVE_UMEM_H) #include "cache/cache_varnishd.h" #include #include #include +#include #include "storage/storage.h" #include "storage/storage_simple.h" @@ -67,6 +68,30 @@ struct smu { struct smu_sc *sc; }; +/* + * We only want the umem slab allocator for cache storage, not also as a + * substitute for malloc and friends. So we don't link with libumem, but + * use dlopen/dlsym to get the slab allocator interface into function + * pointers. + */ +typedef void * (*umem_alloc_f)(size_t size, int flags); +typedef void (*umem_free_f)(void *buf, size_t size); +typedef umem_cache_t * (*umem_cache_create_f)(char *debug_name, size_t bufsize, + size_t align, umem_constructor_t *constructor, + umem_destructor_t *destructor, umem_reclaim_t *reclaim, + void *callback_data, vmem_t *source, int cflags); +typedef void (*umem_cache_destroy_f)(umem_cache_t *cache); +typedef void * (*umem_cache_alloc_f)(umem_cache_t *cache, int flags); +typedef void (*umem_cache_free_f)(umem_cache_t *cache, void *buffer); + +static void *libumem_hndl = NULL; +static umem_alloc_f umem_allocf = NULL; +static umem_free_f umem_freef = NULL; +static umem_cache_create_f umem_cache_createf = NULL; +static umem_cache_destroy_f umem_cache_destroyf = NULL; +static umem_cache_alloc_f umem_cache_allocf = NULL; +static umem_cache_free_f umem_cache_freef = NULL; + /* init required per cache get: smu->sz = size smu->s.ptr; @@ -142,14 +167,14 @@ smu_alloc(const struct stevedore *st, size_t size) * allocations growing another full page, just to accommodate the smu. */ - p = umem_alloc(size, UMEM_DEFAULT); + p = umem_allocf(size, UMEM_DEFAULT); if (p != NULL) { AN(smu_sc->smu_cache); - smu = umem_cache_alloc(smu_sc->smu_cache, UMEM_DEFAULT); + smu = umem_cache_allocf(smu_sc->smu_cache, UMEM_DEFAULT); if (smu != NULL) smu->s.ptr = p; else - umem_free(p, size); + umem_freef(p, size); } if (smu == NULL) { Lck_Lock(&smu_sc->smu_mtx); @@ -191,9 +216,9 @@ smu_free(struct storage *s) sc->stats->g_space += smu->sz; Lck_Unlock(&sc->smu_mtx); - umem_free(smu->s.ptr, smu->sz); + umem_freef(smu->s.ptr, smu->sz); smu_smu_init(smu, sc); - umem_cache_free(sc->smu_cache, smu); + umem_cache_freef(sc->smu_cache, smu); } static VCL_BYTES v_matchproto_(stv_var_used_space) @@ -235,6 +260,28 @@ smu_init(struct stevedore *parent, int ac, char * const *av) if (ac == 0 || *av[0] == '\0') return; + /* Check if these load in the management process. */ + (void) dlerror(); + if ((libumem_hndl = dlopen("libumem.so", RTLD_NOW)) == NULL) + ARGV_ERR("(-sumem) cannot open libumem.so: %s", dlerror()); + +#define DLSYM_UMEM(fptr,sym) \ + do { \ + (void) dlerror(); \ + if ((fptr = dlsym(libumem_hndl, #sym)) == NULL) \ + ARGV_ERR("(-sumem) cannot find symbol " #sym ": %s", \ + dlerror()); \ + } while(0) + + DLSYM_UMEM(umem_allocf, umem_alloc); + DLSYM_UMEM(umem_freef, umem_free); + DLSYM_UMEM(umem_cache_createf, umem_cache_create); + DLSYM_UMEM(umem_cache_destroyf, umem_cache_destroy); + DLSYM_UMEM(umem_cache_allocf, umem_cache_alloc); + DLSYM_UMEM(umem_cache_freef, umem_cache_free); + +#undef DLSYM_UMEM + e = VNUM_2bytes(av[0], &u, 0); if (e != NULL) ARGV_ERR("(-sumem) size \"%s\": %s\n", av[0], e); @@ -262,7 +309,32 @@ smu_open(struct stevedore *st) if (smu_sc->smu_max != SIZE_MAX) smu_sc->stats->g_space = smu_sc->smu_max; - smu_sc->smu_cache = umem_cache_create(st->ident, + /* + * Load the symbols for use in the child process, assert if they + * fail to load. + */ + if (libumem_hndl == NULL) { + libumem_hndl = dlopen("libumem.so", RTLD_NOW); + AN(libumem_hndl); + +#define DLSYM_UMEM(fptr,sym) \ + do { \ + fptr = dlsym(libumem_hndl, #sym); \ + AN(fptr); \ + } while(0) + + DLSYM_UMEM(umem_allocf, umem_alloc); + DLSYM_UMEM(umem_freef, umem_free); + DLSYM_UMEM(umem_cache_createf, umem_cache_create); + DLSYM_UMEM(umem_cache_destroyf, umem_cache_destroy); + DLSYM_UMEM(umem_cache_allocf, umem_cache_alloc); + DLSYM_UMEM(umem_cache_freef, umem_cache_free); + +#undef DLSYM_UMEM + + } + + smu_sc->smu_cache = umem_cache_createf(st->ident, sizeof(struct smu), 0, // align smu_smu_constructor, @@ -285,7 +357,7 @@ smu_close(const struct stevedore *st, int warn) CAST_OBJ_NOTNULL(smu_sc, st->priv, SMU_SC_MAGIC); if (warn) return; - umem_cache_destroy(smu_sc->smu_cache); + umem_cache_destroyf(smu_sc->smu_cache); smu_sc->smu_cache = NULL; /* diff --git a/configure.ac b/configure.ac index db46168..3aeb7bb 100644 --- a/configure.ac +++ b/configure.ac @@ -83,7 +83,7 @@ _VARNISH_CHECK_LIB(nsl, getaddrinfo) AC_SUBST(NET_LIBS, "${SOCKET_LIBS} ${NSL_LIBS}") # Userland slab allocator from Solaris, ported to other systems -AC_CHECK_HEADERS([umem.h], [_VARNISH_CHECK_LIB(umem, umem_alloc)]) +AC_CHECK_HEADERS([umem.h]) # XXX: This _may_ be for OS/X AC_CHECK_LIBM From nils.goroll at uplex.de Wed Apr 4 14:49:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 4 Apr 2018 14:49:11 +0000 (UTC) Subject: [master] 11a50e7 umem: test-load libumem in a separate namespace in the management process Message-ID: <20180404144911.22E0EB0D3E@lists.varnish-cache.org> commit 11a50e7944fe3ae69e649204b9b6683b92d2e463 Author: Nils Goroll Date: Wed Apr 4 09:21:11 2018 +0200 umem: test-load libumem in a separate namespace in the management process libumem's atfork handler will redefine malloc() and friends so if we load it in the manager, we cannot confine its use to the stevedore diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index acb291a..7464a19 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -41,6 +41,7 @@ #include #include #include +#include #include "storage/storage.h" #include "storage/storage_simple.h" @@ -262,7 +263,8 @@ smu_init(struct stevedore *parent, int ac, char * const *av) /* Check if these load in the management process. */ (void) dlerror(); - if ((libumem_hndl = dlopen("libumem.so", RTLD_NOW)) == NULL) + libumem_hndl = dlmopen(LM_ID_NEWLM, "libumem.so", RTLD_LAZY); + if (libumem_hndl == NULL) ARGV_ERR("(-sumem) cannot open libumem.so: %s", dlerror()); #define DLSYM_UMEM(fptr,sym) \ @@ -282,6 +284,9 @@ smu_init(struct stevedore *parent, int ac, char * const *av) #undef DLSYM_UMEM + AZ(dlclose(libumem_hndl)); + libumem_hndl = NULL; + e = VNUM_2bytes(av[0], &u, 0); if (e != NULL) ARGV_ERR("(-sumem) size \"%s\": %s\n", av[0], e); From nils.goroll at uplex.de Wed Apr 4 14:49:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 4 Apr 2018 14:49:11 +0000 (UTC) Subject: [master] e241de0 umem: default options, warn about umem loaded, lazy loading Message-ID: <20180404144911.3E857B0D42@lists.varnish-cache.org> commit e241de074fd3b47437bba0ddb8449fb755102f51 Author: Nils Goroll Date: Wed Apr 4 16:13:14 2018 +0200 umem: default options, warn about umem loaded, lazy loading default options: For co-existance with libc malloc and potentially other allocators, we should use mmap() regions in favor of sbrk(). Also, libumem's perthread cache only makes sense when umem is actually used for all allocations and, by default, requires an additional massive 1MB per thread, which is totally out of scale with how we manage memory in varnish. if libumem is found to already be loaded, we cannot set its options, so warn about that fact. As we only use function pointers to libumem, lazy loading of symbols should be the better option. diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index 7464a19..5471331 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "storage/storage.h" #include "storage/storage_simple.h" @@ -93,6 +94,9 @@ static umem_cache_destroy_f umem_cache_destroyf = NULL; static umem_cache_alloc_f umem_cache_allocf = NULL; static umem_cache_free_f umem_cache_freef = NULL; +static const char * const def_umem_options = "perthread_cache=0,backend=mmap"; +static const char * const env_umem_options = "UMEM_OPTIONS"; + /* init required per cache get: smu->sz = size smu->s.ptr; @@ -241,8 +245,46 @@ smu_free_space(const struct stevedore *st) } static void +smu_umem_loaded_warn(void) +{ + const char *e; + static int warned = 0; + + if (warned++) + return; + + fprintf(stderr, "notice:\tlibumem was already found to be loaded\n" + "\tand will likely be used for all allocations\n"); + + e = getenv(env_umem_options); + if (e == NULL || ! strstr(e, def_umem_options)) + fprintf(stderr, "\tit is recommended to set %s=%s " + "before starting varnish\n", + env_umem_options, def_umem_options); +} + +static int +smu_umem_loaded(void) +{ + void *h = NULL; + + h = dlopen("libumem.so", RTLD_NOLOAD); + if (h) { + AZ(dlclose(h)); + return (1); + } + + h = dlsym(RTLD_DEFAULT, "umem_alloc"); + if (h) + return (1); + + return (0); +} + +static void smu_init(struct stevedore *parent, int ac, char * const *av) { + static int inited = 0; const char *e; uintmax_t u; struct smu_sc *sc; @@ -258,8 +300,25 @@ smu_init(struct stevedore *parent, int ac, char * const *av) if (ac > 1) ARGV_ERR("(-sumem) too many arguments\n"); - if (ac == 0 || *av[0] == '\0') - return; + if (ac == 1 && *av[0] != '\0') { + e = VNUM_2bytes(av[0], &u, 0); + if (e != NULL) + ARGV_ERR("(-sumem) size \"%s\": %s\n", av[0], e); + if ((u != (uintmax_t)(size_t)u)) + ARGV_ERR("(-sumem) size \"%s\": too big\n", av[0]); + if (u < 1024*1024) + ARGV_ERR("(-sumem) size \"%s\": too smull, " + "did you forget to specify M or G?\n", av[0]); + sc->smu_max = u; + } + + if (inited++) + return; + + if (smu_umem_loaded()) + smu_umem_loaded_warn(); + else + AZ(setenv(env_umem_options, def_umem_options, 0)); /* Check if these load in the management process. */ (void) dlerror(); @@ -286,17 +345,45 @@ smu_init(struct stevedore *parent, int ac, char * const *av) AZ(dlclose(libumem_hndl)); libumem_hndl = NULL; +} + +/* + * Load the symbols for use in the child process, assert if they fail to load. + */ +static void +smu_open_init(void) +{ + static int inited = 0; + + if (inited++) { + AN(libumem_hndl); + AN(umem_allocf); + return; + } + + if (smu_umem_loaded()) + smu_umem_loaded_warn(); + else + AN(getenv(env_umem_options)); + + AZ(libumem_hndl); + libumem_hndl = dlopen("libumem.so", RTLD_LAZY); + AN(libumem_hndl); - e = VNUM_2bytes(av[0], &u, 0); - if (e != NULL) - ARGV_ERR("(-sumem) size \"%s\": %s\n", av[0], e); - if ((u != (uintmax_t)(size_t)u)) - ARGV_ERR("(-sumem) size \"%s\": too big\n", av[0]); - if (u < 1024*1024) - ARGV_ERR("(-sumem) size \"%s\": too smull, " - "did you forget to specify M or G?\n", av[0]); +#define DLSYM_UMEM(fptr,sym) \ + do { \ + fptr = dlsym(libumem_hndl, #sym); \ + AN(fptr); \ + } while(0) - sc->smu_max = u; + DLSYM_UMEM(umem_allocf, umem_alloc); + DLSYM_UMEM(umem_freef, umem_free); + DLSYM_UMEM(umem_cache_createf, umem_cache_create); + DLSYM_UMEM(umem_cache_destroyf, umem_cache_destroy); + DLSYM_UMEM(umem_cache_allocf, umem_cache_alloc); + DLSYM_UMEM(umem_cache_freef, umem_cache_free); + +#undef DLSYM_UMEM } static void v_matchproto_(storage_open_f) @@ -314,30 +401,7 @@ smu_open(struct stevedore *st) if (smu_sc->smu_max != SIZE_MAX) smu_sc->stats->g_space = smu_sc->smu_max; - /* - * Load the symbols for use in the child process, assert if they - * fail to load. - */ - if (libumem_hndl == NULL) { - libumem_hndl = dlopen("libumem.so", RTLD_NOW); - AN(libumem_hndl); - -#define DLSYM_UMEM(fptr,sym) \ - do { \ - fptr = dlsym(libumem_hndl, #sym); \ - AN(fptr); \ - } while(0) - - DLSYM_UMEM(umem_allocf, umem_alloc); - DLSYM_UMEM(umem_freef, umem_free); - DLSYM_UMEM(umem_cache_createf, umem_cache_create); - DLSYM_UMEM(umem_cache_destroyf, umem_cache_destroy); - DLSYM_UMEM(umem_cache_allocf, umem_cache_alloc); - DLSYM_UMEM(umem_cache_freef, umem_cache_free); - -#undef DLSYM_UMEM - - } + smu_open_init(); smu_sc->smu_cache = umem_cache_createf(st->ident, sizeof(struct smu), From nils.goroll at uplex.de Wed Apr 4 14:49:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 4 Apr 2018 14:49:11 +0000 (UTC) Subject: [master] 929dd59 document umem specifics Message-ID: <20180404144911.590FBB0D46@lists.varnish-cache.org> commit 929dd59c27383ee344f6b8654afa2c496aadd5e3 Author: Nils Goroll Date: Wed Apr 4 16:47:45 2018 +0200 document umem specifics diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index b6d383a..de7ee86 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -320,6 +320,9 @@ The following storage types are available: umem is a storage backend which is more efficient than malloc on platforms where it is available. + See the section on umem in chapter `Storage backends` of `The + Varnish Users Guide` for details. + -s The file backend stores data in a file on disk. The file will be diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst index a631106..3dd7140 100644 --- a/doc/sphinx/users-guide/storage-backends.rst +++ b/doc/sphinx/users-guide/storage-backends.rst @@ -53,6 +53,8 @@ malloc's performance is bound to memory speed so it is very fast. If the dataset is bigger than available memory performance will depend on the operating systems ability to page effectively. +.. _guide-storage_umem: + umem ~~~~ @@ -69,6 +71,37 @@ implementations. In particular, `libumem`_ is included in the family of OpenSolaris descendent operating systems where jemalloc(3) is not commonly available. +If `libumem`_ is not used otherwise, varnish will only use it for +storage allocations and keep the default libc allocator for all other +varnish memory allocation purposes. + +If `libumem`_ is already loaded when varnish initializes, this message +is output:: + + notice: libumem was already found to be loaded + and will likely be used for all allocations + +to indicate that `libumem`_ will not only be used for storage. Likely +reasons for this to be the case are: + +* some library ``varnishd`` is linked against was linked against + `libumem`_ (most likely ``libpcre``, check with ``ldd``) + +* ``LD_PRELOAD_64=/usr/lib/amd64/libumem.so.1``, + ``LD_PRELOAD_32=/usr/lib/libumem.so.1`` or + ``LD_PRELOAD=/usr/lib/libumem.so.1`` is set + +varnish will also output this message to recommend settings for using +`libumem`_ for all allocations:: + + it is recommended to set UMEM_OPTIONS=perthread_cache=0,backend=mmap + before starting varnish + +This recommendation should be followed to achieve an optimal +`libumem`_ configuration for varnish. Setting this environment +variable before starting varnish is required becuase `libumem`_ cannot +be reconfigured once loaded. + .. _libumem: http://dtrace.org/blogs/ahl/2004/07/13/number-11-of-20-libumem/ file From nils.goroll at uplex.de Wed Apr 4 15:08:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 4 Apr 2018 15:08:10 +0000 (UTC) Subject: [master] bc35cd7 no detail too small Message-ID: <20180404150810.26128B14CA@lists.varnish-cache.org> commit bc35cd716ce1849afe5f16e581e6fe325ded8fda Author: Nils Goroll Date: Wed Apr 4 17:06:58 2018 +0200 no detail too small diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index 5471331..db8178a 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -307,7 +307,7 @@ smu_init(struct stevedore *parent, int ac, char * const *av) if ((u != (uintmax_t)(size_t)u)) ARGV_ERR("(-sumem) size \"%s\": too big\n", av[0]); if (u < 1024*1024) - ARGV_ERR("(-sumem) size \"%s\": too smull, " + ARGV_ERR("(-sumem) size \"%s\": too small, " "did you forget to specify M or G?\n", av[0]); sc->smu_max = u; } From phk at FreeBSD.org Wed Apr 4 21:49:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Apr 2018 21:49:10 +0000 (UTC) Subject: [master] 5e84ab6 Bite the bullet and import Ed Schouten's terminal emulation "Teken" Message-ID: <20180404214910.5774661B46@lists.varnish-cache.org> commit 5e84ab6e75f59f46df7497cc6afd2f5cc5fac48d Author: Poul-Henning Kamp Date: Wed Apr 4 21:37:01 2018 +0000 Bite the bullet and import Ed Schouten's terminal emulation "Teken" This allows us to use TERM=xterm. diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index c2c81d7..06f2259 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -30,6 +30,12 @@ varnishtest_SOURCES = \ cmds.h \ vmods.h \ vtc.h \ + teken.c \ + teken.h \ + teken_scs.h \ + teken_subr.h \ + teken_subr_compat.h \ + teken_wcwidth.h \ vtc.c \ vtc_barrier.c \ vtc_client.c \ @@ -51,7 +57,6 @@ varnishtest_SOURCES = \ vtc_proxy.c \ vtc_server.c \ vtc_subr.c \ - vtc_term.c \ vtc_varnish.c varnishtest_LDADD = \ @@ -67,4 +72,16 @@ varnishtest_CFLAGS = \ -DTOP_BUILDDIR='"${top_builddir}"' EXTRA_DIST = $(top_srcdir)/bin/varnishtest/tests/*.vtc \ - $(top_srcdir)/bin/varnishtest/tests/README + $(top_srcdir)/bin/varnishtest/tests/README \ + $(top_srcdir)/bin/varnishtest/gensequences \ + $(top_srcdir)/bin/varnishtest/sequences \ + $(top_srcdir)/bin/varnishtest/teken.3 + +teken_state.h: $(srcdir)/sequences $(srcdir)/gensequences + awk -f $(srcdir)/gensequences $(srcdir)/sequences \ + > $(builddir)/teken_state.h + +BUILT_SOURCES = teken_state.h + +CLEANFILES = $(BUILT_SOURCES) + diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt index 25eace2..fa51cc8 100644 --- a/bin/varnishtest/flint.lnt +++ b/bin/varnishtest/flint.lnt @@ -1,4 +1,6 @@ ++libh(teken/teken.h) + // Tell FlexeLint when these don't return -function(exit, vtc_fatal) -function(__assert(1), vtc_log(2)) @@ -8,6 +10,8 @@ -emacro({779}, ENC) // String constant in comparison operator '!=' -emacro({506}, CHKFRAME) // Constant value Boolean +-esym(522, teken_subr_*) + -esym(850, av) -esym(534, snprintf) // Only for varnishtest, and not really nice diff --git a/bin/varnishtest/flint.sh b/bin/varnishtest/flint.sh index 49f5e2d..7cc4f26 100755 --- a/bin/varnishtest/flint.sh +++ b/bin/varnishtest/flint.sh @@ -4,6 +4,7 @@ FLOPS=' -DTOP_BUILDDIR="foo" -I../../lib/libvgz *.c + teken/teken.c ' . ../../tools/flint_skel.sh diff --git a/bin/varnishtest/gensequences b/bin/varnishtest/gensequences new file mode 100644 index 0000000..83a3d10 --- /dev/null +++ b/bin/varnishtest/gensequences @@ -0,0 +1,157 @@ +#!/usr/bin/awk -f + +#- +# Copyright (c) 2008-2009 Ed Schouten +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD: head/sys/teken/gensequences 223574 2011-06-26 18:25:10Z ed $ + +function die(msg) { + print msg; + exit 1; +} + +function cchar(str) { + if (str == "^[") + return "\\x1B"; + + return str; +} + +BEGIN { +FS = "\t+" + +while (getline > 0) { + if (NF == 0 || $1 ~ /^#/) + continue; + + if (NF != 3 && NF != 4) + die("Invalid line layout: " NF " columns"); + + split($3, sequence, " +"); + nsequences = 0; + for (s in sequence) + nsequences++; + + prefix = ""; + l_prefix_name[""] = "teken_state_init"; + for (i = 1; i < nsequences; i++) { + n = prefix sequence[i]; + l_prefix_parent[n] = prefix; + l_prefix_suffix[n] = sequence[i]; + if (!l_prefix_name[n]) + l_prefix_name[n] = "teken_state_" ++npr; + prefix = n; + } + + suffix = sequence[nsequences]; + cmd = prefix suffix; + + # Fill lists + if (l_cmd_name[cmd] != "") + die(cmd " already exists"); + l_cmd_prefix[cmd] = prefix; + l_cmd_suffix[cmd] = suffix; + l_cmd_args[cmd] = $4; + l_cmd_abbr[cmd] = $1; + l_cmd_name[cmd] = $2; + l_cmd_c_name[cmd] = "teken_subr_" tolower($2); + gsub(" ", "_", l_cmd_c_name[cmd]); + + if ($4 != "") + l_prefix_numbercmds[prefix]++; +} + +print "/* Generated file. Do not edit. */"; +print ""; + +for (p in l_prefix_name) { + if (l_prefix_name[p] != "teken_state_init") + print "static teken_state_t " l_prefix_name[p] ";"; +} + +for (p in l_prefix_name) { + print ""; + print "/* '" p "' */"; + print "static void"; + print l_prefix_name[p] "(teken_t *t, teken_char_t c)"; + print "{"; + + if (l_prefix_numbercmds[p] > 0) { + print ""; + print "\tif (teken_state_numbers(t, c))"; + print "\t\treturn;"; + } + + print ""; + print "\tswitch (c) {"; + for (c in l_cmd_prefix) { + if (l_cmd_prefix[c] != p) + continue; + + print "\tcase '" cchar(l_cmd_suffix[c]) "': /* " l_cmd_abbr[c] ": " l_cmd_name[c] " */"; + + if (l_cmd_args[c] == "v") { + print "\t\t" l_cmd_c_name[c] "(t, t->t_curnum, t->t_nums);"; + } else { + printf "\t\t%s(t", l_cmd_c_name[c]; + split(l_cmd_args[c], args, " "); + for (a = 1; args[a] != ""; a++) { + if (args[a] == "n") + printf ", (t->t_curnum < %d || t->t_nums[%d] == 0) ? 1 : t->t_nums[%d]", a, (a - 1), (a - 1); + else if (args[a] == "r") + printf ", t->t_curnum < %d ? 0 : t->t_nums[%d]", a, (a - 1); + else + die("Invalid argument type: " args[a]); + } + print ");"; + } + print "\t\tbreak;"; + } + for (pc in l_prefix_parent) { + if (l_prefix_parent[pc] != p) + continue; + print "\tcase '" cchar(l_prefix_suffix[pc]) "':"; + print "\t\tteken_state_switch(t, " l_prefix_name[pc] ");"; + print "\t\treturn;"; + } + + print "\tdefault:"; + if (l_prefix_name[p] == "teken_state_init") { + print "\t\tteken_subr_regular_character(t, c);"; + } else { + print "\t\tteken_printf(\"Unsupported sequence in " l_prefix_name[p] ": %u\\n\", (unsigned int)c);"; + } + print "\t\tbreak;"; + + print "\t}"; + + if (l_prefix_name[p] != "teken_state_init") { + print ""; + print "\tteken_state_switch(t, teken_state_init);"; + } + print "}"; +} + +} diff --git a/bin/varnishtest/sequences b/bin/varnishtest/sequences new file mode 100644 index 0000000..af92df0 --- /dev/null +++ b/bin/varnishtest/sequences @@ -0,0 +1,115 @@ +#- +# Copyright (c) 2008-2009 Ed Schouten +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD: head/sys/teken/sequences 322662 2017-08-18 15:40:40Z bde $ + +# File format is as follows: +# Abbr Abbreviation of sequence name +# Name Sequence name (will be converted to C function name) +# Sequence Bytes that form the sequence +# Arguments Standard value of arguments passed to this sequence +# - `n' non-zero number (0 gets converted to 1) +# - `r' regular numeric argument +# - `v' means a variable number of arguments + +# Abbr Name Sequence Arguments +CBT Cursor Backward Tabulation ^[ [ Z n +CHT Cursor Forward Tabulation ^[ [ I n +CNL Cursor Next Line ^[ [ E n +CPL Cursor Previous Line ^[ [ F n +CPR Cursor Position Report ^[ [ n r +CUB Cursor Backward ^[ [ D n +CUD Cursor Down ^[ [ B n +CUD Cursor Down ^[ [ e n +CUF Cursor Forward ^[ [ C n +CUF Cursor Forward ^[ [ a n +CUP Cursor Position ^[ [ H n n +CUP Cursor Position ^[ [ f n n +CUU Cursor Up ^[ [ A n +DA1 Primary Device Attributes ^[ [ c r +DA2 Secondary Device Attributes ^[ [ > c r +DC Delete character ^[ [ P n +DCS Device Control String ^[ P +DECALN Alignment test ^[ # 8 +DECDHL Double Height Double Width Line Top ^[ # 3 +DECDHL Double Height Double Width Line Bottom ^[ # 4 +DECDWL Single Height Double Width Line ^[ # 6 +DECKPAM Keypad application mode ^[ = +DECKPNM Keypad numeric mode ^[ > +DECRC Restore cursor ^[ 8 +DECRC Restore cursor ^[ [ u +DECRM Reset DEC mode ^[ [ ? l r +DECSC Save cursor ^[ 7 +DECSC Save cursor ^[ [ s +DECSM Set DEC mode ^[ [ ? h r +DECSTBM Set top and bottom margins ^[ [ r r r +DECSWL Single Height Single Width Line ^[ # 5 +DL Delete line ^[ [ M n +DSR Device Status Report ^[ [ ? n r +ECH Erase character ^[ [ X n +ED Erase display ^[ [ J r +EL Erase line ^[ [ K r +G0SCS0 G0 SCS Special Graphics ^[ ( 0 +G0SCS1 G0 SCS US ASCII ^[ ( 1 +G0SCS2 G0 SCS Special Graphics ^[ ( 2 +G0SCSA G0 SCS UK National ^[ ( A +G0SCSB G0 SCS US ASCII ^[ ( B +G1SCS0 G1 SCS Special Graphics ^[ ) 0 +G1SCS1 G1 SCS US ASCII ^[ ) 1 +G1SCS2 G1 SCS Special Graphics ^[ ) 2 +G1SCSA G1 SCS UK National ^[ ) A +G1SCSB G1 SCS US ASCII ^[ ) B +HPA Horizontal Position Absolute ^[ [ G n +HPA Horizontal Position Absolute ^[ [ ` n +HTS Horizontal Tab Set ^[ H +ICH Insert character ^[ [ @ n +IL Insert line ^[ [ L n +IND Index ^[ D +NEL Next line ^[ E +OSC Operating System Command ^[ ] +RI Reverse index ^[ M +RIS Reset to Initial State ^[ c +RM Reset Mode ^[ [ l r +SD Pan Up ^[ [ T n +SGR Set Graphic Rendition ^[ [ m v +SM Set Mode ^[ [ h r +ST String Terminator ^[ \\ +SU Pan Down ^[ [ S n +TBC Tab Clear ^[ [ g r +VPA Vertical Position Absolute ^[ [ d n + +# Cons25 compatibility sequences +C25BLPD Cons25 set bell pitch duration ^[ [ = B r r +C25BORD Cons25 set border ^[ [ = A r +C25DBG Cons25 set default background ^[ [ = G r +C25DFG Cons25 set default foreground ^[ [ = F r +C25GCS Cons25 set global cursor shape ^[ [ = C v +C25LCT Cons25 set local cursor type ^[ [ = S r +C25MODE Cons25 set terminal mode ^[ [ = T r +C25SGR Cons25 set graphic rendition ^[ [ x r r +C25VTSW Cons25 switch virtual terminal ^[ [ z r + +# VT52 compatibility +#DECID VT52 DECID ^[ Z diff --git a/bin/varnishtest/teken.3 b/bin/varnishtest/teken.3 new file mode 100644 index 0000000..c1ece85 --- /dev/null +++ b/bin/varnishtest/teken.3 @@ -0,0 +1,234 @@ +.\" Copyright (c) 2011 Ed Schouten +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD: head/sys/teken/libteken/teken.3 315418 2017-03-16 16:40:54Z bde $ +.\" +.Dd Mar 13, 2017 +.Dt TEKEN 3 +.Os +.Sh NAME +.Nm teken +.Nd xterm-like terminal emulation interface +.Sh LIBRARY +.Lb libteken +.Sh SYNOPSIS +.In teken.h +.Ft void +.Fn teken_init "teken_t *t" "const teken_funcs_t *funcs" "void *thunk" +.Ft void +.Fn teken_input "teken_t *t" "const void *buf" "size_t nbytes" +.Ft const teken_pos_t * +.Fn teken_get_winsize "teken_t *t" +.Ft void +.Fn teken_set_winsize "teken_t *t" "const teken_pos_t *size" +.Ft const teken_pos_t * +.Fn teken_get_cursor "teken_t *t" +.Ft void +.Fn teken_set_cursor "teken_t *t" "const teken_pos_t *pos" +.Ft const teken_attr_t * +.Fn teken_get_curattr "teken_t *t" +.Ft void +.Fn teken_set_curattr "teken_t *t" "const teken_attr_t *attr" +.Ft const teken_attr_t * +.Fn teken_get_defattr "teken_t *t" +.Ft void +.Fn teken_set_defattr "teken_t *t" "const teken_attr_t *attr" +.Ft const char * +.Fn teken_get_sequence "teken_t *t" "unsigned int id" +.Ft teken_color_t +.Fn teken_256to16 "teken_color_t color" +.Ft teken_color_t +.Fn teken_256to8 "teken_color_t color" +.Ft void +.Fn teken_get_defattr_cons25 "teken_t *t" "int *fg" "int *bg" +.Ft void +.Fn teken_set_8bit "teken_t *t" +.Ft void +.Fn teken_set_cons25 "teken_t *t" +.Sh DESCRIPTION +The +.Nm +library implements the input parser of a 256-color xterm-like terminal. +It converts a stream of UTF-8 encoded characters into a series of +primitive drawing instructions that can be used by a console driver or +terminal emulator to render a terminal application. +.Pp +The +.Fn teken_init +function is used to initialize terminal state object +.Fa t , +having type +.Vt teken_t . +The supplied +.Vt teken_funcs_t +structure +.Fa funcs +contains a set of callback functions, which are called when supplying +data to +.Fn teken_input . +The +.Fa thunk +argument stores an arbitrary pointer, which is passed to each invocation +of the callback functions. +.Pp +The +.Vt teken_funcs_t +structure stores the following callbacks: +.Bd -literal -offset indent +typedef struct { + tf_bell_t *tf_bell; /* Audible/visible bell. */ + tf_cursor_t *tf_cursor; /* Move cursor to x/y. */ + tf_putchar_t *tf_putchar; /* Put Unicode character at x/y. */ + tf_fill_t *tf_fill; /* Fill rectangle with character. */ + tf_copy_t *tf_copy; /* Copy rectangle to new location. */ + tf_param_t *tf_param; /* Miscellaneous options. */ + tf_respond_t *tf_respond; /* Send response string to user. */ +} teken_funcs_t; +.Ed +.Pp +All callbacks must be provided, though unimplemented callbacks may some +times be sufficient. +The actual types of these callbacks can be found in +.In teken.h . +.Pp +By default, +.Fn teken_init +initializes the +.Vt teken_t +structure to emulate a terminal having 24 rows and 80 columns. +The +.Fn teken_get_winsize +and +.Fn teken_set_winsize +functions can be used to obtain and modify the dimensions of the +terminal. +.Pp +Even though the cursor position is normally controlled by input of data +through +.Fn teken_input +and returned by the +.Fn tf_cursor +callback, it can be obtained and modified manually using the +.Fn teken_get_cursor +and +.Fn teken_set_cursor +functions. +The same holds for +.Fn teken_get_curattr +and +.Fn teken_set_curattr , +which can be used to change the currently selected font attributes and +foreground and background color. +.Pp +By default, +.Nm +emulates a white-on-black terminal, which means the default foreground +color is white, while the background color is black. +These defaults can be modified using +.Fn teken_get_defattr +and +.Fn teken_set_defattr . +.Pp +The +.Fn teken_get_sequence +function is a utility function that can be used to obtain escape +sequences of special keyboard keys, generated by user input. +The +.Fa id +parameter must be one of the +.Dv TKEY_* +parameters listed in +.In teken.h . +.Sh LEGACY FEATURES +This library also provides a set of functions that shouldn't be used in +any modern applications. +.Pp +The +.Fn teken_256to16 +function converts an xterm-256 256-color code to an xterm 16-color code +whose color with default palettes is as similar as possible (not very +similar). +The lower 3 bits of the result are the ANSI color and the next lowest +bit is brightness. +Other layers (hardare and software) that only support 16 colors can use +this to avoid knowing the details of 256-color codes. +.Pp +The +.Fn teken_256to8 +function is similar to +.Fn teken_256to16 +except it converts to an ANSI 8-color code. +This is more accurate than discarding the brigtness bit in the result of +.Fn teken_256to16 . +.Pp +The +.Fn teken_get_defattr_cons25 +function obtains the default terminal attributes as a pair of foreground +and background colors, using ANSI color numbering. +.Pp +The +.Fn teken_set_8bit +function disables UTF-8 processing and switches to 8-bit character mode, +which can be used to support character sets like CP437 and ISO-8859-1. +.Pp +The +.Fn teken_set_cons25 +function switches terminal emulation to +.Dv cons25 , +which is used by versions of +.Fx +prior to 9.0. +.Sh SEE ALSO +.Xr ncurses 3 , +.Xr termcap 3 , +.Xr syscons 4 +.Sh HISTORY +The +.Nm +library appeared in +.Fx 8.0 , +though it was only available and used inside the kernel. +In +.Fx 9.0 , +the +.Nm +library appeared in userspace. +.Sh AUTHORS +.An Ed Schouten Aq ed at FreeBSD.org +.Sh SECURITY CONSIDERATIONS +The +.Fn tf_respond +callback is used to respond to device status requests commands generated +by an application. +In the past, there have been various security issues, where a malicious +application sends a device status request before termination, causing +the generated response to be interpreted by applications such as +.Xr sh 1 . +.Pp +.Nm +only implements a small subset of responses which are unlikely to cause +any harm. +Still, it is advised to leave +.Fn tf_respond +unimplemented. diff --git a/bin/varnishtest/teken.c b/bin/varnishtest/teken.c new file mode 100644 index 0000000..8f9076d --- /dev/null +++ b/bin/varnishtest/teken.c @@ -0,0 +1,719 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2008-2009 Ed Schouten + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: head/sys/teken/teken.c 326272 2017-11-27 15:23:17Z pfg $ + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "vdef.h" +#include "vas.h" + +#define teken_assert(x) assert(x) + +/* debug messages */ +#define teken_printf(x,...) + +/* Private flags for t_stateflags. */ +#define TS_FIRSTDIGIT 0x0001 /* First numeric digit in escape sequence. */ +#define TS_INSERT 0x0002 /* Insert mode. */ +#define TS_AUTOWRAP 0x0004 /* Autowrap. */ +#define TS_ORIGIN 0x0008 /* Origin mode. */ +#define TS_WRAPPED 0x0010 /* Next character should be printed on col 0. */ +#define TS_8BIT 0x0020 /* UTF-8 disabled. */ +#define TS_CONS25 0x0040 /* cons25 emulation. */ +#define TS_INSTRING 0x0080 /* Inside string. */ +#define TS_CURSORKEYS 0x0100 /* Cursor keys mode. */ + +/* Character that blanks a cell. */ +#define BLANK ' ' + +#include "teken.h" +#include "teken_wcwidth.h" +#include "teken_scs.h" + +static teken_state_t teken_state_init; + +/* + * Wrappers for hooks. + */ + +static inline void +teken_funcs_bell(const teken_t *t) +{ + + if (t->t_funcs->tf_bell != NULL) + t->t_funcs->tf_bell(t->t_softc); +} + +static inline void +teken_funcs_cursor(const teken_t *t) +{ + + teken_assert(t->t_cursor.tp_row < t->t_winsize.tp_row); + teken_assert(t->t_cursor.tp_col < t->t_winsize.tp_col); + + teken_assert(t->t_funcs->tf_cursor != NULL); + t->t_funcs->tf_cursor(t->t_softc, &t->t_cursor); +} + +static inline void +teken_funcs_putchar(const teken_t *t, const teken_pos_t *p, teken_char_t c, + const teken_attr_t *a) +{ + + teken_assert(p->tp_row < t->t_winsize.tp_row); + teken_assert(p->tp_col < t->t_winsize.tp_col); + + teken_assert(t->t_funcs->tf_putchar != NULL); + t->t_funcs->tf_putchar(t->t_softc, p, c, a); +} + +static inline void +teken_funcs_fill(const teken_t *t, const teken_rect_t *r, + const teken_char_t c, const teken_attr_t *a) +{ + + teken_assert(r->tr_end.tp_row > r->tr_begin.tp_row); + teken_assert(r->tr_end.tp_row <= t->t_winsize.tp_row); + teken_assert(r->tr_end.tp_col > r->tr_begin.tp_col); + teken_assert(r->tr_end.tp_col <= t->t_winsize.tp_col); + + teken_assert(t->t_funcs->tf_fill != NULL); + t->t_funcs->tf_fill(t->t_softc, r, c, a); +} + +static inline void +teken_funcs_copy(const teken_t *t, const teken_rect_t *r, const teken_pos_t *p) +{ + + teken_assert(r->tr_end.tp_row > r->tr_begin.tp_row); + teken_assert(r->tr_end.tp_row <= t->t_winsize.tp_row); + teken_assert(r->tr_end.tp_col > r->tr_begin.tp_col); + teken_assert(r->tr_end.tp_col <= t->t_winsize.tp_col); + teken_assert(p->tp_row + (r->tr_end.tp_row - r->tr_begin.tp_row) <= t->t_winsize.tp_row); + teken_assert(p->tp_col + (r->tr_end.tp_col - r->tr_begin.tp_col) <= t->t_winsize.tp_col); + + teken_assert(t->t_funcs->tf_copy != NULL); + t->t_funcs->tf_copy(t->t_softc, r, p); +} + +static inline void +teken_funcs_param(const teken_t *t, int cmd, unsigned int value) +{ + + if (t->t_funcs->tf_param != NULL) + t->t_funcs->tf_param(t->t_softc, cmd, value); +} + +static inline void +teken_funcs_respond(const teken_t *t, const void *buf, size_t len) +{ + + if (t->t_funcs->tf_respond != NULL) + t->t_funcs->tf_respond(t->t_softc, buf, len); +} + +#include "teken_subr.h" +#include "teken_subr_compat.h" + +/* + * Programming interface. + */ + +void +teken_init(teken_t *t, const teken_funcs_t *tf, void *softc) +{ + teken_pos_t tp = { .tp_row = 24, .tp_col = 80 }; + + t->t_funcs = tf; + t->t_softc = softc; + + t->t_nextstate = teken_state_init; + t->t_stateflags = 0; + t->t_utf8_left = 0; + + t->t_defattr.ta_format = 0; + t->t_defattr.ta_fgcolor = TC_WHITE; + t->t_defattr.ta_bgcolor = TC_BLACK; + teken_subr_do_reset(t); + + teken_set_winsize(t, &tp); +} + +static void +teken_input_char(teken_t *t, teken_char_t c) +{ + + /* + * There is no support for DCS and OSC. Just discard strings + * until we receive characters that may indicate string + * termination. + */ + if (t->t_stateflags & TS_INSTRING) { + switch (c) { + case '\x1B': + t->t_stateflags &= ~TS_INSTRING; + break; + case '\a': + t->t_stateflags &= ~TS_INSTRING; + return; + default: + return; + } + } + + switch (c) { + case '\0': + break; + case '\a': + teken_subr_bell(t); + break; + case '\b': + teken_subr_backspace(t); + break; + case '\n': + case '\x0B': + teken_subr_newline(t); + break; + case '\x0C': + teken_subr_newpage(t); + break; + case '\x0E': + if (t->t_stateflags & TS_CONS25) + t->t_nextstate(t, c); + else + t->t_curscs = 1; + break; + case '\x0F': + if (t->t_stateflags & TS_CONS25) + t->t_nextstate(t, c); + else + t->t_curscs = 0; + break; + case '\r': + teken_subr_carriage_return(t); + break; + case '\t': + teken_subr_horizontal_tab(t); + break; + default: + t->t_nextstate(t, c); + break; + } + + /* Post-processing assertions. */ + teken_assert(t->t_cursor.tp_row >= t->t_originreg.ts_begin); + teken_assert(t->t_cursor.tp_row < t->t_originreg.ts_end); + teken_assert(t->t_cursor.tp_row < t->t_winsize.tp_row); + teken_assert(t->t_cursor.tp_col < t->t_winsize.tp_col); + teken_assert(t->t_saved_cursor.tp_row < t->t_winsize.tp_row); + teken_assert(t->t_saved_cursor.tp_col < t->t_winsize.tp_col); + teken_assert(t->t_scrollreg.ts_end <= t->t_winsize.tp_row); + teken_assert(t->t_scrollreg.ts_begin < t->t_scrollreg.ts_end); + /* Origin region has to be window size or the same as scrollreg. */ + teken_assert((t->t_originreg.ts_begin == t->t_scrollreg.ts_begin && + t->t_originreg.ts_end == t->t_scrollreg.ts_end) || + (t->t_originreg.ts_begin == 0 && + t->t_originreg.ts_end == t->t_winsize.tp_row)); +} + +static void +teken_input_byte(teken_t *t, unsigned char c) +{ + + /* + * UTF-8 handling. + */ + if ((c & 0x80) == 0x00 || t->t_stateflags & TS_8BIT) { + /* One-byte sequence. */ + t->t_utf8_left = 0; + teken_input_char(t, c); + } else if ((c & 0xe0) == 0xc0) { + /* Two-byte sequence. */ + t->t_utf8_left = 1; + t->t_utf8_partial = c & 0x1f; + } else if ((c & 0xf0) == 0xe0) { + /* Three-byte sequence. */ + t->t_utf8_left = 2; + t->t_utf8_partial = c & 0x0f; + } else if ((c & 0xf8) == 0xf0) { + /* Four-byte sequence. */ + t->t_utf8_left = 3; + t->t_utf8_partial = c & 0x07; + } else if ((c & 0xc0) == 0x80) { + if (t->t_utf8_left == 0) + return; + t->t_utf8_left--; + t->t_utf8_partial = (t->t_utf8_partial << 6) | (c & 0x3f); + if (t->t_utf8_left == 0) { + teken_printf("Got UTF-8 char %x\n", t->t_utf8_partial); + teken_input_char(t, t->t_utf8_partial); + } + } +} + +void +teken_input(teken_t *t, const void *buf, size_t len) +{ + const char *c = buf; + + while (len-- > 0) + teken_input_byte(t, *c++); +} + +const teken_pos_t * +teken_get_cursor(const teken_t *t) +{ + + return (&t->t_cursor); +} + +void +teken_set_cursor(teken_t *t, const teken_pos_t *p) +{ + + /* XXX: bounds checking with originreg! */ + teken_assert(p->tp_row < t->t_winsize.tp_row); + teken_assert(p->tp_col < t->t_winsize.tp_col); + + t->t_cursor = *p; +} + +const teken_attr_t * +teken_get_curattr(const teken_t *t) +{ + + return (&t->t_curattr); +} + +void +teken_set_curattr(teken_t *t, const teken_attr_t *a) +{ + + t->t_curattr = *a; +} + +const teken_attr_t * +teken_get_defattr(const teken_t *t) +{ + + return (&t->t_defattr); +} + +void +teken_set_defattr(teken_t *t, const teken_attr_t *a) +{ + + t->t_curattr = t->t_saved_curattr = t->t_defattr = *a; +} + +const teken_pos_t * +teken_get_winsize(const teken_t *t) +{ + + return (&t->t_winsize); +} + +static void +teken_trim_cursor_pos(teken_t *t, const teken_pos_t *new) +{ + const teken_pos_t *cur; + + cur = &t->t_winsize; + + if (cur->tp_row < new->tp_row || cur->tp_col < new->tp_col) + return; + if (t->t_cursor.tp_row >= new->tp_row) + t->t_cursor.tp_row = new->tp_row - 1; + if (t->t_cursor.tp_col >= new->tp_col) + t->t_cursor.tp_col = new->tp_col - 1; +} + +void +teken_set_winsize(teken_t *t, const teken_pos_t *p) +{ + + teken_trim_cursor_pos(t, p); + t->t_winsize = *p; + teken_subr_do_reset(t); +} + +void +teken_set_winsize_noreset(teken_t *t, const teken_pos_t *p) +{ + + teken_trim_cursor_pos(t, p); + t->t_winsize = *p; + teken_subr_do_resize(t); +} + +void +teken_set_8bit(teken_t *t) +{ + + t->t_stateflags |= TS_8BIT; +} + +void +teken_set_cons25(teken_t *t) +{ + + t->t_stateflags |= TS_CONS25; +} + +/* + * State machine. + */ + +static void +teken_state_switch(teken_t *t, teken_state_t *s) +{ + + t->t_nextstate = s; + t->t_curnum = 0; + t->t_stateflags |= TS_FIRSTDIGIT; +} + +static int +teken_state_numbers(teken_t *t, teken_char_t c) +{ + + teken_assert(t->t_curnum < T_NUMSIZE); + + if (c >= '0' && c <= '9') { + if (t->t_stateflags & TS_FIRSTDIGIT) { + /* First digit. */ + t->t_stateflags &= ~TS_FIRSTDIGIT; + t->t_nums[t->t_curnum] = c - '0'; + } else if (t->t_nums[t->t_curnum] < UINT_MAX / 100) { + /* + * There is no need to continue parsing input + * once the value exceeds the size of the + * terminal. It would only allow for integer + * overflows when performing arithmetic on the + * cursor position. + * + * Ignore any further digits if the value is + * already UINT_MAX / 100. + */ + t->t_nums[t->t_curnum] = + t->t_nums[t->t_curnum] * 10 + c - '0'; + } + return (1); + } else if (c == ';') { + if (t->t_stateflags & TS_FIRSTDIGIT) + t->t_nums[t->t_curnum] = 0; + + /* Only allow a limited set of arguments. */ + if (++t->t_curnum == T_NUMSIZE) { + teken_state_switch(t, teken_state_init); + return (1); + } + + t->t_stateflags |= TS_FIRSTDIGIT; + return (1); + } else { + if (t->t_stateflags & TS_FIRSTDIGIT && t->t_curnum > 0) { + /* Finish off the last empty argument. */ + t->t_nums[t->t_curnum] = 0; + t->t_curnum++; + } else if ((t->t_stateflags & TS_FIRSTDIGIT) == 0) { + /* Also count the last argument. */ + t->t_curnum++; + } + } + + return (0); +} + +#define k TC_BLACK +#define b TC_BLUE +#define y TC_BROWN +#define c TC_CYAN +#define g TC_GREEN +#define m TC_MAGENTA +#define r TC_RED +#define w TC_WHITE +#define K (TC_BLACK | TC_LIGHT) +#define B (TC_BLUE | TC_LIGHT) +#define Y (TC_BROWN | TC_LIGHT) +#define C (TC_CYAN | TC_LIGHT) +#define G (TC_GREEN | TC_LIGHT) +#define M (TC_MAGENTA | TC_LIGHT) +#define R (TC_RED | TC_LIGHT) +#define W (TC_WHITE | TC_LIGHT) + +/** + * The xterm-256 color map has steps of 0x28 (in the range 0-0xff), except + * for the first step which is 0x5f. Scale to the range 0-6 by dividing + * by 0x28 and rounding down. The range of 0-5 cannot represent the + * larger first step. + * + * This table is generated by the follow rules: + * - if all components are equal, the result is black for (0, 0, 0) and + * (2, 2, 2), else white; otherwise: + * - subtract the smallest component from all components + * - if this gives only one nonzero component, then that is the color + * - else if one component is 2 or more larger than the other nonzero one, + * then that component gives the color + * - else there are 2 nonzero components. The color is that of a small + * equal mixture of these components (cyan, yellow or magenta). E.g., + * (0, 5, 6) (Turquoise2) is a much purer cyan than (0, 2, 3) + * (DeepSkyBlue4), but we map both to cyan since we can't represent + * delicate shades of either blue or cyan and blue would be worse. + * Here it is important that components of 1 never occur. Blue would + * be twice as large as green in (0, 1, 2). + */ +static const teken_color_t teken_256to8tab[] = { + /* xterm normal colors: */ + k, r, g, y, b, m, c, w, + + /* xterm bright colors: */ + k, r, g, y, b, m, c, w, + + /* Red0 submap. */ + k, b, b, b, b, b, + g, c, c, b, b, b, + g, c, c, c, b, b, + g, g, c, c, c, b, + g, g, g, c, c, c, + g, g, g, g, c, c, + + /* Red2 submap. */ + r, m, m, b, b, b, + y, k, b, b, b, b, + y, g, c, c, b, b, + g, g, c, c, c, b, + g, g, g, c, c, c, + g, g, g, g, c, c, + + /* Red3 submap. */ + r, m, m, m, b, b, + y, r, m, m, b, b, + y, y, w, b, b, b, + y, y, g, c, c, b, + g, g, g, c, c, c, + g, g, g, g, c, c, + + /* Red4 submap. */ + r, r, m, m, m, b, + r, r, m, m, m, b, + y, y, r, m, m, b, + y, y, y, w, b, b, + y, y, y, g, c, c, + g, g, g, g, c, c, + + /* Red5 submap. */ + r, r, r, m, m, m, + r, r, r, m, m, m, + r, r, r, m, m, m, + y, y, y, r, m, m, + y, y, y, y, w, b, + y, y, y, y, g, c, + + /* Red6 submap. */ + r, r, r, r, m, m, + r, r, r, r, m, m, + r, r, r, r, m, m, + r, r, r, r, m, m, + y, y, y, y, r, m, + y, y, y, y, y, w, + + /* Grey submap. */ + k, k, k, k, k, k, + k, k, k, k, k, k, + w, w, w, w, w, w, + w, w, w, w, w, w, +}; + +/* + * This table is generated from the previous one by setting TC_LIGHT for + * entries whose luminosity in the xterm256 color map is 60% or larger. + * Thus the previous table is currently not really needed. It will be + * used for different fine tuning of the tables. + */ +static const teken_color_t teken_256to16tab[] = { + /* xterm normal colors: */ + k, r, g, y, b, m, c, w, + + /* xterm bright colors: */ + K, R, G, Y, B, M, C, W, + + /* Red0 submap. */ + k, b, b, b, b, b, + g, c, c, b, b, b, + g, c, c, c, b, b, + g, g, c, c, c, b, + g, g, g, c, c, c, + g, g, g, g, c, c, + + /* Red2 submap. */ + r, m, m, b, b, b, + y, K, b, b, B, B, + y, g, c, c, B, B, + g, g, c, c, C, B, + g, G, G, C, C, C, + g, G, G, G, C, C, + + /* Red3 submap. */ + r, m, m, m, b, b, + y, r, m, m, B, B, + y, y, w, B, B, B, + y, y, G, C, C, B, + g, G, G, C, C, C, + g, G, G, G, C, C, + + /* Red4 submap. */ + r, r, m, m, m, b, + r, r, m, m, M, B, + y, y, R, M, M, B, + y, y, Y, W, B, B, + y, Y, Y, G, C, C, + g, G, G, G, C, C, + + /* Red5 submap. */ + r, r, r, m, m, m, + r, R, R, M, M, M, + r, R, R, M, M, M, + y, Y, Y, R, M, M, + y, Y, Y, Y, W, B, + y, Y, Y, Y, G, C, + + /* Red6 submap. */ + r, r, r, r, m, m, + r, R, R, R, M, M, + r, R, R, R, M, M, + r, R, R, R, M, M, + y, Y, Y, Y, R, M, + y, Y, Y, Y, Y, W, + + /* Grey submap. */ + k, k, k, k, k, k, + K, K, K, K, K, K, + w, w, w, w, w, w, + W, W, W, W, W, W, +}; + +#undef k +#undef b +#undef y +#undef c +#undef g +#undef m +#undef r +#undef w +#undef K +#undef B +#undef Y +#undef C +#undef G +#undef M +#undef R +#undef W + +teken_color_t +teken_256to8(teken_color_t c) +{ + + return (teken_256to8tab[c % 256]); +} + +teken_color_t +teken_256to16(teken_color_t c) +{ + + return (teken_256to16tab[c % 256]); +} + +static const char * const special_strings_cons25[] = { + [TKEY_UP] = "\x1B[A", [TKEY_DOWN] = "\x1B[B", + [TKEY_LEFT] = "\x1B[D", [TKEY_RIGHT] = "\x1B[C", + + [TKEY_HOME] = "\x1B[H", [TKEY_END] = "\x1B[F", + [TKEY_INSERT] = "\x1B[L", [TKEY_DELETE] = "\x7F", + [TKEY_PAGE_UP] = "\x1B[I", [TKEY_PAGE_DOWN] = "\x1B[G", + + [TKEY_F1] = "\x1B[M", [TKEY_F2] = "\x1B[N", + [TKEY_F3] = "\x1B[O", [TKEY_F4] = "\x1B[P", + [TKEY_F5] = "\x1B[Q", [TKEY_F6] = "\x1B[R", + [TKEY_F7] = "\x1B[S", [TKEY_F8] = "\x1B[T", + [TKEY_F9] = "\x1B[U", [TKEY_F10] = "\x1B[V", + [TKEY_F11] = "\x1B[W", [TKEY_F12] = "\x1B[X", +}; + +static const char * const special_strings_ckeys[] = { + [TKEY_UP] = "\x1BOA", [TKEY_DOWN] = "\x1BOB", + [TKEY_LEFT] = "\x1BOD", [TKEY_RIGHT] = "\x1BOC", + + [TKEY_HOME] = "\x1BOH", [TKEY_END] = "\x1BOF", +}; + +static const char * const special_strings_normal[] = { + [TKEY_UP] = "\x1B[A", [TKEY_DOWN] = "\x1B[B", + [TKEY_LEFT] = "\x1B[D", [TKEY_RIGHT] = "\x1B[C", + + [TKEY_HOME] = "\x1B[H", [TKEY_END] = "\x1B[F", + [TKEY_INSERT] = "\x1B[2~", [TKEY_DELETE] = "\x1B[3~", + [TKEY_PAGE_UP] = "\x1B[5~", [TKEY_PAGE_DOWN] = "\x1B[6~", + + [TKEY_F1] = "\x1BOP", [TKEY_F2] = "\x1BOQ", + [TKEY_F3] = "\x1BOR", [TKEY_F4] = "\x1BOS", + [TKEY_F5] = "\x1B[15~", [TKEY_F6] = "\x1B[17~", + [TKEY_F7] = "\x1B[18~", [TKEY_F8] = "\x1B[19~", + [TKEY_F9] = "\x1B[20~", [TKEY_F10] = "\x1B[21~", + [TKEY_F11] = "\x1B[23~", [TKEY_F12] = "\x1B[24~", +}; + +const char * +teken_get_sequence(const teken_t *t, unsigned int k) +{ + + /* Cons25 mode. */ + if (t->t_stateflags & TS_CONS25 && + k < sizeof special_strings_cons25 / sizeof(char *)) + return (special_strings_cons25[k]); + + /* Cursor keys mode. */ + if (t->t_stateflags & TS_CURSORKEYS && + k < sizeof special_strings_ckeys / sizeof(char *)) + return (special_strings_ckeys[k]); + + /* Default xterm sequences. */ + if (k < sizeof special_strings_normal / sizeof(char *)) + return (special_strings_normal[k]); + + return (NULL); +} + +#include "teken_state.h" diff --git a/bin/varnishtest/teken.h b/bin/varnishtest/teken.h new file mode 100644 index 0000000..986c3bd --- /dev/null +++ b/bin/varnishtest/teken.h @@ -0,0 +1,215 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2008-2009 Ed Schouten + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: head/sys/teken/teken.h 326272 2017-11-27 15:23:17Z pfg $ + */ + +#ifndef _TEKEN_H_ +#define _TEKEN_H_ + +#include + +/* + * libteken: terminal emulation library. + * + * This library converts an UTF-8 stream of bytes to terminal drawing + * commands. + */ + +typedef uint32_t teken_char_t; +typedef unsigned short teken_unit_t; +typedef unsigned char teken_format_t; +#define TF_BOLD 0x01 /* Bold character. */ +#define TF_UNDERLINE 0x02 /* Underline character. */ +#define TF_BLINK 0x04 /* Blinking character. */ +#define TF_REVERSE 0x08 /* Reverse rendered character. */ +#define TF_CJK_RIGHT 0x10 /* Right-hand side of CJK character. */ +typedef unsigned char teken_color_t; +#define TC_BLACK 0 +#define TC_RED 1 +#define TC_GREEN 2 +#define TC_BROWN 3 +#define TC_BLUE 4 +#define TC_MAGENTA 5 +#define TC_CYAN 6 +#define TC_WHITE 7 +#define TC_NCOLORS 8 +#define TC_LIGHT 8 /* ORed with the others. */ + +typedef struct { + teken_unit_t tp_row; + teken_unit_t tp_col; +} teken_pos_t; +typedef struct { + teken_pos_t tr_begin; + teken_pos_t tr_end; +} teken_rect_t; +typedef struct { + teken_format_t ta_format; + teken_color_t ta_fgcolor; + teken_color_t ta_bgcolor; +} teken_attr_t; +typedef struct { + teken_unit_t ts_begin; + teken_unit_t ts_end; +} teken_span_t; + +typedef struct __teken teken_t; + +typedef void teken_state_t(teken_t *, teken_char_t); + +/* + * Drawing routines supplied by the user. + */ + +typedef void tf_bell_t(void *); +typedef void tf_cursor_t(void *, const teken_pos_t *); +typedef void tf_putchar_t(void *, const teken_pos_t *, teken_char_t, + const teken_attr_t *); +typedef void tf_fill_t(void *, const teken_rect_t *, teken_char_t, + const teken_attr_t *); +typedef void tf_copy_t(void *, const teken_rect_t *, const teken_pos_t *); +typedef void tf_param_t(void *, int, unsigned int); +#define TP_SHOWCURSOR 0 +#define TP_KEYPADAPP 1 +#define TP_AUTOREPEAT 2 +#define TP_SWITCHVT 3 +#define TP_132COLS 4 +#define TP_SETBELLPD 5 +#define TP_SETBELLPD_PITCH(pd) ((pd) >> 16) +#define TP_SETBELLPD_DURATION(pd) ((pd) & 0xffff) +#define TP_MOUSE 6 +#define TP_SETBORDER 7 +#define TP_SETLOCALCURSOR 8 +#define TP_SETGLOBALCURSOR 9 +typedef void tf_respond_t(void *, const void *, size_t); + +typedef struct { + tf_bell_t *tf_bell; + tf_cursor_t *tf_cursor; + tf_putchar_t *tf_putchar; + tf_fill_t *tf_fill; + tf_copy_t *tf_copy; + tf_param_t *tf_param; + tf_respond_t *tf_respond; +} teken_funcs_t; + +typedef teken_char_t teken_scs_t(const teken_t *, teken_char_t); + +/* + * Terminal state. + */ + +struct __teken { + const teken_funcs_t *t_funcs; + void *t_softc; + + teken_state_t *t_nextstate; + unsigned int t_stateflags; + +#define T_NUMSIZE 8 + unsigned int t_nums[T_NUMSIZE]; + unsigned int t_curnum; + + teken_pos_t t_cursor; + teken_attr_t t_curattr; + teken_pos_t t_saved_cursor; + teken_attr_t t_saved_curattr; + + teken_attr_t t_defattr; + teken_pos_t t_winsize; + + /* For DECSTBM. */ + teken_span_t t_scrollreg; + /* For DECOM. */ + teken_span_t t_originreg; + +#define T_NUMCOL 160 + unsigned int t_tabstops[T_NUMCOL / (sizeof(unsigned int) * 8)]; + + unsigned int t_utf8_left; + teken_char_t t_utf8_partial; + + unsigned int t_curscs; + teken_scs_t *t_saved_curscs; + teken_scs_t *t_scs[2]; +}; + +/* Initialize teken structure. */ +void teken_init(teken_t *, const teken_funcs_t *, void *); + +/* Deliver character input. */ +void teken_input(teken_t *, const void *, size_t); + +/* Get/set teken attributes. */ +const teken_pos_t *teken_get_cursor(const teken_t *); +const teken_attr_t *teken_get_curattr(const teken_t *); +const teken_attr_t *teken_get_defattr(const teken_t *); +void teken_get_defattr_cons25(const teken_t *, int *, int *); +const teken_pos_t *teken_get_winsize(const teken_t *); +void teken_set_cursor(teken_t *, const teken_pos_t *); +void teken_set_curattr(teken_t *, const teken_attr_t *); +void teken_set_defattr(teken_t *, const teken_attr_t *); +void teken_set_winsize(teken_t *, const teken_pos_t *); +void teken_set_winsize_noreset(teken_t *, const teken_pos_t *); + +/* Key input escape sequences. */ +#define TKEY_UP 0x00 +#define TKEY_DOWN 0x01 +#define TKEY_LEFT 0x02 +#define TKEY_RIGHT 0x03 + +#define TKEY_HOME 0x04 +#define TKEY_END 0x05 +#define TKEY_INSERT 0x06 +#define TKEY_DELETE 0x07 +#define TKEY_PAGE_UP 0x08 +#define TKEY_PAGE_DOWN 0x09 + +#define TKEY_F1 0x0a +#define TKEY_F2 0x0b +#define TKEY_F3 0x0c +#define TKEY_F4 0x0d +#define TKEY_F5 0x0e +#define TKEY_F6 0x0f +#define TKEY_F7 0x10 +#define TKEY_F8 0x11 +#define TKEY_F9 0x12 +#define TKEY_F10 0x13 +#define TKEY_F11 0x14 +#define TKEY_F12 0x15 +const char *teken_get_sequence(const teken_t *, unsigned int); + +/* Legacy features. */ +void teken_set_8bit(teken_t *); +void teken_set_cons25(teken_t *); + +/* Color conversion. */ +teken_color_t teken_256to16(teken_color_t); +teken_color_t teken_256to8(teken_color_t); + +#endif /* !_TEKEN_H_ */ diff --git a/bin/varnishtest/teken_scs.h b/bin/varnishtest/teken_scs.h new file mode 100644 index 0000000..fd99de1 --- /dev/null +++ b/bin/varnishtest/teken_scs.h @@ -0,0 +1,83 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2009 Ed Schouten + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: head/sys/teken/teken_scs.h 326272 2017-11-27 15:23:17Z pfg $ + */ + +static inline teken_char_t +teken_scs_process(const teken_t *t, teken_char_t c) +{ + + return (t->t_scs[t->t_curscs](t, c)); +} + +/* Unicode points for VT100 box drawing. */ +static const uint16_t teken_boxdrawing_unicode[31] = { + 0x25c6, 0x2592, 0x2409, 0x240c, 0x240d, 0x240a, 0x00b0, 0x00b1, + 0x2424, 0x240b, 0x2518, 0x2510, 0x250c, 0x2514, 0x253c, 0x23ba, + 0x23bb, 0x2500, 0x23bc, 0x23bd, 0x251c, 0x2524, 0x2534, 0x252c, + 0x2502, 0x2264, 0x2265, 0x03c0, 0x2260, 0x00a3, 0x00b7 +}; + +/* ASCII points for VT100 box drawing. */ +static const uint8_t teken_boxdrawing_8bit[31] = { + '?', '?', 'H', 'F', 'C', 'L', '?', '?', + 'N', 'V', '+', '+', '+', '+', '+', '-', + '-', '-', '-', '-', '+', '+', '+', '+', + '|', '?', '?', '?', '?', '?', '?', +}; + +static teken_char_t +teken_scs_special_graphics(const teken_t *t, teken_char_t c) +{ + + /* Box drawing. */ + if (c >= '`' && c <= '~') + return (t->t_stateflags & TS_8BIT ? + teken_boxdrawing_8bit[c - '`'] : + teken_boxdrawing_unicode[c - '`']); + return (c); +} + +static teken_char_t +teken_scs_uk_national(const teken_t *t, teken_char_t c) +{ + + /* Pound sign. */ + if (c == '#') + return (t->t_stateflags & TS_8BIT ? 0x9c : 0xa3); + return (c); +} + +static teken_char_t +teken_scs_us_ascii(const teken_t *t, teken_char_t c) +{ + + /* No processing. */ + (void)t; + return (c); +} diff --git a/bin/varnishtest/teken_subr.h b/bin/varnishtest/teken_subr.h new file mode 100644 index 0000000..b67ef5c --- /dev/null +++ b/bin/varnishtest/teken_subr.h @@ -0,0 +1,1315 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2008-2009 Ed Schouten + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: head/sys/teken/teken_subr.h 326272 2017-11-27 15:23:17Z pfg $ + */ + +static void teken_subr_cursor_up(teken_t *, unsigned int); +static void teken_subr_erase_line(const teken_t *, unsigned int); +static void teken_subr_regular_character(teken_t *, teken_char_t); +static void teken_subr_reset_to_initial_state(teken_t *); +static void teken_subr_save_cursor(teken_t *); + +static inline int +teken_tab_isset(const teken_t *t, unsigned int col) +{ + unsigned int b, o; + + if (col >= T_NUMCOL) + return ((col % 8) == 0); + + b = col / (sizeof(unsigned int) * 8); + o = col % (sizeof(unsigned int) * 8); + + return (t->t_tabstops[b] & (1U << o)); +} + +static inline void +teken_tab_clear(teken_t *t, unsigned int col) +{ + unsigned int b, o; + + if (col >= T_NUMCOL) + return; + + b = col / (sizeof(unsigned int) * 8); + o = col % (sizeof(unsigned int) * 8); + + t->t_tabstops[b] &= ~(1U << o); +} + +static inline void +teken_tab_set(teken_t *t, unsigned int col) +{ + unsigned int b, o; + + if (col >= T_NUMCOL) + return; + + b = col / (sizeof(unsigned int) * 8); + o = col % (sizeof(unsigned int) * 8); + + t->t_tabstops[b] |= 1U << o; +} + +static void +teken_tab_default(teken_t *t) +{ + unsigned int i; + + memset(t->t_tabstops, 0, T_NUMCOL / 8); + + for (i = 8; i < T_NUMCOL; i += 8) + teken_tab_set(t, i); +} + +static void +teken_subr_do_scroll(const teken_t *t, int amount) +{ + teken_rect_t tr; + teken_pos_t tp; + + teken_assert(t->t_cursor.tp_row <= t->t_winsize.tp_row); + teken_assert(t->t_scrollreg.ts_end <= t->t_winsize.tp_row); + teken_assert(amount != 0); + + /* Copy existing data 1 line up. */ + if (amount > 0) { + /* Scroll down. */ + + /* Copy existing data up. */ + if (t->t_scrollreg.ts_begin + amount < t->t_scrollreg.ts_end) { + tr.tr_begin.tp_row = t->t_scrollreg.ts_begin + amount; + tr.tr_begin.tp_col = 0; + tr.tr_end.tp_row = t->t_scrollreg.ts_end; + tr.tr_end.tp_col = t->t_winsize.tp_col; + tp.tp_row = t->t_scrollreg.ts_begin; + tp.tp_col = 0; + teken_funcs_copy(t, &tr, &tp); + + tr.tr_begin.tp_row = t->t_scrollreg.ts_end - amount; + } else { + tr.tr_begin.tp_row = t->t_scrollreg.ts_begin; + } + + /* Clear the last lines. */ + tr.tr_begin.tp_col = 0; + tr.tr_end.tp_row = t->t_scrollreg.ts_end; + tr.tr_end.tp_col = t->t_winsize.tp_col; + teken_funcs_fill(t, &tr, BLANK, &t->t_curattr); + } else { + /* Scroll up. */ + amount = -amount; + + /* Copy existing data down. */ + if (t->t_scrollreg.ts_begin + amount < t->t_scrollreg.ts_end) { + tr.tr_begin.tp_row = t->t_scrollreg.ts_begin; + tr.tr_begin.tp_col = 0; + tr.tr_end.tp_row = t->t_scrollreg.ts_end - amount; + tr.tr_end.tp_col = t->t_winsize.tp_col; + tp.tp_row = t->t_scrollreg.ts_begin + amount; + tp.tp_col = 0; + teken_funcs_copy(t, &tr, &tp); + + tr.tr_end.tp_row = t->t_scrollreg.ts_begin + amount; + } else { + tr.tr_end.tp_row = t->t_scrollreg.ts_end; + } + + /* Clear the first lines. */ + tr.tr_begin.tp_row = t->t_scrollreg.ts_begin; + tr.tr_begin.tp_col = 0; + tr.tr_end.tp_col = t->t_winsize.tp_col; + teken_funcs_fill(t, &tr, BLANK, &t->t_curattr); + } +} + +static ssize_t +teken_subr_do_cpr(const teken_t *t, unsigned int cmd, char response[16]) +{ + + switch (cmd) { + case 5: /* Operating status. */ + strcpy(response, "0n"); + return (2); + case 6: { /* Cursor position. */ + int len; + + len = snprintf(response, 16, "%u;%uR", + (t->t_cursor.tp_row - t->t_originreg.ts_begin) + 1, + t->t_cursor.tp_col + 1); + + if (len >= 16) + return (-1); + return (len); + } + case 15: /* Printer status. */ + strcpy(response, "13n"); + return (3); + case 25: /* UDK status. */ + strcpy(response, "20n"); + return (3); + case 26: /* Keyboard status. */ + strcpy(response, "27;1n"); + return (5); + default: + teken_printf("Unknown DSR\n"); + return (-1); + } +} + +static void +teken_subr_alignment_test(teken_t *t) +{ + teken_rect_t tr; + + t->t_cursor.tp_row = t->t_cursor.tp_col = 0; + t->t_scrollreg.ts_begin = 0; + t->t_scrollreg.ts_end = t->t_winsize.tp_row; + t->t_originreg = t->t_scrollreg; + t->t_stateflags &= ~(TS_WRAPPED|TS_ORIGIN); + teken_funcs_cursor(t); + + tr.tr_begin.tp_row = 0; + tr.tr_begin.tp_col = 0; + tr.tr_end = t->t_winsize; + teken_funcs_fill(t, &tr, 'E', &t->t_defattr); +} + +static void +teken_subr_backspace(teken_t *t) +{ + + if (t->t_stateflags & TS_CONS25) { + if (t->t_cursor.tp_col == 0) { + if (t->t_cursor.tp_row == t->t_originreg.ts_begin) + return; + t->t_cursor.tp_row--; + t->t_cursor.tp_col = t->t_winsize.tp_col - 1; + } else { + t->t_cursor.tp_col--; + } + } else { + if (t->t_cursor.tp_col == 0) + return; + + t->t_cursor.tp_col--; + t->t_stateflags &= ~TS_WRAPPED; + } + + teken_funcs_cursor(t); +} + +static void +teken_subr_bell(const teken_t *t) +{ + + teken_funcs_bell(t); +} + +static void +teken_subr_carriage_return(teken_t *t) +{ + + t->t_cursor.tp_col = 0; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} + +static void +teken_subr_cursor_backward(teken_t *t, unsigned int ncols) +{ + + if (ncols > t->t_cursor.tp_col) + t->t_cursor.tp_col = 0; + else + t->t_cursor.tp_col -= ncols; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} + +static void +teken_subr_cursor_backward_tabulation(teken_t *t, unsigned int ntabs) +{ + + do { + /* Stop when we've reached the beginning of the line. */ + if (t->t_cursor.tp_col == 0) + break; + + t->t_cursor.tp_col--; + + /* Tab marker set. */ + if (teken_tab_isset(t, t->t_cursor.tp_col)) + ntabs--; + } while (ntabs > 0); + + teken_funcs_cursor(t); +} + +static void +teken_subr_cursor_down(teken_t *t, unsigned int nrows) +{ + + if (t->t_cursor.tp_row + nrows >= t->t_scrollreg.ts_end) + t->t_cursor.tp_row = t->t_scrollreg.ts_end - 1; + else + t->t_cursor.tp_row += nrows; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} + +static void +teken_subr_cursor_forward(teken_t *t, unsigned int ncols) +{ + + if (t->t_cursor.tp_col + ncols >= t->t_winsize.tp_col) + t->t_cursor.tp_col = t->t_winsize.tp_col - 1; + else + t->t_cursor.tp_col += ncols; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} + +static void +teken_subr_cursor_forward_tabulation(teken_t *t, unsigned int ntabs) +{ + + do { + /* Stop when we've reached the end of the line. */ + if (t->t_cursor.tp_col == t->t_winsize.tp_col - 1) + break; + + t->t_cursor.tp_col++; + + /* Tab marker set. */ + if (teken_tab_isset(t, t->t_cursor.tp_col)) + ntabs--; + } while (ntabs > 0); + + teken_funcs_cursor(t); +} + +static void +teken_subr_cursor_next_line(teken_t *t, unsigned int ncols) +{ + + t->t_cursor.tp_col = 0; + teken_subr_cursor_down(t, ncols); +} + +static void +teken_subr_cursor_position(teken_t *t, unsigned int row, unsigned int col) +{ + + row = (row - 1) + t->t_originreg.ts_begin; + t->t_cursor.tp_row = row < t->t_originreg.ts_end ? + row : t->t_originreg.ts_end - 1; + + col--; + t->t_cursor.tp_col = col < t->t_winsize.tp_col ? + col : t->t_winsize.tp_col - 1; + + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} + +static void +teken_subr_cursor_position_report(const teken_t *t, unsigned int cmd) +{ + char response[18] = "\x1B["; + ssize_t len; + + len = teken_subr_do_cpr(t, cmd, response + 2); + if (len < 0) + return; + + teken_funcs_respond(t, response, len + 2); +} + +static void +teken_subr_cursor_previous_line(teken_t *t, unsigned int ncols) +{ + + t->t_cursor.tp_col = 0; + teken_subr_cursor_up(t, ncols); +} + +static void +teken_subr_cursor_up(teken_t *t, unsigned int nrows) +{ + + if (t->t_scrollreg.ts_begin + nrows >= t->t_cursor.tp_row) + t->t_cursor.tp_row = t->t_scrollreg.ts_begin; + else + t->t_cursor.tp_row -= nrows; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} + +static void +teken_subr_delete_character(const teken_t *t, unsigned int ncols) +{ + teken_rect_t tr; + + tr.tr_begin.tp_row = t->t_cursor.tp_row; + tr.tr_end.tp_row = t->t_cursor.tp_row + 1; + tr.tr_end.tp_col = t->t_winsize.tp_col; + + if (t->t_cursor.tp_col + ncols >= t->t_winsize.tp_col) { + tr.tr_begin.tp_col = t->t_cursor.tp_col; + } else { + /* Copy characters to the left. */ + tr.tr_begin.tp_col = t->t_cursor.tp_col + ncols; + teken_funcs_copy(t, &tr, &t->t_cursor); + + tr.tr_begin.tp_col = t->t_winsize.tp_col - ncols; + } + + /* Blank trailing columns. */ + teken_funcs_fill(t, &tr, BLANK, &t->t_curattr); +} + +static void +teken_subr_delete_line(const teken_t *t, unsigned int nrows) +{ + teken_rect_t tr; + + /* Ignore if outside scrolling region. */ + if (t->t_cursor.tp_row < t->t_scrollreg.ts_begin || + t->t_cursor.tp_row >= t->t_scrollreg.ts_end) + return; + + tr.tr_begin.tp_col = 0; + tr.tr_end.tp_row = t->t_scrollreg.ts_end; + tr.tr_end.tp_col = t->t_winsize.tp_col; + + if (t->t_cursor.tp_row + nrows >= t->t_scrollreg.ts_end) { + tr.tr_begin.tp_row = t->t_cursor.tp_row; + } else { + teken_pos_t tp; + + /* Copy rows up. */ + tr.tr_begin.tp_row = t->t_cursor.tp_row + nrows; + tp.tp_row = t->t_cursor.tp_row; + tp.tp_col = 0; + teken_funcs_copy(t, &tr, &tp); + + tr.tr_begin.tp_row = t->t_scrollreg.ts_end - nrows; + } + + /* Blank trailing rows. */ + teken_funcs_fill(t, &tr, BLANK, &t->t_curattr); +} + +static void +teken_subr_device_control_string(teken_t *t) +{ + + teken_printf("Unsupported device control string\n"); + t->t_stateflags |= TS_INSTRING; +} + +static void +teken_subr_device_status_report(const teken_t *t, unsigned int cmd) +{ + char response[19] = "\x1B[?"; + ssize_t len; + + len = teken_subr_do_cpr(t, cmd, response + 3); + if (len < 0) + return; + + teken_funcs_respond(t, response, len + 3); +} + +static void +teken_subr_double_height_double_width_line_top(const teken_t *t) +{ + + (void)t; + teken_printf("double height double width top\n"); +} + +static void +teken_subr_double_height_double_width_line_bottom(const teken_t *t) +{ + + (void)t; + teken_printf("double height double width bottom\n"); +} + +static void +teken_subr_erase_character(const teken_t *t, unsigned int ncols) +{ + teken_rect_t tr; + + tr.tr_begin = t->t_cursor; + tr.tr_end.tp_row = t->t_cursor.tp_row + 1; + + if (t->t_cursor.tp_col + ncols >= t->t_winsize.tp_col) + tr.tr_end.tp_col = t->t_winsize.tp_col; + else + tr.tr_end.tp_col = t->t_cursor.tp_col + ncols; + + teken_funcs_fill(t, &tr, BLANK, &t->t_curattr); +} + +static void +teken_subr_erase_display(const teken_t *t, unsigned int mode) +{ + teken_rect_t r; + + r.tr_begin.tp_col = 0; + r.tr_end.tp_col = t->t_winsize.tp_col; + + switch (mode) { + case 1: /* Erase from the top to the cursor. */ + teken_subr_erase_line(t, 1); + + /* Erase lines above. */ + if (t->t_cursor.tp_row == 0) + return; + r.tr_begin.tp_row = 0; + r.tr_end.tp_row = t->t_cursor.tp_row; + break; + case 2: /* Erase entire display. */ + r.tr_begin.tp_row = 0; + r.tr_end.tp_row = t->t_winsize.tp_row; + break; + default: /* Erase from cursor to the bottom. */ + teken_subr_erase_line(t, 0); + + /* Erase lines below. */ + if (t->t_cursor.tp_row == t->t_winsize.tp_row - 1) + return; + r.tr_begin.tp_row = t->t_cursor.tp_row + 1; + r.tr_end.tp_row = t->t_winsize.tp_row; + break; + } + + teken_funcs_fill(t, &r, BLANK, &t->t_curattr); +} + +static void +teken_subr_erase_line(const teken_t *t, unsigned int mode) +{ + teken_rect_t r; + + r.tr_begin.tp_row = t->t_cursor.tp_row; + r.tr_end.tp_row = t->t_cursor.tp_row + 1; + + switch (mode) { + case 1: /* Erase from the beginning of the line to the cursor. */ + r.tr_begin.tp_col = 0; + r.tr_end.tp_col = t->t_cursor.tp_col + 1; + break; + case 2: /* Erase entire line. */ + r.tr_begin.tp_col = 0; + r.tr_end.tp_col = t->t_winsize.tp_col; + break; + default: /* Erase from cursor to the end of the line. */ + r.tr_begin.tp_col = t->t_cursor.tp_col; + r.tr_end.tp_col = t->t_winsize.tp_col; + break; + } + + teken_funcs_fill(t, &r, BLANK, &t->t_curattr); +} + +static void +teken_subr_g0_scs_special_graphics(teken_t *t) +{ + + t->t_scs[0] = teken_scs_special_graphics; +} + +static void +teken_subr_g0_scs_uk_national(teken_t *t) +{ + + t->t_scs[0] = teken_scs_uk_national; +} + +static void +teken_subr_g0_scs_us_ascii(teken_t *t) +{ + + t->t_scs[0] = teken_scs_us_ascii; +} + +static void +teken_subr_g1_scs_special_graphics(teken_t *t) +{ + + t->t_scs[1] = teken_scs_special_graphics; +} + +static void +teken_subr_g1_scs_uk_national(teken_t *t) +{ + + t->t_scs[1] = teken_scs_uk_national; +} + +static void +teken_subr_g1_scs_us_ascii(teken_t *t) +{ + + t->t_scs[1] = teken_scs_us_ascii; +} + +static void +teken_subr_horizontal_position_absolute(teken_t *t, unsigned int col) +{ + + col--; + t->t_cursor.tp_col = col < t->t_winsize.tp_col ? + col : t->t_winsize.tp_col - 1; + + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} + +static void +teken_subr_horizontal_tab(teken_t *t) +{ + + teken_subr_cursor_forward_tabulation(t, 1); +} + +static void +teken_subr_horizontal_tab_set(teken_t *t) +{ + + teken_tab_set(t, t->t_cursor.tp_col); +} + +static void +teken_subr_index(teken_t *t) +{ + + if (t->t_cursor.tp_row < t->t_scrollreg.ts_end - 1) { + t->t_cursor.tp_row++; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); + } else { + teken_subr_do_scroll(t, 1); + } +} + +static void +teken_subr_insert_character(const teken_t *t, unsigned int ncols) +{ + teken_rect_t tr; + + tr.tr_begin = t->t_cursor; + tr.tr_end.tp_row = t->t_cursor.tp_row + 1; + + if (t->t_cursor.tp_col + ncols >= t->t_winsize.tp_col) { + tr.tr_end.tp_col = t->t_winsize.tp_col; + } else { + teken_pos_t tp; + + /* Copy characters to the right. */ + tr.tr_end.tp_col = t->t_winsize.tp_col - ncols; + tp.tp_row = t->t_cursor.tp_row; + tp.tp_col = t->t_cursor.tp_col + ncols; + teken_funcs_copy(t, &tr, &tp); + + tr.tr_end.tp_col = t->t_cursor.tp_col + ncols; + } + + /* Blank current location. */ + teken_funcs_fill(t, &tr, BLANK, &t->t_curattr); +} + +static void +teken_subr_insert_line(const teken_t *t, unsigned int nrows) +{ + teken_rect_t tr; + + /* Ignore if outside scrolling region. */ + if (t->t_cursor.tp_row < t->t_scrollreg.ts_begin || + t->t_cursor.tp_row >= t->t_scrollreg.ts_end) + return; + + tr.tr_begin.tp_row = t->t_cursor.tp_row; + tr.tr_begin.tp_col = 0; + tr.tr_end.tp_col = t->t_winsize.tp_col; + + if (t->t_cursor.tp_row + nrows >= t->t_scrollreg.ts_end) { + tr.tr_end.tp_row = t->t_scrollreg.ts_end; + } else { + teken_pos_t tp; + + /* Copy lines down. */ + tr.tr_end.tp_row = t->t_scrollreg.ts_end - nrows; + tp.tp_row = t->t_cursor.tp_row + nrows; + tp.tp_col = 0; + teken_funcs_copy(t, &tr, &tp); + + tr.tr_end.tp_row = t->t_cursor.tp_row + nrows; + } + + /* Blank current location. */ + teken_funcs_fill(t, &tr, BLANK, &t->t_curattr); +} + +static void +teken_subr_keypad_application_mode(const teken_t *t) +{ + + teken_funcs_param(t, TP_KEYPADAPP, 1); +} + +static void +teken_subr_keypad_numeric_mode(const teken_t *t) +{ + + teken_funcs_param(t, TP_KEYPADAPP, 0); +} + +static void +teken_subr_newline(teken_t *t) +{ + + t->t_cursor.tp_row++; + + if (t->t_cursor.tp_row >= t->t_scrollreg.ts_end) { + teken_subr_do_scroll(t, 1); + t->t_cursor.tp_row = t->t_scrollreg.ts_end - 1; + } + + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} + +static void +teken_subr_newpage(teken_t *t) +{ + + if (t->t_stateflags & TS_CONS25) { + teken_rect_t tr; + + /* Clear screen. */ + tr.tr_begin.tp_row = t->t_originreg.ts_begin; + tr.tr_begin.tp_col = 0; + tr.tr_end.tp_row = t->t_originreg.ts_end; + tr.tr_end.tp_col = t->t_winsize.tp_col; + teken_funcs_fill(t, &tr, BLANK, &t->t_curattr); + + /* Cursor at top left. */ + t->t_cursor.tp_row = t->t_originreg.ts_begin; + t->t_cursor.tp_col = 0; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); + } else { + teken_subr_newline(t); + } +} + +static void +teken_subr_next_line(teken_t *t) +{ + + t->t_cursor.tp_col = 0; + teken_subr_newline(t); +} + +static void +teken_subr_operating_system_command(teken_t *t) +{ + + teken_printf("Unsupported operating system command\n"); + t->t_stateflags |= TS_INSTRING; +} + +static void +teken_subr_pan_down(const teken_t *t, unsigned int nrows) +{ + + teken_subr_do_scroll(t, (int)nrows); +} + +static void +teken_subr_pan_up(const teken_t *t, unsigned int nrows) +{ + + teken_subr_do_scroll(t, -(int)nrows); +} + +static void +teken_subr_primary_device_attributes(const teken_t *t, unsigned int request) +{ + + if (request == 0) { + const char response[] = "\x1B[?1;2c"; + + teken_funcs_respond(t, response, sizeof response - 1); + } else { + teken_printf("Unknown DA1\n"); + } +} + +static void +teken_subr_do_putchar(const teken_t *t, const teken_pos_t *tp, teken_char_t c, + int width) +{ + + if (t->t_stateflags & TS_INSERT && + tp->tp_col < t->t_winsize.tp_col - width) { + teken_rect_t ctr; + teken_pos_t ctp; + + /* Insert mode. Move existing characters to the right. */ + ctr.tr_begin = *tp; + ctr.tr_end.tp_row = tp->tp_row + 1; + ctr.tr_end.tp_col = t->t_winsize.tp_col - width; + ctp.tp_row = tp->tp_row; + ctp.tp_col = tp->tp_col + width; + teken_funcs_copy(t, &ctr, &ctp); + } + + teken_funcs_putchar(t, tp, c, &t->t_curattr); + + if (width == 2 && tp->tp_col + 1 < t->t_winsize.tp_col) { + teken_pos_t tp2; + teken_attr_t attr; + + /* Print second half of CJK fullwidth character. */ + tp2.tp_row = tp->tp_row; + tp2.tp_col = tp->tp_col + 1; + attr = t->t_curattr; + attr.ta_format |= TF_CJK_RIGHT; + teken_funcs_putchar(t, &tp2, c, &attr); + } +} + +static void +teken_subr_regular_character(teken_t *t, teken_char_t c) +{ + int width; + + if (t->t_stateflags & TS_8BIT) { + if (!(t->t_stateflags & TS_CONS25) && (c <= 0x1b || c == 0x7f)) + return; + c = teken_scs_process(t, c); + width = 1; + } else { + c = teken_scs_process(t, c); + width = teken_wcwidth(c); + /* XXX: Don't process zero-width characters yet. */ + if (width <= 0) + return; + } + + if (t->t_stateflags & TS_CONS25) { + teken_subr_do_putchar(t, &t->t_cursor, c, width); + t->t_cursor.tp_col += width; + + if (t->t_cursor.tp_col >= t->t_winsize.tp_col) { + if (t->t_cursor.tp_row == t->t_scrollreg.ts_end - 1) { + /* Perform scrolling. */ + teken_subr_do_scroll(t, 1); + } else { + /* No scrolling needed. */ + if (t->t_cursor.tp_row < + t->t_winsize.tp_row - 1) + t->t_cursor.tp_row++; + } + t->t_cursor.tp_col = 0; + } + } else if (t->t_stateflags & TS_AUTOWRAP && + ((t->t_stateflags & TS_WRAPPED && + t->t_cursor.tp_col + 1 == t->t_winsize.tp_col) || + t->t_cursor.tp_col + width > t->t_winsize.tp_col)) { + teken_pos_t tp; + + /* + * Perform line wrapping, if: + * - Autowrapping is enabled, and + * - We're in the wrapped state at the last column, or + * - The character to be printed does not fit anymore. + */ + if (t->t_cursor.tp_row == t->t_scrollreg.ts_end - 1) { + /* Perform scrolling. */ + teken_subr_do_scroll(t, 1); + tp.tp_row = t->t_scrollreg.ts_end - 1; + } else { + /* No scrolling needed. */ + tp.tp_row = t->t_cursor.tp_row + 1; + if (tp.tp_row == t->t_winsize.tp_row) { + /* + * Corner case: regular character + * outside scrolling region, but at the + * bottom of the screen. + */ + teken_subr_do_putchar(t, &t->t_cursor, + c, width); + return; + } + } + + tp.tp_col = 0; + teken_subr_do_putchar(t, &tp, c, width); + + t->t_cursor.tp_row = tp.tp_row; + t->t_cursor.tp_col = width; + t->t_stateflags &= ~TS_WRAPPED; + } else { + /* No line wrapping needed. */ + teken_subr_do_putchar(t, &t->t_cursor, c, width); + t->t_cursor.tp_col += width; + + if (t->t_cursor.tp_col >= t->t_winsize.tp_col) { + t->t_stateflags |= TS_WRAPPED; + t->t_cursor.tp_col = t->t_winsize.tp_col - 1; + } else { + t->t_stateflags &= ~TS_WRAPPED; + } + } + + teken_funcs_cursor(t); +} + +static void +teken_subr_reset_dec_mode(teken_t *t, unsigned int cmd) +{ + + switch (cmd) { + case 1: /* Cursor keys mode. */ + t->t_stateflags &= ~TS_CURSORKEYS; + break; + case 2: /* DECANM: ANSI/VT52 mode. */ + teken_printf("DECRST VT52\n"); + break; + case 3: /* 132 column mode. */ + teken_funcs_param(t, TP_132COLS, 0); + teken_subr_reset_to_initial_state(t); + break; + case 5: /* Inverse video. */ + teken_printf("DECRST inverse video\n"); + break; + case 6: /* Origin mode. */ + t->t_stateflags &= ~TS_ORIGIN; + t->t_originreg.ts_begin = 0; + t->t_originreg.ts_end = t->t_winsize.tp_row; + t->t_cursor.tp_row = t->t_cursor.tp_col = 0; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); + break; + case 7: /* Autowrap mode. */ + t->t_stateflags &= ~TS_AUTOWRAP; + break; + case 8: /* Autorepeat mode. */ + teken_funcs_param(t, TP_AUTOREPEAT, 0); + break; + case 25: /* Hide cursor. */ + teken_funcs_param(t, TP_SHOWCURSOR, 0); + break; + case 40: /* Disallow 132 columns. */ + teken_printf("DECRST allow 132\n"); + break; + case 45: /* Disable reverse wraparound. */ + teken_printf("DECRST reverse wraparound\n"); + break; + case 47: /* Switch to alternate buffer. */ + teken_printf("Switch to alternate buffer\n"); + break; + case 1000: /* Mouse input. */ + teken_funcs_param(t, TP_MOUSE, 0); + break; + default: + teken_printf("Unknown DECRST: %u\n", cmd); + } +} + +static void +teken_subr_reset_mode(teken_t *t, unsigned int cmd) +{ + + switch (cmd) { + case 4: + t->t_stateflags &= ~TS_INSERT; + break; + default: + teken_printf("Unknown reset mode: %u\n", cmd); + } +} + +static void +teken_subr_do_resize(teken_t *t) +{ + + t->t_scrollreg.ts_begin = 0; + t->t_scrollreg.ts_end = t->t_winsize.tp_row; + t->t_originreg = t->t_scrollreg; +} + +static void +teken_subr_do_reset(teken_t *t) +{ + + t->t_curattr = t->t_defattr; + t->t_cursor.tp_row = t->t_cursor.tp_col = 0; + t->t_scrollreg.ts_begin = 0; + t->t_scrollreg.ts_end = t->t_winsize.tp_row; + t->t_originreg = t->t_scrollreg; + t->t_stateflags &= TS_8BIT|TS_CONS25; + t->t_stateflags |= TS_AUTOWRAP; + + t->t_scs[0] = teken_scs_us_ascii; + t->t_scs[1] = teken_scs_us_ascii; + t->t_curscs = 0; + + teken_subr_save_cursor(t); + teken_tab_default(t); +} + +static void +teken_subr_reset_to_initial_state(teken_t *t) +{ + + teken_subr_do_reset(t); + teken_subr_erase_display(t, 2); + teken_funcs_param(t, TP_SHOWCURSOR, 1); + teken_funcs_cursor(t); +} + +static void +teken_subr_restore_cursor(teken_t *t) +{ + + t->t_cursor = t->t_saved_cursor; + t->t_curattr = t->t_saved_curattr; + t->t_scs[t->t_curscs] = t->t_saved_curscs; + t->t_stateflags &= ~TS_WRAPPED; + + /* Get out of origin mode when the cursor is moved outside. */ + if (t->t_cursor.tp_row < t->t_originreg.ts_begin || + t->t_cursor.tp_row >= t->t_originreg.ts_end) { + t->t_stateflags &= ~TS_ORIGIN; + t->t_originreg.ts_begin = 0; + t->t_originreg.ts_end = t->t_winsize.tp_row; + } + + teken_funcs_cursor(t); +} + +static void +teken_subr_reverse_index(teken_t *t) +{ + + if (t->t_cursor.tp_row > t->t_scrollreg.ts_begin) { + t->t_cursor.tp_row--; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); + } else { + teken_subr_do_scroll(t, -1); + } +} + +static void +teken_subr_save_cursor(teken_t *t) +{ + + t->t_saved_cursor = t->t_cursor; + t->t_saved_curattr = t->t_curattr; + t->t_saved_curscs = t->t_scs[t->t_curscs]; +} + +static void +teken_subr_secondary_device_attributes(const teken_t *t, unsigned int request) +{ + + if (request == 0) { + const char response[] = "\x1B[>0;10;0c"; + teken_funcs_respond(t, response, sizeof response - 1); + } else { + teken_printf("Unknown DA2\n"); + } +} + +static void +teken_subr_set_dec_mode(teken_t *t, unsigned int cmd) +{ + + switch (cmd) { + case 1: /* Cursor keys mode. */ + t->t_stateflags |= TS_CURSORKEYS; + break; + case 2: /* DECANM: ANSI/VT52 mode. */ + teken_printf("DECSET VT52\n"); + break; + case 3: /* 132 column mode. */ + teken_funcs_param(t, TP_132COLS, 1); + teken_subr_reset_to_initial_state(t); + break; + case 5: /* Inverse video. */ + teken_printf("DECSET inverse video\n"); + break; + case 6: /* Origin mode. */ + t->t_stateflags |= TS_ORIGIN; + t->t_originreg = t->t_scrollreg; + t->t_cursor.tp_row = t->t_scrollreg.ts_begin; + t->t_cursor.tp_col = 0; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); + break; + case 7: /* Autowrap mode. */ + t->t_stateflags |= TS_AUTOWRAP; + break; + case 8: /* Autorepeat mode. */ + teken_funcs_param(t, TP_AUTOREPEAT, 1); + break; + case 25: /* Display cursor. */ + teken_funcs_param(t, TP_SHOWCURSOR, 1); + break; + case 40: /* Allow 132 columns. */ + teken_printf("DECSET allow 132\n"); + break; + case 45: /* Enable reverse wraparound. */ + teken_printf("DECSET reverse wraparound\n"); + break; + case 47: /* Switch to alternate buffer. */ + teken_printf("Switch away from alternate buffer\n"); + break; + case 1000: /* Mouse input. */ + teken_funcs_param(t, TP_MOUSE, 1); + break; + default: + teken_printf("Unknown DECSET: %u\n", cmd); + } +} + +static void +teken_subr_set_mode(teken_t *t, unsigned int cmd) +{ + + switch (cmd) { + case 4: + teken_printf("Insert mode\n"); + t->t_stateflags |= TS_INSERT; + break; + default: + teken_printf("Unknown set mode: %u\n", cmd); + } +} + +static void +teken_subr_set_graphic_rendition(teken_t *t, unsigned int ncmds, + const unsigned int cmds[]) +{ + unsigned int i, n; + + /* No attributes means reset. */ + if (ncmds == 0) { + t->t_curattr = t->t_defattr; + return; + } + + for (i = 0; i < ncmds; i++) { + n = cmds[i]; + + switch (n) { + case 0: /* Reset. */ + t->t_curattr = t->t_defattr; + break; + case 1: /* Bold. */ + t->t_curattr.ta_format |= TF_BOLD; + break; + case 4: /* Underline. */ + t->t_curattr.ta_format |= TF_UNDERLINE; + break; + case 5: /* Blink. */ + t->t_curattr.ta_format |= TF_BLINK; + break; + case 7: /* Reverse. */ + t->t_curattr.ta_format |= TF_REVERSE; + break; + case 22: /* Remove bold. */ + t->t_curattr.ta_format &= ~TF_BOLD; + break; + case 24: /* Remove underline. */ + t->t_curattr.ta_format &= ~TF_UNDERLINE; + break; + case 25: /* Remove blink. */ + t->t_curattr.ta_format &= ~TF_BLINK; + break; + case 27: /* Remove reverse. */ + t->t_curattr.ta_format &= ~TF_REVERSE; + break; + case 30: /* Set foreground color: black */ + case 31: /* Set foreground color: red */ + case 32: /* Set foreground color: green */ + case 33: /* Set foreground color: brown */ + case 34: /* Set foreground color: blue */ + case 35: /* Set foreground color: magenta */ + case 36: /* Set foreground color: cyan */ + case 37: /* Set foreground color: white */ + t->t_curattr.ta_fgcolor = n - 30; + break; + case 38: /* Set foreground color: 256 color mode */ + if (i + 2 >= ncmds || cmds[i + 1] != 5) + continue; + t->t_curattr.ta_fgcolor = cmds[i + 2]; + i += 2; + break; + case 39: /* Set default foreground color. */ + t->t_curattr.ta_fgcolor = t->t_defattr.ta_fgcolor; + break; + case 40: /* Set background color: black */ + case 41: /* Set background color: red */ + case 42: /* Set background color: green */ + case 43: /* Set background color: brown */ + case 44: /* Set background color: blue */ + case 45: /* Set background color: magenta */ + case 46: /* Set background color: cyan */ + case 47: /* Set background color: white */ + t->t_curattr.ta_bgcolor = n - 40; + break; + case 48: /* Set background color: 256 color mode */ + if (i + 2 >= ncmds || cmds[i + 1] != 5) + continue; + t->t_curattr.ta_bgcolor = cmds[i + 2]; + i += 2; + break; + case 49: /* Set default background color. */ + t->t_curattr.ta_bgcolor = t->t_defattr.ta_bgcolor; + break; + case 90: /* Set bright foreground color: black */ + case 91: /* Set bright foreground color: red */ + case 92: /* Set bright foreground color: green */ + case 93: /* Set bright foreground color: brown */ + case 94: /* Set bright foreground color: blue */ + case 95: /* Set bright foreground color: magenta */ + case 96: /* Set bright foreground color: cyan */ + case 97: /* Set bright foreground color: white */ + t->t_curattr.ta_fgcolor = (n - 90) + 8; + break; + case 100: /* Set bright background color: black */ + case 101: /* Set bright background color: red */ + case 102: /* Set bright background color: green */ + case 103: /* Set bright background color: brown */ + case 104: /* Set bright background color: blue */ + case 105: /* Set bright background color: magenta */ + case 106: /* Set bright background color: cyan */ + case 107: /* Set bright background color: white */ + t->t_curattr.ta_bgcolor = (n - 100) + 8; + break; + default: + teken_printf("unsupported attribute %u\n", n); + } + } +} + +static void +teken_subr_set_top_and_bottom_margins(teken_t *t, unsigned int top, + unsigned int bottom) +{ + + /* Adjust top row number. */ + if (top > 0) + top--; + /* Adjust bottom row number. */ + if (bottom == 0 || bottom > t->t_winsize.tp_row) + bottom = t->t_winsize.tp_row; + + /* Invalid arguments. */ + if (top >= bottom - 1) { + top = 0; + bottom = t->t_winsize.tp_row; + } + + /* Apply scrolling region. */ + t->t_scrollreg.ts_begin = top; + t->t_scrollreg.ts_end = bottom; + if (t->t_stateflags & TS_ORIGIN) + t->t_originreg = t->t_scrollreg; + + /* Home cursor to the top left of the scrolling region. */ + t->t_cursor.tp_row = t->t_originreg.ts_begin; + t->t_cursor.tp_col = 0; + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} + +static void +teken_subr_single_height_double_width_line(const teken_t *t) +{ + + (void)t; + teken_printf("single height double width???\n"); +} + +static void +teken_subr_single_height_single_width_line(const teken_t *t) +{ + + (void)t; + teken_printf("single height single width???\n"); +} + +static void +teken_subr_string_terminator(const teken_t *t) +{ + + (void)t; + /* + * Strings are already terminated in teken_input_char() when ^[ + * is inserted. + */ +} + +static void +teken_subr_tab_clear(teken_t *t, unsigned int cmd) +{ + + switch (cmd) { + case 0: + teken_tab_clear(t, t->t_cursor.tp_col); + break; + case 3: + memset(t->t_tabstops, 0, T_NUMCOL / 8); + break; + default: + break; + } +} + +static void +teken_subr_vertical_position_absolute(teken_t *t, unsigned int row) +{ + + row = (row - 1) + t->t_originreg.ts_begin; + t->t_cursor.tp_row = row < t->t_originreg.ts_end ? + row : t->t_originreg.ts_end - 1; + + t->t_stateflags &= ~TS_WRAPPED; + teken_funcs_cursor(t); +} diff --git a/bin/varnishtest/teken_subr_compat.h b/bin/varnishtest/teken_subr_compat.h new file mode 100644 index 0000000..9c84f13 --- /dev/null +++ b/bin/varnishtest/teken_subr_compat.h @@ -0,0 +1,153 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2008-2009 Ed Schouten + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: head/sys/teken/teken_subr_compat.h 326272 2017-11-27 15:23:17Z pfg $ + */ + +static void +teken_subr_cons25_set_border(const teken_t *t, unsigned int c) +{ + + teken_funcs_param(t, TP_SETBORDER, c); +} + +static void +teken_subr_cons25_set_global_cursor_shape(const teken_t *t, unsigned int ncmds, + const unsigned int cmds[]) +{ + unsigned int code, i; + + /* + * Pack the args to work around API deficiencies. This requires + * knowing too much about the low level to be fully compatible. + * Returning when ncmds > 3 is necessary and happens to be + * compatible. Discarding high bits is necessary and happens to + * be incompatible only for invalid args when ncmds == 3. + */ + if (ncmds > 3) + return; + code = 0; + for (i = ncmds; i > 0; i--) + code = (code << 8) | (cmds[i - 1] & 0xff); + code = (code << 8) | ncmds; + teken_funcs_param(t, TP_SETGLOBALCURSOR, code); +} + +static void +teken_subr_cons25_set_local_cursor_type(const teken_t *t, unsigned int type) +{ + + teken_funcs_param(t, TP_SETLOCALCURSOR, type); +} + +static const teken_color_t cons25_colors[8] = { TC_BLACK, TC_BLUE, + TC_GREEN, TC_CYAN, TC_RED, TC_MAGENTA, TC_BROWN, TC_WHITE }; + +static void +teken_subr_cons25_set_default_background(teken_t *t, unsigned int c) +{ + + t->t_defattr.ta_bgcolor = cons25_colors[c % 8] | (c & 8); + t->t_curattr.ta_bgcolor = cons25_colors[c % 8] | (c & 8); +} + +static void +teken_subr_cons25_set_default_foreground(teken_t *t, unsigned int c) +{ + + t->t_defattr.ta_fgcolor = cons25_colors[c % 8] | (c & 8); + t->t_curattr.ta_fgcolor = cons25_colors[c % 8] | (c & 8); +} + +static const teken_color_t cons25_revcolors[8] = { 0, 4, 2, 6, 1, 5, 3, 7 }; + +void +teken_get_defattr_cons25(const teken_t *t, int *fg, int *bg) +{ + + *fg = cons25_revcolors[teken_256to8(t->t_defattr.ta_fgcolor)]; + if (t->t_defattr.ta_format & TF_BOLD) + *fg += 8; + *bg = cons25_revcolors[teken_256to8(t->t_defattr.ta_bgcolor)]; +} + +static void +teken_subr_cons25_switch_virtual_terminal(const teken_t *t, unsigned int vt) +{ + + teken_funcs_param(t, TP_SWITCHVT, vt); +} + +static void +teken_subr_cons25_set_bell_pitch_duration(const teken_t *t, unsigned int pitch, + unsigned int duration) +{ + + teken_funcs_param(t, TP_SETBELLPD, (pitch << 16) | + (duration & 0xffff)); +} + +static void +teken_subr_cons25_set_graphic_rendition(teken_t *t, unsigned int cmd, + unsigned int param) +{ + + (void)param; + switch (cmd) { + case 0: /* Reset. */ + t->t_curattr = t->t_defattr; + break; + default: + teken_printf("unsupported attribute %u\n", cmd); + } +} + +static void +teken_subr_cons25_set_terminal_mode(teken_t *t, unsigned int mode) +{ + + switch (mode) { + case 0: /* Switch terminal to xterm. */ + t->t_stateflags &= ~TS_CONS25; + break; + case 1: /* Switch terminal to cons25. */ + t->t_stateflags |= TS_CONS25; + break; + default: + break; + } +} + +#if 0 +static void +teken_subr_vt52_decid(teken_t *t) +{ + const char response[] = "\x1B/Z"; + + teken_funcs_respond(t, response, sizeof response - 1); +} +#endif diff --git a/bin/varnishtest/teken_wcwidth.h b/bin/varnishtest/teken_wcwidth.h new file mode 100644 index 0000000..6482305 --- /dev/null +++ b/bin/varnishtest/teken_wcwidth.h @@ -0,0 +1,120 @@ +/* + * Markus Kuhn -- 2007-05-26 (Unicode 5.0) + * + * Permission to use, copy, modify, and distribute this software + * for any purpose and without fee is hereby granted. The author + * disclaims all warranties with regard to this software. + * + * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c + * + * $FreeBSD: head/sys/teken/teken_wcwidth.h 186681 2009-01-01 13:26:53Z ed $ + */ + +struct interval { + teken_char_t first; + teken_char_t last; +}; + +/* auxiliary function for binary search in interval table */ +static int bisearch(teken_char_t ucs, const struct interval *table, int max) { + int min = 0; + int mid; + + if (ucs < table[0].first || ucs > table[max].last) + return 0; + while (max >= min) { + mid = (min + max) / 2; + if (ucs > table[mid].last) + min = mid + 1; + else if (ucs < table[mid].first) + max = mid - 1; + else + return 1; + } + + return 0; +} + +static int teken_wcwidth(teken_char_t ucs) +{ + /* sorted list of non-overlapping intervals of non-spacing characters */ + /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */ + static const struct interval combining[] = { + { 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 }, + { 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 }, + { 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 }, + { 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 }, + { 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED }, + { 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A }, + { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 }, + { 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D }, + { 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 }, + { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD }, + { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C }, + { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D }, + { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC }, + { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD }, + { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C }, + { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D }, + { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 }, + { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 }, + { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC }, + { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD }, + { 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D }, + { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 }, + { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E }, + { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC }, + { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 }, + { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E }, + { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 }, + { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 }, + { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 }, + { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F }, + { 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 }, + { 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD }, + { 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD }, + { 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 }, + { 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B }, + { 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 }, + { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 }, + { 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF }, + { 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 }, + { 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F }, + { 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B }, + { 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F }, + { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB }, + { 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F }, + { 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 }, + { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD }, + { 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F }, + { 0xE0100, 0xE01EF } + }; + + /* test for 8-bit control characters */ + if (ucs == 0) + return 0; + if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0)) + return -1; + + /* binary search in table of non-spacing characters */ + if (bisearch(ucs, combining, + sizeof(combining) / sizeof(struct interval) - 1)) + return 0; + + /* if we arrive here, ucs is not a combining or C0/C1 control character */ + + return 1 + + (ucs >= 0x1100 && + (ucs <= 0x115f || /* Hangul Jamo init. consonants */ + ucs == 0x2329 || ucs == 0x232a || + (ucs >= 0x2e80 && ucs <= 0xa4cf && + ucs != 0x303f) || /* CJK ... Yi */ + (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */ + (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */ + (ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */ + (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */ + (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */ + (ucs >= 0xffe0 && ucs <= 0xffe6) || + (ucs >= 0x20000 && ucs <= 0x2fffd) || + (ucs >= 0x30000 && ucs <= 0x3fffd))); +} diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc index 5b6922b..8a88ad7 100644 --- a/bin/varnishtest/tests/u00008.vtc +++ b/bin/varnishtest/tests/u00008.vtc @@ -36,5 +36,5 @@ process p1 -need-bytes 5000 -screen_dump process p1 -winsz 25 132 -process p1 -need-bytes 7000 -screen_dump -write {q} -wait +process p1 -need-bytes 4000 -screen_dump -write {q} -wait diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 94dbf14..6776e88 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -135,9 +135,3 @@ void vtc_expect(struct vtclog *, const char *, const char *, const char *, const char *, const char *); void vtc_wait4(struct vtclog *, long, int, int, int); void *vtc_record(struct vtclog *, int, struct vsb *); - -/* vtc_term.c */ -struct term *Term_New(struct vtclog *, int, int); -void Term_Feed(struct term *, const char *, const char *); -void Term_Dump(const struct term *); -void Term_SetSize(struct term *, int, int); diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 7c37103..fadfb14 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -54,6 +54,8 @@ #include "vsb.h" #include "vsub.h" +#include "teken.h" + struct process { unsigned magic; #define PROCESS_MAGIC 0x1617b43e @@ -84,15 +86,137 @@ struct process { pthread_t tp; unsigned hasthread; - struct term *term; - int lin; - int col; + int nlin; + int ncol; + char **vram; + teken_t tek[1]; }; static VTAILQ_HEAD(, process) processes = VTAILQ_HEAD_INITIALIZER(processes); /********************************************************************** + * Terminal emulation + */ + +static void +term_cursor(void *priv, const teken_pos_t *pos) +{ + (void)priv; + (void)pos; +} + +static void +term_putchar(void *priv, const teken_pos_t *pos, teken_char_t ch, + const teken_attr_t *at) +{ + struct process *pp; + + CAST_OBJ_NOTNULL(pp, priv, PROCESS_MAGIC); + (void)at; + if (ch > 126 || ch < 32) + ch = '?'; + assert(pos->tp_row < pp->nlin); + assert(pos->tp_col < pp->ncol); + pp->vram[pos->tp_row][pos->tp_col] = ch; +} + +static void +term_fill(void *priv, const teken_rect_t *r, teken_char_t c, + const teken_attr_t *a) +{ + teken_pos_t p; + + /* Braindead implementation of fill() - just call putchar(). */ + for (p.tp_row = r->tr_begin.tp_row; + p.tp_row < r->tr_end.tp_row; p.tp_row++) + for (p.tp_col = r->tr_begin.tp_col; + p.tp_col < r->tr_end.tp_col; p.tp_col++) + term_putchar(priv, &p, c, a); +} + +static void +term_copy(void *priv, const teken_rect_t *r, const teken_pos_t *p) +{ + struct process *pp; + int nrow, ncol, y; /* Has to be signed - >= 0 comparison */ + + /* + * Copying is a little tricky. We must make sure we do it in + * correct order, to make sure we don't overwrite our own data. + */ + CAST_OBJ_NOTNULL(pp, priv, PROCESS_MAGIC); + + nrow = r->tr_end.tp_row - r->tr_begin.tp_row; + ncol = r->tr_end.tp_col - r->tr_begin.tp_col; + + if (p->tp_row < r->tr_begin.tp_row) { + /* Copy from top to bottom. */ + for (y = 0; y < nrow; y++) + memmove(&pp->vram[p->tp_row + y][p->tp_col], + &pp->vram[r->tr_begin.tp_row + y][r->tr_begin.tp_col], ncol); + } else { + /* Copy from bottom to top. */ + for (y = nrow - 1; y >= 0; y--) + memmove(&pp->vram[p->tp_row + y][p->tp_col], + &pp->vram[r->tr_begin.tp_row + y][r->tr_begin.tp_col], ncol); + } +} + +static const teken_funcs_t process_teken_func = { + .tf_cursor = term_cursor, + .tf_putchar = term_putchar, + .tf_fill = term_fill, + .tf_copy = term_copy, +}; + +static void +term_screen_dump(const struct process *pp) +{ + int i; + + for (i = 0; i < pp->nlin; i++) + vtc_dump(pp->vl, 3, "screen", pp->vram[i], pp->ncol); +} + +static void +term_resize(struct process *pp, int lin, int col) +{ + teken_pos_t pos; + char **vram; + int i, j; + + vram = calloc(lin, sizeof *pp->vram); + AN(vram); + for (i = 0; i < lin; i++) { + vram[i] = malloc(col + 1L); + AN(vram[i]); + memset(vram[i], ' ', col); + vram[i][col] = '\0'; + } + if (pp->vram != NULL) { + for (i = 0; i < lin; i++) { + if (i >= pp->nlin) + break; + j = col; + if (j > pp->ncol) + j = pp->ncol; + memcpy(vram[i], pp->vram[i], j); + } + for (i = 0; i < pp->nlin; i++) + free(pp->vram[i]); + free(pp->vram); + } + pp->vram = vram; + pp->nlin = lin; + pp->ncol = col; + + pos.tp_row = lin; + pos.tp_col = col; + teken_set_winsize(pp->tek, &pos); +} + +/********************************************************************** * Allocate and initialize a process */ @@ -131,10 +255,8 @@ process_new(const char *name) p->fd_term = -1; VTAILQ_INSERT_TAIL(&processes, p, list); - p->lin = 25; - p->col = 80; - p->term = Term_New(p->vl, p->lin, p->col); - AN(p->term); + teken_init(p->tek, &process_teken_func, p); + term_resize(p, 25, 80); return (p); } @@ -214,7 +336,7 @@ process_stdout(const struct vev *ev, int what) else if (p->log == 3) vtc_hexdump(p->vl, 4, "stdout", buf, i); (void)write(p->f_stdout, buf, i); - Term_Feed(p->term, buf, buf + i); + teken_input(p->tek, buf, i); return (0); } @@ -311,14 +433,14 @@ process_thread(void *priv) } static void -process_winsz(struct process *p, int fd, int lin, int col) +process_winsz(struct process *p, int fd) { struct winsize ws; int i; memset(&ws, 0, sizeof ws); - ws.ws_row = (short)lin; - ws.ws_col = (short)col; + ws.ws_row = (short)p->nlin; + ws.ws_col = (short)p->ncol; i = ioctl(fd, TIOCSWINSZ, &ws); if (i) vtc_log(p->vl, 4, "TIOCWINSZ %d %s", i, strerror(errno)); @@ -330,7 +452,7 @@ process_init_term(struct process *p, int fd) struct termios tt; int i; - process_winsz(p, fd, p->lin, p->col); + process_winsz(p, fd); memset(&tt, 0, sizeof tt); tt.c_cflag = CREAD | CS8 | HUPCL; @@ -407,7 +529,7 @@ process_start(struct process *p) VSUB_closefrom(STDERR_FILENO + 1); process_init_term(p, slave); - AZ(setenv("TERM", "ansi.sys", 1)); + AZ(setenv("TERM", "xterm", 1)); AZ(unsetenv("TERMCAP")); // Not using NULL because GCC is now even more demented... assert(write(STDERR_FILENO, "+", 1) == 1); @@ -510,6 +632,23 @@ process_write(const struct process *p, const char *text) } static void +process_write_hex(const struct process *p, const char *text) +{ + struct vsb *vsb; + int j; + + if (!p->hasthread) + vtc_fatal(p->vl, "Cannot write to a non-running process"); + + vsb = vtc_hex_to_bin(p->vl, text); + assert(VSB_len(vsb) >= 0); + vtc_hexdump(p->vl, 4, "sendhex", VSB_data(vsb), VSB_len(vsb)); + j = write(p->fd_term, VSB_data(vsb), VSB_len(vsb)); + assert(j == VSB_len(vsb)); + VSB_destroy(&vsb); +} + +static void process_close(struct process *p) { @@ -614,6 +753,7 @@ cmd_process(CMD_ARGS) { struct process *p, *p2; uintmax_t u, v; + unsigned lin,col; (void)priv; (void)cmd; @@ -713,7 +853,7 @@ cmd_process(CMD_ARGS) continue; } if (!strcmp(*av, "-screen_dump")) { - Term_Dump(p->term); + term_screen_dump(p); continue; } if (!strcmp(*av, "-start")) { @@ -730,19 +870,24 @@ cmd_process(CMD_ARGS) continue; } if (!strcmp(*av, "-winsz")) { - p->lin = atoi(av[1]); - assert(p->lin > 1); - p->col = atoi(av[2]); - assert(p->col > 1); + lin = atoi(av[1]); + assert(lin > 1); + col = atoi(av[2]); + assert(col > 1); av += 2; - Term_SetSize(p->term, p->lin, p->col); - process_winsz(p, p->fd_term, p->lin, p->col); + term_resize(p, lin, col); + process_winsz(p, p->fd_term); } if (!strcmp(*av, "-write")) { process_write(p, av[1]); av++; continue; } + if (!strcmp(*av, "-writehex")) { + process_write_hex(p, av[1]); + av++; + continue; + } if (!strcmp(*av, "-writeln")) { process_write(p, av[1]); process_write(p, "\n"); diff --git a/bin/varnishtest/vtc_term.c b/bin/varnishtest/vtc_term.c deleted file mode 100644 index ee9426d..0000000 --- a/bin/varnishtest/vtc_term.c +++ /dev/null @@ -1,327 +0,0 @@ -/*- - * Copyright (c) 2018 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * A trivial ANSI terminal emulation - */ - -#include "config.h" - -#include -#include -#include - -#include "vtc.h" - -struct term { - unsigned magic; -#define TERM_MAGIC 0x1c258f0f - - struct vtclog *vl; - unsigned state; -#define NTERMARG 10 - int arg[NTERMARG]; - int *argp; - int nlin; - int ncol; - char **vram; - int col; - int line; -}; - -static void -term_clear(char * const *vram, int lin, int col) -{ - int i; - - for (i = 0; i < lin; i++) { - memset(vram[i], ' ', col); - vram[i][col] = '\0'; - } -} - -static void -term_scroll(const struct term *tp) -{ - int i; - char *l; - - l = tp->vram[0]; - for (i = 0; i < tp->nlin -1; i++) - tp->vram[i] = tp->vram[i + 1]; - tp->vram[i] = l; - memset(l, ' ', tp->ncol); -} - -void -Term_Dump(const struct term *tp) -{ - int i; - - for (i = 0; i < tp->nlin; i++) - vtc_dump(tp->vl, 3, "screen", tp->vram[i], tp->ncol); -} - -static void -term_escape(struct term *tp, int c, int n) -{ - int i; - - switch (c) { - case 'A': - // CUU - Cursor up - if (tp->arg[0] == -1) tp->arg[0] = 1; - tp->line -= tp->arg[0]; - if (tp->line < 0) - vtc_fatal(tp->vl, "ANSI A[%d] outside vram", - tp->arg[0]); - break; - case 'B': - // CUD - Cursor down - if (tp->arg[0] == -1) tp->arg[0] = 1; - if (tp->arg[0] > tp->nlin) - vtc_fatal(tp->vl, "ANSI B[%d] outside vram", - tp->arg[0]); - tp->line += tp->arg[0]; - while (tp->line >= tp->nlin) { - term_scroll(tp); - tp->line--; - } - break; - case 'C': - // CUF - Cursor forward - if (tp->arg[0] == -1) tp->arg[0] = 1; - tp->col += tp->arg[0]; - if (tp->col >= tp->ncol) - tp->col = tp->ncol - 1; - break; - case 'D': - // CUB - Cursor backward - if (tp->arg[0] == -1) tp->arg[0] = 1; - tp->col -= tp->arg[0]; - if (tp->col < 0) - tp->col = 0; - break; - case 'h': - // SM - Set Mode (mostly ignored XXX?) - tp->col = 0; - tp->line = 0; - break; - case 'H': - // CUP - Cursor Position - if (tp->arg[0] == -1) tp->arg[0] = 1; - if (tp->arg[1] == -1) tp->arg[1] = 1; - if (tp->arg[0] > tp->nlin || tp->arg[1] > tp->ncol) - vtc_fatal(tp->vl, "ANSI H[%d,%d] outside vram", - tp->arg[0], tp->arg[1]); - tp->line = tp->arg[0] - 1; - tp->col = tp->arg[1] - 1; - break; - case 'J': - // ED - Erase in Display (0=below, 1=above, 2=all) - switch (tp->arg[0]) { - case 2: - term_clear(tp->vram, tp->nlin, tp->ncol); - break; - default: - vtc_fatal(tp->vl, "ANSI J[%d]", tp->arg[0]); - } - break; - case 'K': - // EL - Erase in line (0=right, 1=left, 2=full line) - if (tp->arg[0] == -1) tp->arg[0] = 0; - switch (tp->arg[0]) { - case 0: - for (i = tp->col; i < tp->ncol; i++) - tp->vram[tp->line][i] = ' '; - break; - case 1: - for (i = 0; i < tp->col; i++) - tp->vram[tp->line][i] = ' '; - break; - case 2: - for (i = 0; i < tp->ncol; i++) - tp->vram[tp->line][i] = ' '; - break; - default: - vtc_fatal(tp->vl, "ANSI K[%d]", tp->arg[0]); - } - break; - case 'm': - // SGG - Character Attributes (ignored) - break; - default: - for (i = 0; i < n; i++) - vtc_log(tp->vl, 4, "ANSI arg %d", tp->arg[i]); - vtc_fatal(tp->vl, "ANSI unknown (%c)", c); - break; - } -} - -static void -term_char(struct term *tp, char c) -{ - assert(tp->col < tp->ncol); - assert(tp->line < tp->nlin); - assert(tp->state <= 3); - switch (c) { - case 0x00: - break; - case '\b': - if (tp->col > 0) - tp->col--; - break; - case '\t': - while (++tp->col % 8) - continue; - if (tp->col >= tp->ncol) { - tp->col = 0; - term_char(tp, '\n'); - } - break; - case '\n': - if (tp->line == tp->nlin - 1) - term_scroll(tp); - else - tp->line++; - break; - case '\r': - tp->col = 0; - break; - default: - if (c < ' ' || c > '~') - c = '?'; - tp->vram[tp->line][tp->col++] = c; - if (tp->col >= tp->ncol) { - tp->col = 0; - term_char(tp, '\n'); - } - } -} - -void -Term_Feed(struct term *tp, const char *b, const char *e) -{ - int i; - - while (b < e) { - assert(tp->col < tp->ncol); - assert(tp->line < tp->nlin); - assert(tp->state <= 3); - switch (tp->state) { - case 0: - if (*b == '\x1b') - tp->state = 1; - else if (*(const uint8_t*)b == 0x9b) - tp->state = 2; - else - term_char(tp, *b); - b++; - break; - case 1: - if (*b++ != '[') - vtc_fatal(tp->vl, "ANSI not '[' (0x%x)", - b[-1] & 0xff); - tp->state = 2; - break; - case 2: - tp->argp = tp->arg; - for (i=0; i < NTERMARG; i++) - tp->arg[i] = -1; - tp->state = 3; - if (*b == '?') - b++; - break; - case 3: - if (tp->argp - tp->arg >= NTERMARG) - vtc_fatal(tp->vl, "ANSI too many args"); - - if (isdigit(*b)) { - if (*tp->argp == -1) - *tp->argp = 0; - *tp->argp *= 10; - *tp->argp += *b++ - '0'; - continue; - } - if (*b == ';') { - tp->argp++; - tp->state = 3; - b++; - continue; - } - term_escape(tp, *b++, tp->argp - tp->arg); - tp->state = 0; - break; - default: - WRONG("Wrong ansi state"); - } - } -} - -void -Term_SetSize(struct term *tp, int lin, int col) -{ - char **vram; - int i, j; - - vram = calloc(lin, sizeof *tp->vram); - AN(vram); - for (i = 0; i < lin; i++) { - vram[i] = malloc(col + 1L); - AN(vram[i]); - } - term_clear(vram, lin, col); - if (tp->vram != NULL) { - for (i = 0; i < lin; i++) { - if (i >= tp->nlin) - break; - j = col; - if (j > tp->ncol) - j = tp->ncol; - memcpy(vram[i], tp->vram[i], j); - } - for (i = 0; i < tp->nlin; i++) - free(tp->vram[i]); - free(tp->vram); - } - tp->vram = vram; - tp->nlin = lin; - tp->ncol = col; -} - -struct term * -Term_New(struct vtclog *vl, int lin, int col) -{ - struct term *tp; - - ALLOC_OBJ(tp, TERM_MAGIC); - AN(tp); - tp->vl = vl; - Term_SetSize(tp, lin, col); - tp->line = tp->nlin - 1; - return (tp); -} - From phk at FreeBSD.org Thu Apr 5 07:31:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 5 Apr 2018 07:31:14 +0000 (UTC) Subject: [master] 362c24c Introduce '-expect-text LIN COL PAT' to wait for text somewhere on screen. Message-ID: <20180405073114.B7B47974B8@lists.varnish-cache.org> commit 362c24c3339fb778dd29829337d756729fa365bc Author: Poul-Henning Kamp Date: Thu Apr 5 07:28:40 2018 +0000 Introduce '-expect-text LIN COL PAT' to wait for text somewhere on screen. Also reappropriate test a00001 to exercise the Teken code with the vttest (http://invisible-island.net/vttest/) program, if installed. diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt index fa51cc8..78c17a9 100644 --- a/bin/varnishtest/flint.lnt +++ b/bin/varnishtest/flint.lnt @@ -1,5 +1,5 @@ -+libh(teken/teken.h) ++libh(teken.h) // Tell FlexeLint when these don't return -function(exit, vtc_fatal) diff --git a/bin/varnishtest/flint.sh b/bin/varnishtest/flint.sh index 7cc4f26..49f5e2d 100755 --- a/bin/varnishtest/flint.sh +++ b/bin/varnishtest/flint.sh @@ -4,7 +4,6 @@ FLOPS=' -DTOP_BUILDDIR="foo" -I../../lib/libvgz *.c - teken/teken.c ' . ../../tools/flint_skel.sh diff --git a/bin/varnishtest/tests/a00001.vtc b/bin/varnishtest/tests/a00001.vtc index e15969e..1dd3f36 100644 --- a/bin/varnishtest/tests/a00001.vtc +++ b/bin/varnishtest/tests/a00001.vtc @@ -1,26 +1,44 @@ -# the first token in a varnishtest file must be "varnishtest", - # but whitespace (SP) - # and (TAB) and comments are fine -varnishtest "basic default HTTP transactions with expect" - -server s1 { - rxreq - expect req.method == GET - expect req.proto == HTTP/1.1 - expect req.url == "/" - txresp -} - -server s1 -start - -client c1 -connect ${s1_sock} { - txreq - rxresp - expect resp.proto == HTTP/1.1 - expect resp.status == 200 - expect resp.reason == OK -} - -client c1 -run - -server s1 -wait +varnishtest "Test Teken terminal emulator" + +feature cmd "vttest --version 2>&1 | grep -q Usage" + +process p4 -dump {vttest} -start + +process p4 -expect-text 21 11 "Enter choice number (0 - 12):" +process p4 -screen_dump + +process p4 -writehex "31 0d" +process p4 -expect-text 14 61 "RETURN" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 14 87 "RETURN" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 22 7 "RETURN" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 3 132 "i" +process p4 -expect-text 22 7 "RETURN" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 9 7 "RETURN" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 4 1 "This is a correct sentence" +process p4 -expect-text 20 7 "RETURN" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 21 11 "Enter choice number (0 - 12):" +process p4 -screen_dump + +process p4 -writehex "30 0d" +process p4 -expect-text 12 30 "That's all, folks!" +process p4 -screen_dump + +process p4 -wait diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index fadfb14..0dbe46f 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -95,6 +95,8 @@ struct process { static VTAILQ_HEAD(, process) processes = VTAILQ_HEAD_INITIALIZER(processes); +static void term_resize(struct process *pp, int lin, int col); + /********************************************************************** * Terminal emulation */ @@ -163,11 +165,40 @@ term_copy(void *priv, const teken_rect_t *r, const teken_pos_t *p) } } +static void +term_respond(void *priv, const void *p, size_t l) +{ + struct process *pp; + int r; + + CAST_OBJ_NOTNULL(pp, priv, PROCESS_MAGIC); + + vtc_dump(pp->vl, 4, "term_response", p, l); + r = write(pp->fd_term, p, l); + if (r != l) + vtc_fatal(pp->vl, "Could not write to process: %s", + strerror(errno)); +} + +static void +term_param(void *priv, int p, unsigned int v) +{ + struct process *pp; + + CAST_OBJ_NOTNULL(pp, priv, PROCESS_MAGIC); + if (p == TP_132COLS && v) + term_resize(pp, pp->nlin, 132); + if (p == TP_132COLS && !v) + term_resize(pp, pp->nlin, 80); +} + static const teken_funcs_t process_teken_func = { .tf_cursor = term_cursor, .tf_putchar = term_putchar, .tf_fill = term_fill, .tf_copy = term_copy, + .tf_respond = term_respond, + .tf_param = term_param, }; static void @@ -216,6 +247,69 @@ term_resize(struct process *pp, int lin, int col) teken_set_winsize(pp->tek, &pos); } +static int +term_match_textline(const struct process *pp, int *x, int y, const char *pat) +{ + const char *t; + + if (*x == 0) { + t = strstr(pp->vram[y], pat); + if (t != NULL) { + *x = 1 + (t - pp->vram[y]); + return (1); + } + } else if (*x <= pp->ncol) { + t = pp->vram[y] + *x - 1; + if (!memcmp(t, pat, strlen(pat))) + return (1); + } + return (0); +} + +static int +term_match_text(const struct process *pp, int *x, int *y, const char *pat) +{ + int yy; + + if (*y == 0) { + for (yy = 0; yy < pp->nlin; yy++) { + if (term_match_textline(pp, x, yy, pat)) { + *y = yy + 1; + return (1); + } + } + } else if (*y <= pp->nlin) { + if (term_match_textline(pp, x, *y - 1, pat)) + return (1); + } + return (0); +} + +static void +term_expect_text(struct process *pp, + const char *lin, const char *col, const char *pat) +{ + int x, y, l; + char *t; + + y = strtoul(lin, NULL, 0); + x = strtoul(col, NULL, 0); + l = strlen(pat); + AZ(pthread_mutex_lock(&pp->mtx)); + while (!term_match_text(pp, &x, &y, pat)) { + if (x != 0 && y != 0) { + t = pp->vram[y - 1] + x - 1; + vtc_log(pp->vl, 4, + "text at %d,%d: '%.*s'", y, x, l, t); + } + AZ(pthread_mutex_unlock(&pp->mtx)); + usleep(1000000); + AZ(pthread_mutex_lock(&pp->mtx)); + } + AZ(pthread_mutex_unlock(&pp->mtx)); + vtc_log(pp->vl, 4, "found expected text at %d,%d: '%s'", y, x, pat); +} + /********************************************************************** * Allocate and initialize a process */ @@ -256,7 +350,7 @@ process_new(const char *name) VTAILQ_INSERT_TAIL(&processes, p, list); teken_init(p->tek, &process_teken_func, p); - term_resize(p, 25, 80); + term_resize(p, 24, 80); return (p); } @@ -336,7 +430,9 @@ process_stdout(const struct vev *ev, int what) else if (p->log == 3) vtc_hexdump(p->vl, 4, "stdout", buf, i); (void)write(p->f_stdout, buf, i); + AZ(pthread_mutex_lock(&p->mtx)); teken_input(p->tek, buf, i); + AZ(pthread_mutex_unlock(&p->mtx)); return (0); } @@ -740,6 +836,15 @@ process_close(struct process *p) * \-writeln STRING * Same as -write followed by a newline (\\n). * + * \-writehex HEXSTRING + * Same as -write but interpreted as hexadecimal bytes. + * + * \-expect-text LIN COL PAT + * Wait for PAT to appear at LIN,COL on the virtual screen. + * Lines and columns are numbered 1...N + * LIN==0 means "on any line" + * COL==0 means "anywhere on the line" + * * \-close * Alias for "-kill HUP" * @@ -852,6 +957,14 @@ cmd_process(CMD_ARGS) process_wait(p); continue; } + if (!strcmp(*av, "-expect-text")) { + AN(av[1]); + AN(av[2]); + AN(av[3]); + term_expect_text(p, av[1], av[2], av[3]); + av += 3; + continue; + } if (!strcmp(*av, "-screen_dump")) { term_screen_dump(p); continue; @@ -875,7 +988,9 @@ cmd_process(CMD_ARGS) col = atoi(av[2]); assert(col > 1); av += 2; + AZ(pthread_mutex_lock(&p->mtx)); term_resize(p, lin, col); + AZ(pthread_mutex_unlock(&p->mtx)); process_winsz(p, p->fd_term); } if (!strcmp(*av, "-write")) { From phk at FreeBSD.org Thu Apr 5 07:34:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 5 Apr 2018 07:34:09 +0000 (UTC) Subject: [master] a65c99a Remove the "feature term" test now that we use TERM=xterm Message-ID: <20180405073409.794FF975BB@lists.varnish-cache.org> commit a65c99a585e014465d7e3c728fc083ae42358524 Author: Poul-Henning Kamp Date: Thu Apr 5 07:32:41 2018 +0000 Remove the "feature term" test now that we use TERM=xterm diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc index 8a88ad7..052a1c9 100644 --- a/bin/varnishtest/tests/u00008.vtc +++ b/bin/varnishtest/tests/u00008.vtc @@ -1,7 +1,5 @@ varnishtest "trivial run of varnistat in curses mode" -feature term - server s1 -repeat 4 { rxreq txresp diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc index 89e6ea7..5825179 100644 --- a/bin/varnishtest/tests/u00009.vtc +++ b/bin/varnishtest/tests/u00009.vtc @@ -1,7 +1,5 @@ varnishtest "trivial run of varnishhist in curses mode" -feature term - server s1 { rxreq txresp diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc index a143c72..009199d 100644 --- a/bin/varnishtest/tests/u00010.vtc +++ b/bin/varnishtest/tests/u00010.vtc @@ -1,7 +1,5 @@ varnishtest "trivial run of varnishtop in curses mode" -feature term - server s1 { rxreq txresp diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc index 90fcfe2..c1a6008 100644 --- a/bin/varnishtest/tests/u00011.vtc +++ b/bin/varnishtest/tests/u00011.vtc @@ -1,7 +1,5 @@ varnishtest "trivial run of varnishadm in curses mode" -feature term - server s1 { rxreq txresp diff --git a/bin/varnishtest/tests/u00012.vtc b/bin/varnishtest/tests/u00012.vtc index 0660d07..5fb9b35 100644 --- a/bin/varnishtest/tests/u00012.vtc +++ b/bin/varnishtest/tests/u00012.vtc @@ -1,7 +1,5 @@ varnishtest "trivial run of varnishadm in pass mode" -feature term - server s1 { rxreq txresp diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 06b0167..8ffe696 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -341,8 +341,6 @@ cmd_delay(CMD_ARGS) * ignore_unknown_macro * Do not fail the test if a string of the form ${...} is not * recognized as a macro. - * term - * Support for ansi.sys terminal * * persistent_storage * Varnish was built with the deprecated persistent storage. @@ -358,23 +356,6 @@ static const unsigned with_persistent_storage = 1; static const unsigned with_persistent_storage = 0; #endif -static int -test_term(struct vtclog *vl) -{ - FILE *p; - int a, b; - - p = popen("tput -T ansi.sys clear 2>&1", "r"); - if (p == NULL) - return (0); - a = fgetc(p); - b = fgetc(p); - if (a == 0x1b && b == '[') - return (1); - vtc_log(vl, 3, "No 'ansi.sys' terminfo entry."); - return (0); -} - void v_matchproto_(cmd_f) cmd_feature(CMD_ARGS) { @@ -423,7 +404,6 @@ cmd_feature(CMD_ARGS) FEATURE("user_varnish", getpwnam("varnish") != NULL); FEATURE("user_vcache", getpwnam("vcache") != NULL); FEATURE("group_varnish", getgrnam("varnish") != NULL); - FEATURE("term", test_term(vl)); FEATURE("persistent_storage", with_persistent_storage); if (!strcmp(*av, "disable_aslr")) { From nils.goroll at uplex.de Thu Apr 5 08:53:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Apr 2018 08:53:10 +0000 (UTC) Subject: [master] 6285c39 umem: suncc happyness, set fptrs NULL in mgmt Message-ID: <20180405085310.168A598D7C@lists.varnish-cache.org> commit 6285c390f915fbc928a1ebf16ada0afeb730048f Author: Nils Goroll Date: Thu Apr 5 10:52:16 2018 +0200 umem: suncc happyness, set fptrs NULL in mgmt diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index db8178a..e366939 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -329,9 +329,11 @@ smu_init(struct stevedore *parent, int ac, char * const *av) #define DLSYM_UMEM(fptr,sym) \ do { \ (void) dlerror(); \ - if ((fptr = dlsym(libumem_hndl, #sym)) == NULL) \ - ARGV_ERR("(-sumem) cannot find symbol " #sym ": %s", \ - dlerror()); \ + if (dlsym(libumem_hndl, #sym) == NULL) \ + ARGV_ERR("(-sumem) cannot find symbol " \ + #sym ": %s", \ + dlerror()); \ + fptr = NULL; \ } while(0) DLSYM_UMEM(umem_allocf, umem_alloc); @@ -372,7 +374,7 @@ smu_open_init(void) #define DLSYM_UMEM(fptr,sym) \ do { \ - fptr = dlsym(libumem_hndl, #sym); \ + fptr = (sym ## _f) dlsym(libumem_hndl, #sym); \ AN(fptr); \ } while(0) From nils.goroll at uplex.de Thu Apr 5 08:59:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Apr 2018 08:59:08 +0000 (UTC) Subject: [master] c1877d4 suncc happyness Message-ID: <20180405085908.C708998FCC@lists.varnish-cache.org> commit c1877d4687623a0e2198ad6e891902b4608f144e Author: Nils Goroll Date: Thu Apr 5 10:58:40 2018 +0200 suncc happyness diff --git a/bin/varnishtest/teken.c b/bin/varnishtest/teken.c index 8f9076d..ad06c57 100644 --- a/bin/varnishtest/teken.c +++ b/bin/varnishtest/teken.c @@ -42,7 +42,7 @@ #define teken_assert(x) assert(x) /* debug messages */ -#define teken_printf(x,...) +#define teken_printf(...) /* Private flags for t_stateflags. */ #define TS_FIRSTDIGIT 0x0001 /* First numeric digit in escape sequence. */ From phk at FreeBSD.org Thu Apr 5 09:55:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 5 Apr 2018 09:55:11 +0000 (UTC) Subject: [master] bee5fc1 Replace N-1 -need-bytes with -expect-text for better test stability. Message-ID: <20180405095511.D6E049A43B@lists.varnish-cache.org> commit bee5fc165dd51301eb553853e9840184342128f4 Author: Poul-Henning Kamp Date: Thu Apr 5 09:54:11 2018 +0000 Replace N-1 -need-bytes with -expect-text for better test stability. Last -need-bytes kept to test that it still works. diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index fe5682d..affa747 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -47,9 +47,9 @@ process p2 "stty -a ; sleep 1" -run -screen_dump process p3 "stty raw -echo ; stty -a ; sleep 1" -run -screen_dump -process p4 -hexdump {stty raw -echo; echo "*" ; cat} -start +process p4 -hexdump {stty raw -echo; echo "*" ; sleep 2 ; cat} -start -process p4 -need-bytes 2 +process p4 -expect-text 0 0 "*" process p4 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H" process p4 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n" @@ -60,4 +60,5 @@ process p4 -write "333\x0d\x0a111111112222222333333\x0d\x0a\x1b[12" process p4 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b" process p4 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a" -process p4 -need-bytes 252 -screen_dump -stop +process p4 -need-bytes 252 -expect-text 3 1 "line3 <" +process p4 -screen_dump -stop diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index fe5b798..7e426ec 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -2,7 +2,7 @@ varnishtest "Simple process tests" process p1 "cat" -start process p1 -writeln "foo" -process p1 -need-bytes 3 +process p1 -expect-text 2 1 foo process p1 -stop process p1 -wait shell "grep -q foo ${p1_out}" @@ -10,7 +10,7 @@ shell "test -f ${p1_err} -a ! -s ${p1_err}" process p2 -log "cat" -start process p2 -writeln "bar" -process p2 -need-bytes 3 +process p2 -expect-text 2 1 bar process p2 -write "\x04" process p2 -wait shell "grep -q bar ${p2_out}" @@ -18,7 +18,7 @@ shell "test -f ${p2_err} -a ! -s ${p2_err}" process p3 -dump "cat" -start process p3 -writeln "baz" -process p3 -need-bytes 3 +process p3 -expect-text 2 1 baz process p3 -kill KILL process p3 -wait shell "grep -q baz ${p3_out}" @@ -26,7 +26,7 @@ shell "test -f ${p3_err} -a ! -s ${p3_err}" process p4 -hexdump "cat" -start process p4 -writeln "b\001z" -process p4 -need-bytes 3 +process p4 -expect-text 2 1 "b" process p4 -kill TERM process p4 -wait -screen_dump diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc index 052a1c9..5e1beff 100644 --- a/bin/varnishtest/tests/u00008.vtc +++ b/bin/varnishtest/tests/u00008.vtc @@ -13,26 +13,33 @@ varnish v1 -vcl+backend { process p1 -dump {varnishstat -n ${v1_name}} -start -process p1 -need-bytes 1 +process p1 -expect-text 0 0 "VBE.vcl1.s1.happy" +process p1 -screen_dump client c1 { txreq rxresp } -run -process p1 -need-bytes 1000 +process p1 -expect-text 0 0 "MAIN.s_sess" +process p1 -screen_dump process p1 -write {vG} +process p1 -expect-text 0 0 "VBE.vcl1.s1.req" +process p1 -expect-text 0 0 "DIAG" +process p1 -screen_dump varnish v1 -stop +process p1 -expect-text 2 1 "Uptime child: Not Running" +process p1 -screen_dump process p1 -write {dek} - -delay 1 - -process p1 -need-bytes 5000 -screen_dump +process p1 -expect-text 0 1 "Concurrent connections to backend:" +process p1 -screen_dump process p1 -winsz 25 132 +process p1 -expect-text 4 124 "AVG_1000" +process p1 -expect-text 22 108 "UNSEEN DIAG" -process p1 -need-bytes 4000 -screen_dump -write {q} -wait +process p1 -screen_dump -write {q} -wait diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc index 5825179..eb587bc 100644 --- a/bin/varnishtest/tests/u00009.vtc +++ b/bin/varnishtest/tests/u00009.vtc @@ -9,11 +9,13 @@ varnish v1 -vcl+backend {} -start process p1 -dump {varnishhist -n ${v1_name}} -start -process p1 -need-bytes 1 +process p1 -expect-text 0 0 {1e2} client c1 { txreq rxresp } -run -process p1 -need-bytes 200 -screen_dump -write {q} -wait +process p1 -expect-text 0 0 {#} + +process p1 -screen_dump -write {q} -wait diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc index 009199d..9104c48 100644 --- a/bin/varnishtest/tests/u00010.vtc +++ b/bin/varnishtest/tests/u00010.vtc @@ -9,11 +9,12 @@ varnish v1 -vcl+backend {} -start process p1 -dump {varnishtop -n ${v1_name}} -start -process p1 -need-bytes 1 +process p1 -expect-text 1 1 {list length} client c1 { txreq rxresp } -run -process p1 -need-bytes 500 -screen_dump -write {q} -wait +process p1 -expect-text 0 0 {Fetch_Body} +process p1 -screen_dump -write {q} -wait diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc index c1a6008..d6e8098 100644 --- a/bin/varnishtest/tests/u00011.vtc +++ b/bin/varnishtest/tests/u00011.vtc @@ -16,18 +16,18 @@ varnish v1 -vsl_catchup process p1 -log {varnishadm -n ${v1_name}} -start -process p1 -need-bytes 1 +process p1 -expect-text 0 1 "Type 'quit' to close CLI session." process p1 -write "pi\t\r" -process p1 -need-bytes 100 +process p1 -expect-text 0 1 "PONG" process p1 -write "vcl.li\t\r" -process p1 -need-bytes 200 +process p1 -expect-text 0 1 "active auto/warm" process p1 -write "vcl.s\t\th\tvcl1\r" -process p1 -need-bytes 590 +process p1 -expect-text 0 1 "backend s1" process p1 -screen_dump -write "quit\r" -wait diff --git a/bin/varnishtest/tests/u00012.vtc b/bin/varnishtest/tests/u00012.vtc index 5fb9b35..e982221 100644 --- a/bin/varnishtest/tests/u00012.vtc +++ b/bin/varnishtest/tests/u00012.vtc @@ -18,14 +18,14 @@ process p1 -log {cat | varnishadm -n ${v1_name}} -start process p1 -write "ping\r" -process p1 -need-bytes 30 +process p1 -expect-text 0 1 "PONG" process p1 -write "vcl.list\r" -process p1 -need-bytes 80 +process p1 -expect-text 0 0 "auto/warm" process p1 -write "vcl.show vcl1\r" -process p1 -need-bytes 100 +process p1 -expect-text 0 0 "backend s1" process p1 -screen_dump -write "\x04" -wait From fgsch at lodoss.net Thu Apr 5 10:02:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 5 Apr 2018 10:02:08 +0000 (UTC) Subject: [master] eda8219 Sync Message-ID: <20180405100208.874759A6BA@lists.varnish-cache.org> commit eda8219c99740763225a3a2c7f5a6d7888828802 Author: Federico G. Schwindt Date: Thu Apr 5 11:00:34 2018 +0100 Sync diff --git a/.gitignore b/.gitignore index 2453bd5..74f73ae 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ cscope.*out /bin/varnishncsa/varnishncsa /bin/varnishstat/varnishstat /bin/varnishstat/vsc2rst +/bin/varnishtest/teken_state.h /bin/varnishtest/varnishtest /bin/varnishtop/varnishtop From fgsch at lodoss.net Thu Apr 5 10:02:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 5 Apr 2018 10:02:08 +0000 (UTC) Subject: [master] 72bacde Rework to avoid running the tests twice Message-ID: <20180405100208.9C3FF9A6BD@lists.varnish-cache.org> commit 72bacde7fbffc50c2a62c9988b99d8a830b60d30 Author: Federico G. Schwindt Date: Thu Apr 5 11:00:48 2018 +0100 Rework to avoid running the tests twice diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index 06f2259..392b02a 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -77,11 +77,10 @@ EXTRA_DIST = $(top_srcdir)/bin/varnishtest/tests/*.vtc \ $(top_srcdir)/bin/varnishtest/sequences \ $(top_srcdir)/bin/varnishtest/teken.3 +teken.c: teken_state.h + teken_state.h: $(srcdir)/sequences $(srcdir)/gensequences awk -f $(srcdir)/gensequences $(srcdir)/sequences \ > $(builddir)/teken_state.h -BUILT_SOURCES = teken_state.h - -CLEANFILES = $(BUILT_SOURCES) - +CLEANFILES = $(builddir)/teken_state.h From fgsch at lodoss.net Thu Apr 5 10:02:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 5 Apr 2018 10:02:08 +0000 (UTC) Subject: [master] 8da1584 Correct path Message-ID: <20180405100208.B81709A6C1@lists.varnish-cache.org> commit 8da15843514621c1f6d9a54dacacea9bdeece845 Author: Federico G. Schwindt Date: Thu Apr 5 11:01:01 2018 +0100 Correct path diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 2d556a7..b7d81d4 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -737,7 +737,7 @@ vcc_Var_Init(struct vcc *tl) """) -parse_var_doc(join(buildroot, "doc/sphinx/reference/vcl_var.rst")) +parse_var_doc(join(srcroot, "doc/sphinx/reference/vcl_var.rst")) fo.write("}\n") for i in stv_variables: From nils.goroll at uplex.de Thu Apr 5 10:08:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Apr 2018 10:08:09 +0000 (UTC) Subject: [master] 1193581 flexelint 743 Negative character constant Message-ID: <20180405100809.CC1559A9EC@lists.varnish-cache.org> commit 1193581dd3abaf7e723c24452a54f408c17a2edc Author: Nils Goroll Date: Thu Apr 5 11:40:33 2018 +0200 flexelint 743 Negative character constant diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c index ef72eeb..1c1f13a 100644 --- a/bin/varnishd/cache/cache_esi_parse.c +++ b/bin/varnishd/cache/cache_esi_parse.c @@ -616,7 +616,8 @@ VEP_Parse(struct vep_state *vep, const char *p, size_t l) */ if (vep->state == VEP_START) { - if (FEATURE(FEATURE_ESI_REMOVE_BOM) && *p == '\xeb') { + if (FEATURE(FEATURE_ESI_REMOVE_BOM) && + *p == (char)0xeb) { vep->match = vep_match_bom; vep->state = VEP_MATCH; } else @@ -638,7 +639,7 @@ VEP_Parse(struct vep_state *vep, const char *p, size_t l) if (p < e && *p == '<') { p++; vep->state = VEP_STARTTAG; - } else if (p < e && *p == '\xeb') { + } else if (p < e && *p == (char)0xeb) { VSLb(vep->vc->wrk->vsl, SLT_ESI_xmlerror, "No ESI processing, " "first char not '<' but BOM." From nils.goroll at uplex.de Thu Apr 5 10:08:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Apr 2018 10:08:09 +0000 (UTC) Subject: [master] c0acb4e flexelint 534: Ignoring return value of function Message-ID: <20180405100809.E5F079A9EF@lists.varnish-cache.org> commit c0acb4e3b9c31d3bd4252f31913e7375d1bc396a Author: Nils Goroll Date: Thu Apr 5 11:43:39 2018 +0200 flexelint 534: Ignoring return value of function diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index e2381a4..ad7e312 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -102,14 +102,14 @@ THR_SetName(const char *name) AZ(pthread_setspecific(name_key, name)); #if defined(HAVE_PTHREAD_SET_NAME_NP) - pthread_set_name_np(pthread_self(), name); + (void)pthread_set_name_np(pthread_self(), name); #elif defined(HAVE_PTHREAD_SETNAME_NP) #if defined(__APPLE__) - pthread_setname_np(name); + (void)pthread_setname_np(name); #elif defined(__NetBSD__) - pthread_setname_np(pthread_self(), "%s", (char *)(uintptr_t)name); + (void)pthread_setname_np(pthread_self(), "%s", (char *)(uintptr_t)name); #else - pthread_setname_np(pthread_self(), name); + (void)pthread_setname_np(pthread_self(), name); #endif #endif } From nils.goroll at uplex.de Thu Apr 5 10:08:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Apr 2018 10:08:10 +0000 (UTC) Subject: [master] 1b4f3e7 gc unused includes Message-ID: <20180405100810.0F9EC9A9F6@lists.varnish-cache.org> commit 1b4f3e7022626db23a0aa64bc1575a1b52496cfc Author: Nils Goroll Date: Thu Apr 5 11:45:13 2018 +0200 gc unused includes diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 04e425c..9ae4833 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -35,7 +35,6 @@ #include #include -#include "vend.h" #include "vgz.h" #include "vsl_priv.h" #include "vmb.h" From nils.goroll at uplex.de Thu Apr 5 10:08:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Apr 2018 10:08:10 +0000 (UTC) Subject: [master] 1cab51e flexelint 838: Previously assigned value ... has not been used Message-ID: <20180405100810.30A859A9FC@lists.varnish-cache.org> commit 1cab51e780dbd51947e69359c04d6c633de619eb Author: Nils Goroll Date: Thu Apr 5 11:50:34 2018 +0200 flexelint 838: Previously assigned value ... has not been used diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 5f821ac..022224d 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -311,7 +311,7 @@ vpx_proto2(const struct worker *wrk, struct req *req) int l, hdr_len; uintptr_t *up; uint16_t *tlv_len_p; - uint16_t tlv_len = 0; + uint16_t tlv_len; const uint8_t *p; char *d, *tlv_start; sa_family_t pfam = 0xff; From nils.goroll at uplex.de Thu Apr 5 11:57:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Apr 2018 11:57:11 +0000 (UTC) Subject: [master] f3b28af the use case referenced here never made it upstream Message-ID: <20180405115711.1DF43A0CAD@lists.varnish-cache.org> commit f3b28af13495149a2a8072f0235954fcfc11e7b7 Author: Nils Goroll Date: Thu Apr 5 12:44:37 2018 +0200 the use case referenced here never made it upstream diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c index 8925d92..3b8de08 100644 --- a/bin/varnishd/mgt/mgt_jail_solaris.c +++ b/bin/varnishd/mgt/mgt_jail_solaris.c @@ -355,8 +355,6 @@ vjs_add_permitted(priv_set_t *pset, enum jail_gen_e jge) case JAILG_SUBPROC_VCLLOAD: break; case JAILG_SUBPROC_WORKER: - /* for raising limits in cache_waiter_ports.c */ - AZ(priv_addset(pset, PRIV_SYS_RESOURCE)); break; default: INCOMPL(); From nils.goroll at uplex.de Thu Apr 5 12:17:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Apr 2018 12:17:08 +0000 (UTC) Subject: [master] 5879f40 oops Message-ID: <20180405121708.C6A17A26A9@lists.varnish-cache.org> commit 5879f40414f8639a5c4585f87a7cdc523cb06f73 Author: Nils Goroll Date: Thu Apr 5 14:16:31 2018 +0200 oops diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c index 3b8de08..ef80c90 100644 --- a/bin/varnishd/mgt/mgt_jail_solaris.c +++ b/bin/varnishd/mgt/mgt_jail_solaris.c @@ -349,6 +349,7 @@ vjs_add_effective(priv_set_t *pset, enum jail_gen_e jge) static void vjs_add_permitted(priv_set_t *pset, enum jail_gen_e jge) { + (void) pset; switch (jge) { case JAILG_SUBPROC_VCC: case JAILG_SUBPROC_CC: From nils.goroll at uplex.de Thu Apr 5 13:18:12 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Apr 2018 13:18:12 +0000 (UTC) Subject: [master] 9351778 support for vmod_unix on solar-ish OSes Message-ID: <20180405131812.28F52A3865@lists.varnish-cache.org> commit 93517783f3daf2c0f84d8417c19024641ed8a858 Author: Nils Goroll Date: Thu Apr 5 13:58:19 2018 +0200 support for vmod_unix on solar-ish OSes note on jail_solaris: ideally, vmods could tell the jails about privileges they require, but for now (and because vmod_unix lives in varnish-cache), just add the required privilege to the permitted set. I have also considered the option to add additional privileges via the -j argument (and actually would still want to add that somewhen), but for this purpose, varnish should really DTRT by default. note on priv_allocset: Most of the sun folk had proven to be good interface designers, but an API which requires dynamic allocation/deallocation really does not play well with my efficiency fetish. So good we got library constructors/destructors. note on getpeerucred: basically the same thing, but this time they at least added ucred_size(). Not intended for use like this, but anyway.... diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c index ef80c90..dc19212 100644 --- a/bin/varnishd/mgt/mgt_jail_solaris.c +++ b/bin/varnishd/mgt/mgt_jail_solaris.c @@ -356,6 +356,8 @@ vjs_add_permitted(priv_set_t *pset, enum jail_gen_e jge) case JAILG_SUBPROC_VCLLOAD: break; case JAILG_SUBPROC_WORKER: + /* vmod_unix getpeerucred() */ + AZ(priv_addset(pset, PRIV_PROC_INFO)); break; default: INCOMPL(); diff --git a/configure.ac b/configure.ac index 3aeb7bb..48ab568 100644 --- a/configure.ac +++ b/configure.ac @@ -217,6 +217,7 @@ AC_CHECK_FUNCS([fallocate]) AC_CHECK_FUNCS([closefrom]) AC_CHECK_FUNCS([sigaltstack]) AC_CHECK_FUNCS([getpeereid]) +AC_CHECK_FUNCS([getpeerucred]) save_LIBS="${LIBS}" LIBS="${PTHREAD_LIBS}" diff --git a/lib/libvmod_unix/cred_compat.h b/lib/libvmod_unix/cred_compat.h index 6a79636..3c5ff08 100644 --- a/lib/libvmod_unix/cred_compat.h +++ b/lib/libvmod_unix/cred_compat.h @@ -25,8 +25,6 @@ * */ -#include "config.h" - #include #include #include @@ -37,11 +35,35 @@ #if defined(HAVE_GETPEERUCRED) #include +# if defined(HAVE_SETPPRIV) +# include +static priv_set_t *priv_proc_info = NULL; +# endif #endif #define CREDS_FAIL -1 #define NOT_SUPPORTED -2 +#if defined(HAVE_GETPEERUCRED) && defined(HAVE_SETPPRIV) +static void __attribute__((constructor)) +cred_compat_init(void) +{ + AZ(priv_proc_info); + priv_proc_info = priv_allocset(); + AN(priv_proc_info); + AZ(priv_addset(priv_proc_info, PRIV_PROC_INFO)); +} + +static void __attribute__((destructor)) +cred_compat_fini(void) +{ + if (priv_proc_info == NULL) + return; + priv_freeset(priv_proc_info); + priv_proc_info = NULL; +} +#endif + static int get_ids(int fd, uid_t *uid, gid_t *gid) { @@ -65,6 +87,33 @@ get_ids(int fd, uid_t *uid, gid_t *gid) return (CREDS_FAIL); return (0); +#elif defined(HAVE_GETPEERUCRED) + char buf[ucred_size()]; + ucred_t *ucredp = (ucred_t *)buf; + +# if defined(HAVE_SETPPRIV) + priv_set_t *priv = NULL; + + errno = 0; + if (! priv_ineffect(PRIV_PROC_INFO)) { + priv = priv_proc_info; + if (setppriv(PRIV_ON, PRIV_EFFECTIVE, priv)) + return (CREDS_FAIL); + } +# endif + + errno = 0; + if (getpeerucred(fd, &ucredp)) + return (CREDS_FAIL); + *uid = ucred_getruid(ucredp); + *gid = ucred_getrgid(ucredp); + +# if defined(HAVE_SETPPRIV) + if (priv != NULL) + AZ(setppriv(PRIV_OFF, PRIV_EFFECTIVE, priv)); // waive +# endif + + return (0); #else (void) fd; (void) uid; diff --git a/lib/libvmod_unix/vmod_unix.c b/lib/libvmod_unix/vmod_unix.c index ee1f3e0..f51c108 100644 --- a/lib/libvmod_unix/vmod_unix.c +++ b/lib/libvmod_unix/vmod_unix.c @@ -25,7 +25,7 @@ * */ -#include "cred_compat.h" +#include "config.h" #include #include @@ -35,6 +35,7 @@ #include "vcl.h" #include "common/heritage.h" +#include "cred_compat.h" #include "vcc_if.h" #define FAIL(ctx, msg) \ From phk at FreeBSD.org Fri Apr 6 10:54:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 6 Apr 2018 10:54:14 +0000 (UTC) Subject: [master] c4788f9 More code coverage of Teken Message-ID: <20180406105414.506B9658E3@lists.varnish-cache.org> commit c4788f90cdeee0886fa612835fdc9299219cc7ca Author: Poul-Henning Kamp Date: Fri Apr 6 10:52:53 2018 +0000 More code coverage of Teken diff --git a/bin/varnishtest/tests/a00001.vtc b/bin/varnishtest/tests/a00001.vtc index 1dd3f36..16f0437 100644 --- a/bin/varnishtest/tests/a00001.vtc +++ b/bin/varnishtest/tests/a00001.vtc @@ -2,7 +2,7 @@ varnishtest "Test Teken terminal emulator" feature cmd "vttest --version 2>&1 | grep -q Usage" -process p4 -dump {vttest} -start +process p4 {vttest} -start process p4 -expect-text 21 11 "Enter choice number (0 - 12):" process p4 -screen_dump @@ -37,6 +37,73 @@ process p4 -writehex 0d process p4 -expect-text 21 11 "Enter choice number (0 - 12):" process p4 -screen_dump +process p4 -writehex "32 0d" +process p4 -expect-text 8 1 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 5 1 "should look the same. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 0 0 "This is 132 column mode, light background.Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 0 0 "This is 80 column mode, light background.Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 0 0 "This is 132 column mode, dark background.Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 0 0 "This is 80 column mode, dark background.Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 0 1 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 3 1 "Soft scroll down region [1..24] size 24 Line 28" +process p4 -expect-text 1 1 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 0 1 "Jump scroll down region [12..13] size 2 Line 29" +process p4 -expect-text 0 1 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 3 1 "Jump scroll down region [1..24] size 24 Line 28" +process p4 -expect-text 1 1 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 23 0 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 1 0 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 23 1 "Dark background. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 23 1 "Light background. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 24 1 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 21 11 "Enter choice number (0 - 12):" +process p4 -screen_dump + process p4 -writehex "30 0d" process p4 -expect-text 12 30 "That's all, folks!" process p4 -screen_dump From geoff at uplex.de Fri Apr 6 12:54:14 2018 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 6 Apr 2018 12:54:14 +0000 (UTC) Subject: [master] 52d8dd8 VMOD unix obtains effective uid/gid on solarishy platforms. Message-ID: <20180406125414.45A7F92E82@lists.varnish-cache.org> commit 52d8dd87226db6b665076b690d25513d038d7647 Author: Geoff Simmons Date: Fri Apr 6 14:51:45 2018 +0200 VMOD unix obtains effective uid/gid on solarishy platforms. Not real ids; for consistency with the other platforms. diff --git a/lib/libvmod_unix/cred_compat.h b/lib/libvmod_unix/cred_compat.h index 3c5ff08..843793a 100644 --- a/lib/libvmod_unix/cred_compat.h +++ b/lib/libvmod_unix/cred_compat.h @@ -105,8 +105,8 @@ get_ids(int fd, uid_t *uid, gid_t *gid) errno = 0; if (getpeerucred(fd, &ucredp)) return (CREDS_FAIL); - *uid = ucred_getruid(ucredp); - *gid = ucred_getrgid(ucredp); + *uid = ucred_geteuid(ucredp); + *gid = ucred_getegid(ucredp); # if defined(HAVE_SETPPRIV) if (priv != NULL) From geoff at uplex.de Fri Apr 6 13:38:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 6 Apr 2018 13:38:10 +0000 (UTC) Subject: [master] 4e085f9 Document support of SunOS and friends in VMOD unix. Message-ID: <20180406133810.3CEB193C3F@lists.varnish-cache.org> commit 4e085f9a9cb934e45bc8065983803c83bf01d18e Author: Geoff Simmons Date: Fri Apr 6 15:37:22 2018 +0200 Document support of SunOS and friends in VMOD unix. diff --git a/lib/libvmod_unix/vmod.vcc b/lib/libvmod_unix/vmod.vcc index d698a91..e70a092 100644 --- a/lib/libvmod_unix/vmod.vcc +++ b/lib/libvmod_unix/vmod.vcc @@ -53,6 +53,12 @@ one of the following: * the socket option ``SO_PEERCRED`` for ``getsockopt(2)`` (Linux) +* ``getpeerucred(3C)`` (SunOS and descendants) + +On SunOS and friends, the ``PRIV_PROC_INFO`` privilege set is added to +the Varnish child process while the VMOD is loaded, see +``setppriv(2)``. + On most platforms, the value returned is the effective user or group that was valid when the peer process initiated the connection. @@ -104,3 +110,5 @@ SEE ALSO * :ref:`vcl(7)` * ``getpeereid(3)`` * ``getsockopt(2)`` +* ``getpeerucred(3C)`` +* ``setppriv(2)`` From geoff at uplex.de Fri Apr 6 13:59:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 6 Apr 2018 13:59:10 +0000 (UTC) Subject: [master] fabdf63 Document VMOD unix getpeerucred(3) support in changes.rst. Message-ID: <20180406135910.2D0919432C@lists.varnish-cache.org> commit fabdf63ac2e4d9e3b4bccf02bce2f309d662980b Author: Geoff Simmons Date: Fri Apr 6 15:57:44 2018 +0200 Document VMOD unix getpeerucred(3) support in changes.rst. diff --git a/doc/changes.rst b/doc/changes.rst index 68bfe51..a8dd755 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -2,6 +2,11 @@ Varnish Cache Trunk (ongoing) ============================= +VCL and bundled VMODs +--------------------- + +* VMOD unix now supports the ``getpeerucred(3)`` case. + bundled tools ------------- From nils.goroll at uplex.de Fri Apr 6 15:20:13 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 6 Apr 2018 15:20:13 +0000 (UTC) Subject: [master] 55667f6 varnishstat: get uptime for averages even if not in filter Message-ID: <20180406152013.5530F95B9B@lists.varnish-cache.org> commit 55667f6a7485174a65c222e2e5fc876a5d6643bc Author: Nils Goroll Date: Fri Apr 6 17:18:19 2018 +0200 varnishstat: get uptime for averages even if not in filter fixes #2639 diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index 6b33e00..0a78a45 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -199,12 +199,17 @@ do_once_cb(void *priv, const struct VSC_point * const pt) static void do_once(struct vsm *vsm, struct vsc *vsc) { + struct vsc *vsconce = VSC_New(); struct once_priv op; + AN(vsconce); + AN(VSC_Arg(vsconce, 'f', "MAIN.uptime")); + memset(&op, 0, sizeof op); op.pad = 18; - (void)VSC_Iter(vsc, vsm, do_once_cb_first, &op); + (void)VSC_Iter(vsconce, vsm, do_once_cb_first, &op); + VSC_Destroy(&vsconce, vsm); (void)VSC_Iter(vsc, vsm, do_once_cb, &op); } From nils.goroll at uplex.de Fri Apr 6 15:24:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 6 Apr 2018 15:24:11 +0000 (UTC) Subject: [master] 40625f5 fix proxy backend in pipe mode Message-ID: <20180406152411.8A8FC95CF5@lists.varnish-cache.org> commit 40625f511564b1489b27aa2c8c76208b840d9878 Author: Nils Goroll Date: Thu Mar 15 16:58:42 2018 +0100 fix proxy backend in pipe mode Fixes #2613 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 4c8fad0..e327c0d 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -693,6 +693,8 @@ cnt_pipe(struct worker *wrk, struct req *req) VSLb(bo->vsl, SLT_Begin, "bereq %u pipe", VXID(req->vsl->wid)); VSLb(req->vsl, SLT_Link, "bereq %u pipe", VXID(bo->vsl->wid)); THR_SetBusyobj(bo); + bo->sp = req->sp; + SES_Ref(bo->sp); HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod); http_FilterReq(bo->bereq, req->http, 0); // XXX: 0 ? @@ -726,6 +728,7 @@ cnt_pipe(struct worker *wrk, struct req *req) WRONG("Illegal return from vcl_pipe{}"); } http_Teardown(bo->bereq); + SES_Rel(bo->sp); VBO_ReleaseBusyObj(wrk, &bo); THR_SetBusyobj(NULL); return (nxt); diff --git a/bin/varnishtest/tests/o00002.vtc b/bin/varnishtest/tests/o00002.vtc index 13fba7d..9217afa 100644 --- a/bin/varnishtest/tests/o00002.vtc +++ b/bin/varnishtest/tests/o00002.vtc @@ -34,6 +34,13 @@ server s1 { expect req.http.xyzzy1 == 4444 expect req.http.x-forwarded-for == "1:f::2, 1:f::2" txresp -body "proxy4" + + rxreq + expect req.url == "/pipe" + expect req.http.xyzzy1 == req.http.xyzzy2 + expect req.http.xyzzy1 == 5555 + expect req.http.x-forwarded-for == "1:f::2, 1:f::2" + txresp -hdr "Connection: close" -body "pipe" } -start varnish v1 -proto PROXY -vcl+backend { @@ -64,6 +71,9 @@ varnish v2 -proto PROXY -vcl { } else { set req.backend_hint = bp2; } + if (req.url ~ "^/pipe") { + return (pipe); + } } sub vcl_deliver { set resp.http.connection = "close"; @@ -97,3 +107,11 @@ client c1 -connect ${v2_sock} -proxy1 "[1:f::2]:4444 [5:a::8]:5678" { expect resp.body == "proxy4" } -run delay .2 + +client c1 -connect ${v2_sock} -proxy1 "[1:f::2]:5555 [5:a::8]:5678" { + txreq -url /pipe + rxresp + expect resp.body == "pipe" + expect resp.http.Connection == "close" + expect_close +} -run From phk at FreeBSD.org Fri Apr 6 19:40:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 6 Apr 2018 19:40:11 +0000 (UTC) Subject: [master] 5545e60 Specifying the process->spec twice in one line is almost certainly an error, so make it one. Message-ID: <20180406194011.4FFED9AD22@lists.varnish-cache.org> commit 5545e602918df430f4c3de84c7b5ddc29d469a39 Author: Poul-Henning Kamp Date: Fri Apr 6 19:38:22 2018 +0000 Specifying the process->spec twice in one line is almost certainly an error, so make it one. diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 0dbe46f..1bbf137 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -859,6 +859,7 @@ cmd_process(CMD_ARGS) struct process *p, *p2; uintmax_t u, v; unsigned lin,col; + int spec_set = 0; (void)priv; (void)cmd; @@ -1009,9 +1010,10 @@ cmd_process(CMD_ARGS) av++; continue; } - if (**av == '-') + if (**av == '-' || spec_set) vtc_fatal(p->vl, "Unknown process argument: %s", *av); REPLACE(p->spec, *av); + spec_set = 1; } } From phk at FreeBSD.org Fri Apr 6 21:43:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 6 Apr 2018 21:43:11 +0000 (UTC) Subject: [master] f69b4ae Report if killing haproxy fails Message-ID: <20180406214311.68B0FA26C4@lists.varnish-cache.org> commit f69b4ae7d3e70eaa4c34a0d6454fe8c334f07b00 Author: Poul-Henning Kamp Date: Fri Apr 6 21:41:46 2018 +0000 Report if killing haproxy fails diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index f8eec12..21a6cb3 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -324,6 +324,9 @@ haproxy_wait(struct haproxy *h) vtc_log(h->vl, 2, "Stop HAproxy pid=%ld", (long)h->pid); h->kill_status = kill(h->pid, HAPROXY_SIGNAL); h->kill_errno = errno; + if (h->kill_status) + vtc_log(h->vl, 4, "Kill=%d: %s", + h->kill_status, strerror(h->kill_errno)); h->expect_signal = -HAPROXY_SIGNAL; // XXX: loop over kills to ESRCH ? } From phk at FreeBSD.org Sat Apr 7 05:54:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 7 Apr 2018 05:54:13 +0000 (UTC) Subject: [master] 661c189 Try even harder to kill haproxy during cleanup Message-ID: <20180407055413.242A0AD1F9@lists.varnish-cache.org> commit 661c189bb1d2d609e132d77319543e66ff55f02d Author: Poul-Henning Kamp Date: Sat Apr 7 05:53:05 2018 +0000 Try even harder to kill haproxy during cleanup diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 21a6cb3..ea66f81 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -70,8 +70,7 @@ struct haproxy { pthread_t tp; int expect_exit; int expect_signal; - int kill_status; - int kill_errno; + int its_dead_jim; const char *cli_fn; @@ -213,7 +212,9 @@ haproxy_thread(void *priv) struct haproxy *h; CAST_OBJ_NOTNULL(h, priv, HAPROXY_MAGIC); - return (vtc_record(h->vl, h->fds[0], h->msgs)); + (void)vtc_record(h->vl, h->fds[0], h->msgs); + h->its_dead_jim = 1; + return (NULL); } /********************************************************************** @@ -311,6 +312,7 @@ static void haproxy_wait(struct haproxy *h) { void *p; + int i; vtc_log(h->vl, 2, "Wait"); @@ -319,16 +321,16 @@ haproxy_wait(struct haproxy *h) closefd(&h->fds[1]); - if (!h->opt_check_mode) { + while (!h->opt_check_mode && !h->its_dead_jim) { assert(h->pid > 0); vtc_log(h->vl, 2, "Stop HAproxy pid=%ld", (long)h->pid); - h->kill_status = kill(h->pid, HAPROXY_SIGNAL); - h->kill_errno = errno; - if (h->kill_status) - vtc_log(h->vl, 4, "Kill=%d: %s", - h->kill_status, strerror(h->kill_errno)); + i= kill(h->pid, HAPROXY_SIGNAL); h->expect_signal = -HAPROXY_SIGNAL; - // XXX: loop over kills to ESRCH ? + if (i && errno == ESRCH) + break; + if (i) + vtc_log(h->vl, 4, "Kill=%d: %s", i, strerror(errno)); + usleep(100000); } AZ(pthread_join(h->tp, &p)); From phk at FreeBSD.org Sat Apr 7 10:28:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 7 Apr 2018 10:28:10 +0000 (UTC) Subject: [master] 815e32a Escalate the signals if HAproxy refuses to die Message-ID: <20180407102810.22BFBB2249@lists.varnish-cache.org> commit 815e32a024c0c4c8969d649fc8518528fc3157d2 Author: Poul-Henning Kamp Date: Sat Apr 7 10:27:04 2018 +0000 Escalate the signals if HAproxy refuses to die diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index ea66f81..73f42b8 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -312,7 +312,7 @@ static void haproxy_wait(struct haproxy *h) { void *p; - int i; + int i, n, sig; vtc_log(h->vl, 2, "Wait"); @@ -321,16 +321,28 @@ haproxy_wait(struct haproxy *h) closefd(&h->fds[1]); + sig = SIGINT; + n = 0; + vtc_log(h->vl, 2, "Stop HAproxy pid=%ld", (long)h->pid); while (!h->opt_check_mode && !h->its_dead_jim) { assert(h->pid > 0); - vtc_log(h->vl, 2, "Stop HAproxy pid=%ld", (long)h->pid); - i= kill(h->pid, HAPROXY_SIGNAL); - h->expect_signal = -HAPROXY_SIGNAL; - if (i && errno == ESRCH) - break; - if (i) - vtc_log(h->vl, 4, "Kill=%d: %s", i, strerror(errno)); + if (n == 0) { + i= kill(h->pid, sig); + vtc_log(h->vl, 4, + "Kill(%d)=%d: %s", sig, i, strerror(errno)); + h->expect_signal = -sig; + if (i && errno == ESRCH) + break; + } usleep(100000); + if (++n == 20) { + switch (sig) { + case SIGINT: sig = SIGTERM ; break; + case SIGTERM: sig = SIGKILL ; break; + default: break; + } + n = 0; + } } AZ(pthread_join(h->tp, &p)); From phk at FreeBSD.org Sat Apr 7 13:00:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 7 Apr 2018 13:00:13 +0000 (UTC) Subject: [master] 6d40577 Even more Teken coverage Message-ID: <20180407130013.95185B4E94@lists.varnish-cache.org> commit 6d40577825d8096bcf6678e7cfcda5c1ce5f7d32 Author: Poul-Henning Kamp Date: Sat Apr 7 12:59:33 2018 +0000 Even more Teken coverage diff --git a/bin/varnishtest/tests/a00001.vtc b/bin/varnishtest/tests/a00001.vtc index 16f0437..dfca677 100644 --- a/bin/varnishtest/tests/a00001.vtc +++ b/bin/varnishtest/tests/a00001.vtc @@ -7,6 +7,7 @@ process p4 {vttest} -start process p4 -expect-text 21 11 "Enter choice number (0 - 12):" process p4 -screen_dump +# 1. Test of cursor movements process p4 -writehex "31 0d" process p4 -expect-text 14 61 "RETURN" process p4 -screen_dump @@ -37,6 +38,7 @@ process p4 -writehex 0d process p4 -expect-text 21 11 "Enter choice number (0 - 12):" process p4 -screen_dump +# 2. Test of screen features process p4 -writehex "32 0d" process p4 -expect-text 8 1 "Push " process p4 -screen_dump @@ -104,6 +106,105 @@ process p4 -writehex 0d process p4 -expect-text 21 11 "Enter choice number (0 - 12):" process p4 -screen_dump +# 4. Test of double-sized characters +process p4 -writehex "34 0d" +process p4 -expect-text 21 1 "This is not a double-width line" +process p4 -expect-text 23 1 "Push " +process p4 -screen_dump + +process p4 -writehex "0d" +process p4 -expect-text 21 1 "This **is** a double-width line" +process p4 -expect-text 23 1 "Push " +process p4 -screen_dump + +process p4 -writehex "0d" +process p4 -expect-text 21 1 "This is not a double-width line" +process p4 -expect-text 23 1 "Push " +process p4 -screen_dump + +process p4 -writehex "0d" +process p4 -expect-text 21 1 "This **is** a double-width line" +process p4 -expect-text 23 1 "Push " +process p4 -screen_dump + +process p4 -writehex "0d" +delay 2 +process p4 -expect-text 23 41 "Push " +process p4 -screen_dump + +process p4 -writehex "0d" +delay 2 +process p4 -expect-text 1 1 "Exactly half of the box should remain. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 21 11 "Enter choice number (0 - 12):" +process p4 -screen_dump + +# 8. Test of VT102 features (Insert/Delete Char/Line) +process p4 -writehex "38 0d" +process p4 -expect-text 4 1 "Screen accordion test (Insert & Delete Line). Push D" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 2 45 "nothing more. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 4 59 "*B'. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 4 52 "'AB'. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 5 1 "by one. Push E" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 5 1 "by one. Push EEEEEEEEEEEEE " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 10 1 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 4 1 "Screen accordion test (Insert & Delete Line). Push D" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 2 45 "nothing more. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 4 59 "*B'. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 4 52 "'AB'. Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 5 1 "by one. Push E" +process p4 -screen_dump + + +process p4 -writehex 0d +process p4 -expect-text 5 59 "EEE " +process p4 -expect-text 5 1 "by one. Push E" +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 10 1 "Push " +process p4 -screen_dump + +process p4 -writehex 0d +process p4 -expect-text 21 11 "Enter choice number (0 - 12):" +process p4 -screen_dump + +# 0. Exit process p4 -writehex "30 0d" process p4 -expect-text 12 30 "That's all, folks!" process p4 -screen_dump diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 1bbf137..be76323 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -289,7 +289,7 @@ static void term_expect_text(struct process *pp, const char *lin, const char *col, const char *pat) { - int x, y, l; + int x, y, l, d = 10000; char *t; y = strtoul(lin, NULL, 0); @@ -303,8 +303,10 @@ term_expect_text(struct process *pp, "text at %d,%d: '%.*s'", y, x, l, t); } AZ(pthread_mutex_unlock(&pp->mtx)); - usleep(1000000); + usleep(d); AZ(pthread_mutex_lock(&pp->mtx)); + if (d < 300000) + d += d; } AZ(pthread_mutex_unlock(&pp->mtx)); vtc_log(pp->vl, 4, "found expected text at %d,%d: '%s'", y, x, pat); From phk at FreeBSD.org Sat Apr 7 14:42:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 7 Apr 2018 14:42:09 +0000 (UTC) Subject: [master] 67f5fad Add a delay to allow the utils to attach the shmem before running the transaction. Message-ID: <20180407144209.9C5ADB6AFE@lists.varnish-cache.org> commit 67f5fadd287e217864b84a3c0b06b741d0dc2955 Author: Poul-Henning Kamp Date: Sat Apr 7 14:41:07 2018 +0000 Add a delay to allow the utils to attach the shmem before running the transaction. diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc index eb587bc..90b7a3d 100644 --- a/bin/varnishtest/tests/u00009.vtc +++ b/bin/varnishtest/tests/u00009.vtc @@ -9,13 +9,15 @@ varnish v1 -vcl+backend {} -start process p1 -dump {varnishhist -n ${v1_name}} -start -process p1 -expect-text 0 0 {1e2} +process p1 -expect-text 24 0 {1e2} + +delay 1 client c1 { txreq rxresp } -run -process p1 -expect-text 0 0 {#} +process p1 -expect-text 22 0 {#} process p1 -screen_dump -write {q} -wait diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc index 9104c48..fc555ff 100644 --- a/bin/varnishtest/tests/u00010.vtc +++ b/bin/varnishtest/tests/u00010.vtc @@ -11,6 +11,8 @@ process p1 -dump {varnishtop -n ${v1_name}} -start process p1 -expect-text 1 1 {list length} +delay 1 + client c1 { txreq rxresp From phk at FreeBSD.org Sun Apr 8 08:03:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 8 Apr 2018 08:03:13 +0000 (UTC) Subject: [master] 6d4a95e Remove parts of the libvgz API which we dont use. Message-ID: <20180408080313.3A492A2F01@lists.varnish-cache.org> commit 6d4a95e0376bb33d4d95335d22166a6bf08a1a1e Author: Poul-Henning Kamp Date: Sun Apr 8 08:02:23 2018 +0000 Remove parts of the libvgz API which we dont use. diff --git a/lib/libvgz/Makefile.am b/lib/libvgz/Makefile.am index 74afccf..db9a48b 100644 --- a/lib/libvgz/Makefile.am +++ b/lib/libvgz/Makefile.am @@ -9,13 +9,11 @@ libvgz_a_CFLAGS = -D_LARGEFILE64_SOURCE=1 -DZLIB_CONST \ libvgz_a_SOURCES = \ adler32.c \ - compress.c \ crc32.c \ crc32.h \ deflate.c \ deflate.h \ gzguts.h \ - infback.c \ inffast.c \ inffast.h \ inffixed.h \ @@ -25,7 +23,6 @@ libvgz_a_SOURCES = \ inftrees.h \ trees.c \ trees.h \ - uncompr.c \ zconf.h \ vgz.h \ zutil.c \ diff --git a/lib/libvgz/compress.c b/lib/libvgz/compress.c deleted file mode 100644 index 6134fad..0000000 --- a/lib/libvgz/compress.c +++ /dev/null @@ -1,86 +0,0 @@ -/* compress.c -- compress a memory buffer - * Copyright (C) 1995-2005, 2014, 2016 Jean-loup Gailly, Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* @(#) $Id$ */ - -#define ZLIB_INTERNAL -#include "vgz.h" - -/* =========================================================================== - Compresses the source buffer into the destination buffer. The level - parameter has the same meaning as in deflateInit. sourceLen is the byte - length of the source buffer. Upon entry, destLen is the total size of the - destination buffer, which must be at least 0.1% larger than sourceLen plus - 12 bytes. Upon exit, destLen is the actual size of the compressed buffer. - - compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough - memory, Z_BUF_ERROR if there was not enough room in the output buffer, - Z_STREAM_ERROR if the level parameter is invalid. -*/ -int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) - Bytef *dest; - uLongf *destLen; - const Bytef *source; - uLong sourceLen; - int level; -{ - z_stream stream; - int err; - const uInt max = (uInt)-1; - uLong left; - - left = *destLen; - *destLen = 0; - - stream.zalloc = (alloc_func)0; - stream.zfree = (free_func)0; - stream.opaque = (voidpf)0; - - err = deflateInit(&stream, level); - if (err != Z_OK) return err; - - stream.next_out = dest; - stream.avail_out = 0; - stream.next_in = (z_const Bytef *)source; - stream.avail_in = 0; - - do { - if (stream.avail_out == 0) { - stream.avail_out = left > (uLong)max ? max : (uInt)left; - left -= stream.avail_out; - } - if (stream.avail_in == 0) { - stream.avail_in = sourceLen > (uLong)max ? max : (uInt)sourceLen; - sourceLen -= stream.avail_in; - } - err = deflate(&stream, sourceLen ? Z_NO_FLUSH : Z_FINISH); - } while (err == Z_OK); - - *destLen = stream.total_out; - deflateEnd(&stream); - return err == Z_STREAM_END ? Z_OK : err; -} - -/* =========================================================================== - */ -int ZEXPORT compress (dest, destLen, source, sourceLen) - Bytef *dest; - uLongf *destLen; - const Bytef *source; - uLong sourceLen; -{ - return compress2(dest, destLen, source, sourceLen, Z_DEFAULT_COMPRESSION); -} - -/* =========================================================================== - If the default memLevel or windowBits for deflateInit() is changed, then - this function needs to be updated. - */ -uLong ZEXPORT compressBound (sourceLen) - uLong sourceLen; -{ - return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + - (sourceLen >> 25) + 13; -} diff --git a/lib/libvgz/infback.c b/lib/libvgz/infback.c deleted file mode 100644 index d632415..0000000 --- a/lib/libvgz/infback.c +++ /dev/null @@ -1,640 +0,0 @@ -/* infback.c -- inflate using a call-back interface - * Copyright (C) 1995-2016 Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* - This code is largely copied from inflate.c. Normally either infback.o or - inflate.o would be linked into an application--not both. The interface - with inffast.c is retained so that optimized assembler-coded versions of - inflate_fast() can be used with either inflate.c or infback.c. - */ - -#include "zutil.h" -#include "inftrees.h" -#include "inflate.h" -#include "inffast.h" - -/* function prototypes */ -local void fixedtables OF((struct inflate_state FAR *state)); - -/* - strm provides memory allocation functions in zalloc and zfree, or - Z_NULL to use the library memory allocation functions. - - windowBits is in the range 8..15, and window is a user-supplied - window and output buffer that is 2**windowBits bytes. - */ -int ZEXPORT inflateBackInit_(strm, windowBits, window, version, stream_size) -z_streamp strm; -int windowBits; -unsigned char FAR *window; -const char *version; -int stream_size; -{ - struct inflate_state FAR *state; - - if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || - stream_size != (int)(sizeof(z_stream))) - return Z_VERSION_ERROR; - if (strm == Z_NULL || window == Z_NULL || - windowBits < 8 || windowBits > 15) - return Z_STREAM_ERROR; - strm->msg = Z_NULL; /* in case we return an error */ - if (strm->zalloc == (alloc_func)0) { -#ifdef Z_SOLO - return Z_STREAM_ERROR; -#else - strm->zalloc = zcalloc; - strm->opaque = (voidpf)0; -#endif - } - if (strm->zfree == (free_func)0) -#ifdef Z_SOLO - return Z_STREAM_ERROR; -#else - strm->zfree = zcfree; -#endif - state = (struct inflate_state FAR *)ZALLOC(strm, 1, - sizeof(struct inflate_state)); - if (state == Z_NULL) return Z_MEM_ERROR; - Tracev((stderr, "inflate: allocated\n")); - strm->state = (struct internal_state FAR *)state; - state->dmax = 32768U; - state->wbits = (uInt)windowBits; - state->wsize = 1U << windowBits; - state->window = window; - state->wnext = 0; - state->whave = 0; - return Z_OK; -} - -/* - Return state with length and distance decoding tables and index sizes set to - fixed code decoding. Normally this returns fixed tables from inffixed.h. - If BUILDFIXED is defined, then instead this routine builds the tables the - first time it's called, and returns those tables the first time and - thereafter. This reduces the size of the code by about 2K bytes, in - exchange for a little execution time. However, BUILDFIXED should not be - used for threaded applications, since the rewriting of the tables and virgin - may not be thread-safe. - */ -local void fixedtables(state) -struct inflate_state FAR *state; -{ -#ifdef BUILDFIXED - static int virgin = 1; - static code *lenfix, *distfix; - static code fixed[544]; - - /* build fixed huffman tables if first call (may not be thread safe) */ - if (virgin) { - unsigned sym, bits; - static code *next; - - /* literal/length table */ - sym = 0; - while (sym < 144) state->lens[sym++] = 8; - while (sym < 256) state->lens[sym++] = 9; - while (sym < 280) state->lens[sym++] = 7; - while (sym < 288) state->lens[sym++] = 8; - next = fixed; - lenfix = next; - bits = 9; - inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); - - /* distance table */ - sym = 0; - while (sym < 32) state->lens[sym++] = 5; - distfix = next; - bits = 5; - inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); - - /* do this just once */ - virgin = 0; - } -#else /* !BUILDFIXED */ -# include "inffixed.h" -#endif /* BUILDFIXED */ - state->lencode = lenfix; - state->lenbits = 9; - state->distcode = distfix; - state->distbits = 5; -} - -/* Macros for inflateBack(): */ - -/* Load returned state from inflate_fast() */ -#define LOAD() \ - do { \ - put = strm->next_out; \ - left = strm->avail_out; \ - next = strm->next_in; \ - have = strm->avail_in; \ - hold = state->hold; \ - bits = state->bits; \ - } while (0) - -/* Set state from registers for inflate_fast() */ -#define RESTORE() \ - do { \ - strm->next_out = put; \ - strm->avail_out = left; \ - strm->next_in = next; \ - strm->avail_in = have; \ - state->hold = hold; \ - state->bits = bits; \ - } while (0) - -/* Clear the input bit accumulator */ -#define INITBITS() \ - do { \ - hold = 0; \ - bits = 0; \ - } while (0) - -/* Assure that some input is available. If input is requested, but denied, - then return a Z_BUF_ERROR from inflateBack(). */ -#define PULL() \ - do { \ - if (have == 0) { \ - have = in(in_desc, &next); \ - if (have == 0) { \ - next = Z_NULL; \ - ret = Z_BUF_ERROR; \ - goto inf_leave; \ - } \ - } \ - } while (0) - -/* Get a byte of input into the bit accumulator, or return from inflateBack() - with an error if there is no input available. */ -#define PULLBYTE() \ - do { \ - PULL(); \ - have--; \ - hold += (unsigned long)(*next++) << bits; \ - bits += 8; \ - } while (0) - -/* Assure that there are at least n bits in the bit accumulator. If there is - not enough available input to do that, then return from inflateBack() with - an error. */ -#define NEEDBITS(n) \ - do { \ - while (bits < (unsigned)(n)) \ - PULLBYTE(); \ - } while (0) - -/* Return the low n bits of the bit accumulator (n < 16) */ -#define BITS(n) \ - ((unsigned)hold & ((1U << (n)) - 1)) - -/* Remove n bits from the bit accumulator */ -#define DROPBITS(n) \ - do { \ - hold >>= (n); \ - bits -= (unsigned)(n); \ - } while (0) - -/* Remove zero to seven bits as needed to go to a byte boundary */ -#define BYTEBITS() \ - do { \ - hold >>= bits & 7; \ - bits -= bits & 7; \ - } while (0) - -/* Assure that some output space is available, by writing out the window - if it's full. If the write fails, return from inflateBack() with a - Z_BUF_ERROR. */ -#define ROOM() \ - do { \ - if (left == 0) { \ - put = state->window; \ - left = state->wsize; \ - state->whave = left; \ - if (out(out_desc, put, left)) { \ - ret = Z_BUF_ERROR; \ - goto inf_leave; \ - } \ - } \ - } while (0) - -/* - strm provides the memory allocation functions and window buffer on input, - and provides information on the unused input on return. For Z_DATA_ERROR - returns, strm will also provide an error message. - - in() and out() are the call-back input and output functions. When - inflateBack() needs more input, it calls in(). When inflateBack() has - filled the window with output, or when it completes with data in the - window, it calls out() to write out the data. The application must not - change the provided input until in() is called again or inflateBack() - returns. The application must not change the window/output buffer until - inflateBack() returns. - - in() and out() are called with a descriptor parameter provided in the - inflateBack() call. This parameter can be a structure that provides the - information required to do the read or write, as well as accumulated - information on the input and output such as totals and check values. - - in() should return zero on failure. out() should return non-zero on - failure. If either in() or out() fails, than inflateBack() returns a - Z_BUF_ERROR. strm->next_in can be checked for Z_NULL to see whether it - was in() or out() that caused in the error. Otherwise, inflateBack() - returns Z_STREAM_END on success, Z_DATA_ERROR for an deflate format - error, or Z_MEM_ERROR if it could not allocate memory for the state. - inflateBack() can also return Z_STREAM_ERROR if the input parameters - are not correct, i.e. strm is Z_NULL or the state was not initialized. - */ -int ZEXPORT inflateBack(strm, in, in_desc, out, out_desc) -z_streamp strm; -in_func in; -void FAR *in_desc; -out_func out; -void FAR *out_desc; -{ - struct inflate_state FAR *state; - z_const unsigned char FAR *next; /* next input */ - unsigned char FAR *put; /* next output */ - unsigned have, left; /* available input and output */ - unsigned long hold; /* bit buffer */ - unsigned bits; /* bits in bit buffer */ - unsigned copy; /* number of stored or match bytes to copy */ - unsigned char FAR *from; /* where to copy match bytes from */ - code here; /* current decoding table entry */ - code last; /* parent table entry */ - unsigned len; /* length to copy for repeats, bits to drop */ - int ret; /* return code */ - static const unsigned short order[19] = /* permutation of code lengths */ - {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; - - /* Check that the strm exists and that the state was initialized */ - if (strm == Z_NULL || strm->state == Z_NULL) - return Z_STREAM_ERROR; - state = (struct inflate_state FAR *)strm->state; - - /* Reset the state */ - strm->msg = Z_NULL; - state->mode = TYPE; - state->last = 0; - state->whave = 0; - next = strm->next_in; - have = next != Z_NULL ? strm->avail_in : 0; - hold = 0; - bits = 0; - put = state->window; - left = state->wsize; - - /* Inflate until end of block marked as last */ - for (;;) - switch (state->mode) { - case TYPE: - /* determine and dispatch block type */ - if (state->last) { - BYTEBITS(); - state->mode = DONE; - break; - } - NEEDBITS(3); - state->last = BITS(1); - DROPBITS(1); - switch (BITS(2)) { - case 0: /* stored block */ - Tracev((stderr, "inflate: stored block%s\n", - state->last ? " (last)" : "")); - state->mode = STORED; - break; - case 1: /* fixed block */ - fixedtables(state); - Tracev((stderr, "inflate: fixed codes block%s\n", - state->last ? " (last)" : "")); - state->mode = LEN; /* decode codes */ - break; - case 2: /* dynamic block */ - Tracev((stderr, "inflate: dynamic codes block%s\n", - state->last ? " (last)" : "")); - state->mode = TABLE; - break; - case 3: - strm->msg = "invalid block type"; - state->mode = BAD; - } - DROPBITS(2); - break; - - case STORED: - /* get and verify stored block length */ - BYTEBITS(); /* go to byte boundary */ - NEEDBITS(32); - if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { - strm->msg = "invalid stored block lengths"; - state->mode = BAD; - break; - } - state->length = (unsigned)hold & 0xffff; - Tracev((stderr, "inflate: stored length %u\n", - state->length)); - INITBITS(); - - /* copy stored block from input to output */ - while (state->length != 0) { - copy = state->length; - PULL(); - ROOM(); - if (copy > have) copy = have; - if (copy > left) copy = left; - zmemcpy(put, next, copy); - have -= copy; - next += copy; - left -= copy; - put += copy; - state->length -= copy; - } - Tracev((stderr, "inflate: stored end\n")); - state->mode = TYPE; - break; - - case TABLE: - /* get dynamic table entries descriptor */ - NEEDBITS(14); - state->nlen = BITS(5) + 257; - DROPBITS(5); - state->ndist = BITS(5) + 1; - DROPBITS(5); - state->ncode = BITS(4) + 4; - DROPBITS(4); -#ifndef PKZIP_BUG_WORKAROUND - if (state->nlen > 286 || state->ndist > 30) { - strm->msg = "too many length or distance symbols"; - state->mode = BAD; - break; - } -#endif - Tracev((stderr, "inflate: table sizes ok\n")); - - /* get code length code lengths (not a typo) */ - state->have = 0; - while (state->have < state->ncode) { - NEEDBITS(3); - state->lens[order[state->have++]] = (unsigned short)BITS(3); - DROPBITS(3); - } - while (state->have < 19) - state->lens[order[state->have++]] = 0; - state->next = state->codes; - state->lencode = (code const FAR *)(state->next); - state->lenbits = 7; - ret = inflate_table(CODES, state->lens, 19, &(state->next), - &(state->lenbits), state->work); - if (ret) { - strm->msg = "invalid code lengths set"; - state->mode = BAD; - break; - } - Tracev((stderr, "inflate: code lengths ok\n")); - - /* get length and distance code code lengths */ - state->have = 0; - while (state->have < state->nlen + state->ndist) { - for (;;) { - here = state->lencode[BITS(state->lenbits)]; - if ((unsigned)(here.bits) <= bits) break; - PULLBYTE(); - } - if (here.val < 16) { - DROPBITS(here.bits); - state->lens[state->have++] = here.val; - } - else { - if (here.val == 16) { - NEEDBITS(here.bits + 2); - DROPBITS(here.bits); - if (state->have == 0) { - strm->msg = "invalid bit length repeat"; - state->mode = BAD; - break; - } - len = (unsigned)(state->lens[state->have - 1]); - copy = 3 + BITS(2); - DROPBITS(2); - } - else if (here.val == 17) { - NEEDBITS(here.bits + 3); - DROPBITS(here.bits); - len = 0; - copy = 3 + BITS(3); - DROPBITS(3); - } - else { - NEEDBITS(here.bits + 7); - DROPBITS(here.bits); - len = 0; - copy = 11 + BITS(7); - DROPBITS(7); - } - if (state->have + copy > state->nlen + state->ndist) { - strm->msg = "invalid bit length repeat"; - state->mode = BAD; - break; - } - while (copy--) - state->lens[state->have++] = (unsigned short)len; - } - } - - /* handle error breaks in while */ - if (state->mode == BAD) break; - - /* check for end-of-block code (better have one) */ - if (state->lens[256] == 0) { - strm->msg = "invalid code -- missing end-of-block"; - state->mode = BAD; - break; - } - - /* build code tables -- note: do not change the lenbits or distbits - values here (9 and 6) without reading the comments in inftrees.h - concerning the ENOUGH constants, which depend on those values */ - state->next = state->codes; - state->lencode = (code const FAR *)(state->next); - state->lenbits = 9; - ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), - &(state->lenbits), state->work); - if (ret) { - strm->msg = "invalid literal/lengths set"; - state->mode = BAD; - break; - } - state->distcode = (code const FAR *)(state->next); - state->distbits = 6; - ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, - &(state->next), &(state->distbits), state->work); - if (ret) { - strm->msg = "invalid distances set"; - state->mode = BAD; - break; - } - Tracev((stderr, "inflate: codes ok\n")); - state->mode = LEN; - - case LEN: - /* use inflate_fast() if we have enough input and output */ - if (have >= 6 && left >= 258) { - RESTORE(); - if (state->whave < state->wsize) - state->whave = state->wsize - left; - inflate_fast(strm, state->wsize); - LOAD(); - break; - } - - /* get a literal, length, or end-of-block code */ - for (;;) { - here = state->lencode[BITS(state->lenbits)]; - if ((unsigned)(here.bits) <= bits) break; - PULLBYTE(); - } - if (here.op && (here.op & 0xf0) == 0) { - last = here; - for (;;) { - here = state->lencode[last.val + - (BITS(last.bits + last.op) >> last.bits)]; - if ((unsigned)(last.bits + here.bits) <= bits) break; - PULLBYTE(); - } - DROPBITS(last.bits); - } - DROPBITS(here.bits); - state->length = (unsigned)here.val; - - /* process literal */ - if (here.op == 0) { - Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? - "inflate: literal '%c'\n" : - "inflate: literal 0x%02x\n", here.val)); - ROOM(); - *put++ = (unsigned char)(state->length); - left--; - state->mode = LEN; - break; - } - - /* process end of block */ - if (here.op & 32) { - Tracevv((stderr, "inflate: end of block\n")); - state->mode = TYPE; - break; - } - - /* invalid code */ - if (here.op & 64) { - strm->msg = "invalid literal/length code"; - state->mode = BAD; - break; - } - - /* length code -- get extra bits, if any */ - state->extra = (unsigned)(here.op) & 15; - if (state->extra != 0) { - NEEDBITS(state->extra); - state->length += BITS(state->extra); - DROPBITS(state->extra); - } - Tracevv((stderr, "inflate: length %u\n", state->length)); - - /* get distance code */ - for (;;) { - here = state->distcode[BITS(state->distbits)]; - if ((unsigned)(here.bits) <= bits) break; - PULLBYTE(); - } - if ((here.op & 0xf0) == 0) { - last = here; - for (;;) { - here = state->distcode[last.val + - (BITS(last.bits + last.op) >> last.bits)]; - if ((unsigned)(last.bits + here.bits) <= bits) break; - PULLBYTE(); - } - DROPBITS(last.bits); - } - DROPBITS(here.bits); - if (here.op & 64) { - strm->msg = "invalid distance code"; - state->mode = BAD; - break; - } - state->offset = (unsigned)here.val; - - /* get distance extra bits, if any */ - state->extra = (unsigned)(here.op) & 15; - if (state->extra != 0) { - NEEDBITS(state->extra); - state->offset += BITS(state->extra); - DROPBITS(state->extra); - } - if (state->offset > state->wsize - (state->whave < state->wsize ? - left : 0)) { - strm->msg = "invalid distance too far back"; - state->mode = BAD; - break; - } - Tracevv((stderr, "inflate: distance %u\n", state->offset)); - - /* copy match from window to output */ - do { - ROOM(); - copy = state->wsize - state->offset; - if (copy < left) { - from = put + copy; - copy = left - copy; - } - else { - from = put - state->offset; - copy = left; - } - if (copy > state->length) copy = state->length; - state->length -= copy; - left -= copy; - do { - *put++ = *from++; - } while (--copy); - } while (state->length != 0); - break; - - case DONE: - /* inflate stream terminated properly -- write leftover output */ - ret = Z_STREAM_END; - if (left < state->wsize) { - if (out(out_desc, state->window, state->wsize - left)) - ret = Z_BUF_ERROR; - } - goto inf_leave; - - case BAD: - ret = Z_DATA_ERROR; - goto inf_leave; - - default: /* can't happen, but makes compilers happy */ - ret = Z_STREAM_ERROR; - goto inf_leave; - } - - /* Return unused input */ - inf_leave: - strm->next_in = next; - strm->avail_in = have; - return ret; -} - -int ZEXPORT inflateBackEnd(strm) -z_streamp strm; -{ - if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) - return Z_STREAM_ERROR; - ZFREE(strm, strm->state); - strm->state = Z_NULL; - Tracev((stderr, "inflate: end\n")); - return Z_OK; -} diff --git a/lib/libvgz/uncompr.c b/lib/libvgz/uncompr.c deleted file mode 100644 index a1212de..0000000 --- a/lib/libvgz/uncompr.c +++ /dev/null @@ -1,93 +0,0 @@ -/* uncompr.c -- decompress a memory buffer - * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* @(#) $Id$ */ - -#define ZLIB_INTERNAL -#include "vgz.h" - -/* =========================================================================== - Decompresses the source buffer into the destination buffer. *sourceLen is - the byte length of the source buffer. Upon entry, *destLen is the total size - of the destination buffer, which must be large enough to hold the entire - uncompressed data. (The size of the uncompressed data must have been saved - previously by the compressor and transmitted to the decompressor by some - mechanism outside the scope of this compression library.) Upon exit, - *destLen is the size of the decompressed data and *sourceLen is the number - of source bytes consumed. Upon return, source + *sourceLen points to the - first unused input byte. - - uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough - memory, Z_BUF_ERROR if there was not enough room in the output buffer, or - Z_DATA_ERROR if the input data was corrupted, including if the input data is - an incomplete zlib stream. -*/ -int ZEXPORT uncompress2 (dest, destLen, source, sourceLen) - Bytef *dest; - uLongf *destLen; - const Bytef *source; - uLong *sourceLen; -{ - z_stream stream; - int err; - const uInt max = (uInt)-1; - uLong len, left; - Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ - - len = *sourceLen; - if (*destLen) { - left = *destLen; - *destLen = 0; - } - else { - left = 1; - dest = buf; - } - - stream.next_in = (z_const Bytef *)source; - stream.avail_in = 0; - stream.zalloc = (alloc_func)0; - stream.zfree = (free_func)0; - stream.opaque = (voidpf)0; - - err = inflateInit(&stream); - if (err != Z_OK) return err; - - stream.next_out = dest; - stream.avail_out = 0; - - do { - if (stream.avail_out == 0) { - stream.avail_out = left > (uLong)max ? max : (uInt)left; - left -= stream.avail_out; - } - if (stream.avail_in == 0) { - stream.avail_in = len > (uLong)max ? max : (uInt)len; - len -= stream.avail_in; - } - err = inflate(&stream, Z_NO_FLUSH); - } while (err == Z_OK); - - *sourceLen -= len + stream.avail_in; - if (dest != buf) - *destLen = stream.total_out; - else if (stream.total_out && err == Z_BUF_ERROR) - left = 1; - - inflateEnd(&stream); - return err == Z_STREAM_END ? Z_OK : - err == Z_NEED_DICT ? Z_DATA_ERROR : - err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : - err; -} - -int ZEXPORT uncompress (dest, destLen, source, sourceLen) - Bytef *dest; - uLongf *destLen; - const Bytef *source; - uLong sourceLen; -{ - return uncompress2(dest, destLen, source, &sourceLen); -} From nils.goroll at uplex.de Mon Apr 9 12:24:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 9 Apr 2018 12:24:10 +0000 (UTC) Subject: [master] 3eba5f8 warning and advise about backend proxy connections Message-ID: <20180409122410.A2FAF96EB9@lists.varnish-cache.org> commit 3eba5f8ce18e3d8e06a7016f9099e689f5186bc7 Author: Nils Goroll Date: Mon Apr 9 14:23:06 2018 +0200 warning and advise about backend proxy connections Ref #2622 diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index d2eec3d..d2263dc 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -252,6 +252,21 @@ parameters. The following attributes are available: The PROXY protocol version Varnish should use when connecting to this backend. Allowed values are ``1`` and ``2``. + *Notice* this setting will lead to backend connections being used + for a single request only (subject to future improvements). Thus, + extra care should be taken to avoid running into failing backend + connections with EADDRNOTAVAIL due to no local ports being + available. Possible options are: + + * Use additional backend connections to extra IP addresses or TCP + ports + + * Increase the number of available ports (Linux sysctl + ``net.ipv4.ip_local_port_range``) + + * Reuse backend connection ports early (Linux sysctl + ``net.ipv4.tcp_tw_reuse``) + ``.max_connections`` Maximum number of open connections towards this backend. If Varnish reaches the maximum Varnish it will start failing From phk at FreeBSD.org Tue Apr 10 09:00:21 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 10 Apr 2018 09:00:21 +0000 (UTC) Subject: [master] abad294 Rewrite the proxy TLV wandering code, hoping coverity can make better sense of it now. Message-ID: <20180410090021.30AB8B6979@lists.varnish-cache.org> commit abad29495ef956876e4141898143d89d68951730 Author: Poul-Henning Kamp Date: Tue Apr 10 08:55:46 2018 +0000 Rewrite the proxy TLV wandering code, hoping coverity can make better sense of it now. diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 022224d..32ddd6f 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -42,6 +42,13 @@ #include "vsa.h" #include "vtcp.h" +struct vpx_tlv { + unsigned magic; +#define VPX_TLV_MAGIC 0xdeb9a4a5 + unsigned len; + char tlv[1]; +}; + /********************************************************************** * PROXY 1 protocol */ @@ -156,17 +163,6 @@ vpx_proto1(const struct worker *wrk, const struct req *req) * PROXY 2 protocol */ -struct pp2_tlv { - uint8_t type; - uint8_t length_hi; - uint8_t length_lo; -}__attribute__((packed)); - -struct pp2_tlv_ssl { - uint8_t client; - uint32_t verify; -}__attribute__((packed)); - static const char vpx2_sig[] = { '\r', '\n', '\r', '\n', '\0', '\r', '\n', 'Q', 'U', 'I', 'T', '\n', @@ -248,60 +244,85 @@ static uint32_t crc32c(const uint8_t *buf, int len) return (crc ^ 0xffffffff); } +struct vpx_tlv_iter { + uint8_t t; + void *p; + uint16_t l; + const char *e; + + unsigned char *_p; + uint16_t _l; +}; + +static void +vpx_tlv_iter0(struct vpx_tlv_iter *vpi, void *p, unsigned l) +{ + + AN(p); + assert(l < 65536); + memset(vpi, 0, sizeof *vpi); + vpi->_p = p; + vpi->_l = l; +} + +static int +vpx_tlv_itern(struct vpx_tlv_iter *vpi) +{ + if (vpi->_l == 0 || vpi->e != NULL) + return (0); + if (vpi->_l < 3) { + vpi->e = "Dribble bytes"; + return (0); + } + vpi->t = *vpi->_p; + vpi->l = vbe16dec(vpi->_p + 1); + if (vpi->l + 3 > vpi->_l) { + vpi->e = "Length Error"; + return (0); + } + vpi->p = vpi->_p + 3; + vpi->_p += 3 + vpi->l; + vpi->_l -= 3 + vpi->l; + return (1); +} + +#define VPX_TLV_FOREACH(ptr, len, itv) \ + for(vpx_tlv_iter0(itv, ptr, len); vpx_tlv_itern(itv);) + int -VPX_tlv(const struct req *req, int tlv, void **dst, int *len) +VPX_tlv(const struct req *req, int typ, void **dst, int *len) { - uintptr_t *p; - uint16_t *tlv_len_p; - uint16_t l; - char *d; - int ssltlv = 0; + struct vpx_tlv *tlv; + struct vpx_tlv_iter vpi[1], vpi2[1]; + uintptr_t *up; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); + AN(dst); + AN(len); + *dst = NULL; + *len = 0; - if (SES_Get_xport_priv(req->sp, &p) != 0) { - *dst = NULL; + if (SES_Get_proxy_tlv(req->sp, &up) != 0 || *up == 0) return (-1); - } - tlv_len_p = (void *)(*p); - l = *tlv_len_p; - d = (char *)(tlv_len_p + 1); + CAST_OBJ_NOTNULL(tlv, (void*)(*up), VPX_TLV_MAGIC); - if (tlv > PP2_TYPE_SSL && tlv <= PP2_SUBTYPE_SSL_MAX) { - ssltlv = tlv; - tlv = PP2_TYPE_SSL; - } - while (l > sizeof(struct pp2_tlv)) { - uint16_t v_len = vbe16dec(d + 1); - if (d[0] == tlv) { - if (ssltlv) { - char *sd; - int sl; - sd = d + sizeof(struct pp2_tlv) + - sizeof(struct pp2_tlv_ssl); - sl = l - sizeof(struct pp2_tlv) - - sizeof(struct pp2_tlv_ssl); - while (sl > sizeof(struct pp2_tlv)) { - uint16_t subv_len = vbe16dec(sd + 1); - if (sd[0] == ssltlv) { - *dst = sd + 3; - *len = subv_len; - return (0); - } - sd += (subv_len + 3); - sl -= (subv_len + 3); - } - } else { - *dst = d + 3; - *len = v_len; + VPX_TLV_FOREACH(tlv->tlv, tlv->len, vpi) { + if (vpi->t == typ) { + *dst = vpi->p; + *len = vpi->l; + return (0); + } + if (vpi->t != PP2_TYPE_SSL) + continue; + VPX_TLV_FOREACH((char*)vpi->p + 5, vpi->l - 5, vpi2) { + if (vpi2->t == typ) { + *dst = vpi2->p; + *len = vpi2->l; return (0); } } - d += (v_len + 3); - l -= (v_len + 3); } - *dst = NULL; return (-1); } @@ -310,7 +331,6 @@ vpx_proto2(const struct worker *wrk, struct req *req) { int l, hdr_len; uintptr_t *up; - uint16_t *tlv_len_p; uint16_t tlv_len; const uint8_t *p; char *d, *tlv_start; @@ -322,6 +342,8 @@ vpx_proto2(const struct worker *wrk, struct req *req) char pa[VTCP_PORTBUFSIZE]; char hb[VTCP_ADDRBUFSIZE]; char pb[VTCP_PORTBUFSIZE]; + struct vpx_tlv_iter vpi[1], vpi2[1]; + struct vpx_tlv *tlv; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -374,27 +396,6 @@ vpx_proto2(const struct worker *wrk, struct req *req) } l -= 12; d += 12; - break; - case 0x21: - /* IPv6|TCP */ - pfam = AF_INET6; - if (l < 36) { - VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY2: Ignoring short IPv6 addresses (%d)", l); - return (0); - } - l -= 36; - d += 36; - break; - default: - /* Ignore proxy header */ - VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY2: Ignoring unsupported protocol (0x%02x)", p[13]); - return (0); - } - - switch (pfam) { - case AF_INET: memset(&sin4, 0, sizeof sin4); sin4.sin_family = pfam; @@ -411,7 +412,16 @@ vpx_proto2(const struct worker *wrk, struct req *req) SES_Reserve_client_addr(req->sp, &sa); AN(VSA_Build(sa, &sin4, sizeof sin4)); break; - case AF_INET6: + case 0x21: + /* IPv6|TCP */ + pfam = AF_INET6; + if (l < 36) { + VSL(SLT_ProxyGarbage, req->sp->vxid, + "PROXY2: Ignoring short IPv6 addresses (%d)", l); + return (0); + } + l -= 36; + d += 36; memset(&sin6, 0, sizeof sin6); sin6.sin6_family = pfam; @@ -429,7 +439,10 @@ vpx_proto2(const struct worker *wrk, struct req *req) AN(VSA_Build(sa, &sin6, sizeof sin6)); break; default: - WRONG("Wrong pfam"); + /* Ignore proxy header */ + VSL(SLT_ProxyGarbage, req->sp->vxid, + "PROXY2: Ignoring unsupported protocol (0x%02x)", p[13]); + return (0); } AN(sa); @@ -442,62 +455,36 @@ vpx_proto2(const struct worker *wrk, struct req *req) tlv_start = d; tlv_len = l; - while (l > sizeof(struct pp2_tlv)) { - int el = vbe16dec(d + 1) + 3; - if (el > l) { - VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY2: Ignoring TLV"); - return (0); - } - switch (d[0]) { - case PP2_TYPE_CRC32C: - { - uint32_t n_crc32c = vbe32dec(d+3); - *(d+3) = 0; *(d+4) = 0; *(d+5) = 0; *(d+6) = 0; + VPX_TLV_FOREACH(d, l, vpi) { + if (vpi->t == PP2_TYPE_SSL) { + VPX_TLV_FOREACH((char*)vpi->p + 5, vpi->l - 5, vpi2) { + } + vpi->e = vpi2->e; + } else if (vpi->t == PP2_TYPE_CRC32C) { + uint32_t n_crc32c = vbe32dec(vpi->p); + vbe32enc(vpi->p, 0); if (crc32c(p, hdr_len) != n_crc32c) { VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: CRC error"); return (-1); } - break; - } - case PP2_TYPE_SSL: - { - const char *sd; - int sl; - sd = d + sizeof(struct pp2_tlv) + - sizeof(struct pp2_tlv_ssl); - sl = l - sizeof(struct pp2_tlv) - - sizeof(struct pp2_tlv_ssl); - while (sl > sizeof(struct pp2_tlv)) { - int esl = vbe16dec(sd + 1) + 3; - if (esl > sl) { - VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY2: Ignoring SSL TLV"); - return (0); - } - sd += esl; - sl -= esl; - } - break; } - } - d += el; - l -= el; } - if (l) { - VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY2: header length mismatch"); - return (0); + if (vpi->e != NULL) { + VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY2: TLV %s", vpi->e); + return (-1); } - if (tlv_len && WS_Reserve(req->sp->ws, 2 + tlv_len)) { - tlv_len_p = (void *)req->sp->ws->f; - *tlv_len_p = tlv_len; - memcpy(req->sp->ws->f + 2, tlv_start, tlv_len); - WS_Release(req->sp->ws, 2 + tlv_len); - SES_Reserve_xport_priv(req->sp, &up); - *up = (uintptr_t)tlv_len_p; + tlv = WS_Alloc(req->sp->ws, sizeof *tlv + tlv_len); + if (tlv == NULL) { + VSL(SLT_ProxyGarbage, req->sp->vxid, + "PROXY2: TLV overflows WS"); + return (-1); } + INIT_OBJ(tlv, VPX_TLV_MAGIC); + tlv->len = tlv_len; + memcpy(tlv->tlv, tlv_start, tlv_len); + SES_Reserve_proxy_tlv(req->sp, &up); + *up = (uintptr_t)tlv; return (0); } diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc index d40e353..96154e8 100644 --- a/bin/varnishtest/tests/o00005.vtc +++ b/bin/varnishtest/tests/o00005.vtc @@ -31,16 +31,31 @@ varnish v1 -proto "PROXY" -vcl+backend { } } -start +logexpect l1 -v v1 -g raw { + expect * 1000 Begin "sess 0 PROXY" + expect * 1000 Proxy "2 217.70.181.33 60822 95.142.168.34 443" + expect * 1000 Link "req 1001 rxreq" +} -start + client c1 { # PROXY2 with CRC32C TLV - sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a 21 11 00 65" - sendhex "d9 46 b5 21 5f 8e a8 22 ed 96 01 bb 03 00 04 95" - sendhex "03 ee 75 01 00 02 68 32 02 00 0a 68 6f 63 64 65" - sendhex "74 2e 6e 65 74 20 00 3d 01 00 00 00 00 21 00 07" - sendhex "54 4c 53 76 31 2e 33 25 00 05 45 43 32 35 36 24" - sendhex "00 0a 52 53 41 2d 53 48 41 32 35 36 23 00 16 41" - sendhex "45 41 44 2d 41 45 53 31 32 38 2d 47 43 4d 2d 53" - sendhex "48 41 32 35 36" + sendhex { + 0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a + 21 11 00 65 + d9 46 b5 21 + 5f 8e a8 22 + ed 96 + 01 bb + 03 00 04 95 03 ee 75 + 01 00 02 68 32 + 02 00 0a 68 6f 63 64 65 74 2e 6e 65 74 + 20 00 3d + 01 00 00 00 00 + 21 00 07 54 4c 53 76 31 2e 33 + 25 00 05 45 43 32 35 36 + 24 00 0a 52 53 41 2d 53 48 41 32 35 36 + 23 00 16 41 45 41 44 2d 41 45 53 31 32 38 2d 47 43 4d 2d 53 48 41 32 35 36 + } txreq rxresp expect resp.status == 200 @@ -57,16 +72,136 @@ client c1 { expect resp.http.cn == "" } -run +varnish v1 -vsl_catchup + +logexpect l1 -wait + +logexpect l1 -v v1 -g raw { + expect * 1003 Begin "sess 0 PROXY" + expect * 1003 ProxyGarbage "PROXY2: CRC error" +} -start + +client c1 { + # PROXY2 with CRC32C TLV and bad checksum + sendhex { + 0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a + 21 11 00 65 + d9 46 b5 21 + 5f 8e a8 22 + ed 96 + 01 bb + 03 00 04 95 03 ee 74 + 01 00 02 68 32 + 02 00 0a 68 6f 63 64 65 74 2e 6e 65 74 + 20 00 3d + 01 00 00 00 00 + 21 00 07 54 4c 53 76 31 2e 33 + 25 00 05 45 43 32 35 36 + 24 00 0a 52 53 41 2d 53 48 41 32 35 36 + 23 00 16 41 45 41 44 2d 41 45 53 31 32 38 2d 47 43 4d 2d 53 48 41 32 35 36 + } + txreq + expect_close +} -run + +varnish v1 -vsl_catchup + +logexpect l1 -wait + +logexpect l1 -v v1 -g raw { + expect * 1004 Begin "sess 0 PROXY" + expect * 1004 ProxyGarbage "PROXY2: TLV Dribble bytes" +} -start + +client c1 { + # PROXY2 with CRC32C TLV and bad checksum + sendhex { + 0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a + 21 11 00 67 + d9 46 b5 21 + 5f 8e a8 22 + ed 96 + 01 bb + ff 00 04 95 03 ee 74 + 01 00 02 68 32 + 02 00 0a 68 6f 63 64 65 74 2e 6e 65 74 + 20 00 3d + 01 00 00 00 00 + 21 00 07 54 4c 53 76 31 2e 33 + 25 00 05 45 43 32 35 36 + 24 00 0a 52 53 41 2d 53 48 41 32 35 36 + 23 00 16 41 45 41 44 2d 41 45 53 31 32 38 2d 47 43 4d 2d 53 48 41 32 35 36 + ff ff + } + txreq + expect_close +} -run + +varnish v1 -vsl_catchup + +logexpect l1 -wait + +logexpect l1 -v v1 -g raw { + expect * 1005 Begin "sess 0 PROXY" + expect * 1005 ProxyGarbage "PROXY2: TLV Length Error" +} -start + +client c1 { + # PROXY2 with CRC32C TLV and bad checksum + sendhex { + 0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a + 21 11 00 60 + d9 46 b5 21 + 5f 8e a8 22 + ed 96 + 01 bb + ff 00 04 95 03 ee 74 + 01 00 02 68 32 + 02 00 0a 68 6f 63 64 65 74 2e 6e 65 74 + 20 00 3d + 01 00 00 00 00 + 21 00 07 54 4c 53 76 31 2e 33 + 25 00 05 45 43 32 35 36 + 24 00 0a 52 53 41 2d 53 48 41 32 35 36 + 23 00 16 41 45 41 44 2d 41 45 53 31 32 38 2d 47 43 4d 2d 53 48 41 32 35 36 + } + txreq + expect_close +} -run + +varnish v1 -vsl_catchup + +logexpect l1 -wait + +logexpect l1 -v v1 -g raw { + expect * 1006 Begin "sess 0 PROXY" + expect * 1006 ProxyGarbage "PROXY2: TLV Length Error" +} -start + client c1 { # PROXY2 with CRC32C TLV and bad checksum - sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a 21 11 00 65" - sendhex "d9 46 b5 21 5f 8e a8 22 ed 96 01 bb 03 00 04 95" - sendhex "03 ee 75 01 00 02 68 32 02 00 0a 68 6f 63 64 65" - sendhex "74 2e 6e 65 74 20 00 3d 01 00 00 00 00 21 00 07" - sendhex "54 4c 53 76 31 2e 33 25 00 05 45 43 32 35 36 24" - sendhex "00 0a 52 53 41 2d 53 48 41 32 35 36 23 00 16 41" - sendhex "45 41 44 2d 41 45 53 31 32 38 2d 47 43 4d 2d 53" - sendhex "48 41 32 35 35" + sendhex { + 0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a + 21 11 00 65 + d9 46 b5 21 + 5f 8e a8 22 + ed 96 + 01 bb + ff 00 04 95 03 ee 74 + 01 00 02 68 32 + 02 00 0a 68 6f 63 64 65 74 2e 6e 65 74 + 20 00 3c + 01 00 00 00 00 + 21 00 07 54 4c 53 76 31 2e 33 + 25 00 05 45 43 32 35 36 + 24 00 0a 52 53 41 2d 53 48 41 32 35 36 + 23 00 16 41 45 41 44 2d 41 45 53 31 32 38 2d 47 43 4d 2d 53 48 41 32 35 36 + } txreq expect_close } -run + +varnish v1 -vsl_catchup + +logexpect l1 -wait + diff --git a/include/tbl/sess_attr.h b/include/tbl/sess_attr.h index 6b83fb3..5276a18 100644 --- a/include/tbl/sess_attr.h +++ b/include/tbl/sess_attr.h @@ -39,7 +39,7 @@ SESS_ATTR(CLIENT_ADDR, client_addr, struct suckaddr, vsa_suckaddr_len) SESS_ATTR(SERVER_ADDR, server_addr, struct suckaddr, vsa_suckaddr_len) SESS_ATTR(CLIENT_IP, client_ip, char, -1) SESS_ATTR(CLIENT_PORT, client_port, char, -1) -SESS_ATTR(XPORT_PRIV, xport_priv, uintptr_t, sizeof(uintptr_t)) +SESS_ATTR(PROXY_TLV, proxy_tlv, uintptr_t, sizeof(uintptr_t)) SESS_ATTR(PROTO_PRIV, proto_priv, uintptr_t, sizeof(uintptr_t)) #undef SESS_ATTR From phk at FreeBSD.org Tue Apr 10 09:00:21 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 10 Apr 2018 09:00:21 +0000 (UTC) Subject: [master] c3f9564 Unused #include Message-ID: <20180410090021.4448FB697C@lists.varnish-cache.org> commit c3f956422b89c4f963c8f184e931f4ef5c22c0a9 Author: Poul-Henning Kamp Date: Tue Apr 10 08:58:29 2018 +0000 Unused #include diff --git a/lib/libvmod_proxy/vmod_proxy.c b/lib/libvmod_proxy/vmod_proxy.c index 8c6973c..fa7b961 100644 --- a/lib/libvmod_proxy/vmod_proxy.c +++ b/lib/libvmod_proxy/vmod_proxy.c @@ -35,7 +35,6 @@ #include "cache/cache.h" #include "vend.h" -#include "vcl.h" #include "proxy/cache_proxy.h" From phk at FreeBSD.org Tue Apr 10 11:07:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 10 Apr 2018 11:07:11 +0000 (UTC) Subject: [master] 774d8df Attempt to stabilize test g00005 by making the test do what it purports to say on the tin, and by removing the quantum-test it trips over. Message-ID: <20180410110711.A958AB8FA8@lists.varnish-cache.org> commit 774d8df5a8bc43e05a2ad336b50f72cbf2923b25 Author: Poul-Henning Kamp Date: Tue Apr 10 11:05:30 2018 +0000 Attempt to stabilize test g00005 by making the test do what it purports to say on the tin, and by removing the quantum-test it trips over. diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index aee17d3..7cb317c 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -315,21 +315,18 @@ vdp_gunzip(struct req *req, enum vdp_action act, void **priv, http_Unset(req->resp, H_Content_Encoding); req->resp_len = -1; - if (req->objcore->boc != NULL) - return (0); /* No idea about length (yet) */ p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl); - if (p == NULL || dl != 32) - return (0); /* No OA_GZIPBITS yet */ - - u = vbe64dec(p + 24); - /* - * If the size is non-zero AND we are the top - * VDP (ie: no ESI), we know what size the output will be. - */ - if (u != 0 && VTAILQ_FIRST(&req->vdc->vdp)->vdp == &VDP_gunzip) - req->resp_len = u; - + if (p != NULL || dl == 32) { + u = vbe64dec(p + 24); + /* + * If the size is non-zero AND we are the top VDP + * (ie: no ESI), we know what size the output will be. + */ + if (u != 0 && + VTAILQ_FIRST(&req->vdc->vdp)->vdp == &VDP_gunzip) + req->resp_len = u; + } return (0); } diff --git a/bin/varnishtest/tests/g00005.vtc b/bin/varnishtest/tests/g00005.vtc index 9eacb10..24b3b60 100644 --- a/bin/varnishtest/tests/g00005.vtc +++ b/bin/varnishtest/tests/g00005.vtc @@ -3,7 +3,17 @@ varnishtest "test gunzip for client + Range" server s1 -repeat 3 { rxreq expect req.http.accept-encoding == "gzip" - txresp -gzipbody FOOBARBARF + txresp -nolen -hdr "Transfer-encoding: chunked" \ + -hdr "Content-encoding: gzip" + delay 1 + # Compressed "FOOBARBARF" + sendhex { + 31 43 0d 0a + 1f 8b 08 00 75 96 cc 5a 02 03 73 f3 f7 77 72 0c + 02 22 37 00 06 8e 8c 83 0a 00 00 00 + 0d 0a + 30 0d 0a 0d 0a + } } -start varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { @@ -21,33 +31,49 @@ client c1 { expect resp.status == 200 expect resp.bodylen == "10" expect resp.http.content-encoding == +} -run + +varnish v1 -vsl_catchup +client c1 { txreq -hdr "Accept-encoding: gzip;q=0.1" rxresp expect resp.http.content-encoding == "gzip" gunzip expect resp.bodylen == "10" +} -run - # This delay attempts to ensure that the busyobj - # is completed before we attempt the range request - delay 1 +varnish v1 -vsl_catchup + +# This delay attempts to ensure that the busyobj +# is completed before we attempt the range request +delay 2 +client c1 { txreq -hdr "Range: bytes=3-5" rxresp expect resp.status == 206 expect resp.http.content-encoding == "" expect resp.bodylen == "3" expect resp.body == "BAR" +} -run +varnish v1 -vsl_catchup + +client c1 { txreq -url "/nostreamcachemiss" -hdr "Range: bytes=3-5" rxresp expect resp.status == 206 expect resp.http.content-encoding == "" expect resp.bodylen == "3" expect resp.body == "BAR" +} -run + +varnish v1 -vsl_catchup - # simple cache miss no gunzip - txreq -url "/2" -hdr "Range: bytes=3-5" -hdr "Accept-Encoding: gzip" +client c1 { + # simple cache miss, no stream, no gunzip + txreq -url "/nostream2" -hdr "Range: bytes=3-5" -hdr "Accept-Encoding: gzip" rxresp expect resp.status == 206 expect resp.http.content-encoding == "gzip" From phk at FreeBSD.org Tue Apr 10 12:55:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 10 Apr 2018 12:55:11 +0000 (UTC) Subject: [master] 3aa35e6 Duh! && not ||. Message-ID: <20180410125511.4C345630A7@lists.varnish-cache.org> commit 3aa35e60dddf959e1a606f6a2e5f5b372fac7ed9 Author: Poul-Henning Kamp Date: Tue Apr 10 12:54:38 2018 +0000 Duh! && not ||. Spotted by: fgs diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 7cb317c..72a86dd 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -317,7 +317,7 @@ vdp_gunzip(struct req *req, enum vdp_action act, void **priv, req->resp_len = -1; p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl); - if (p != NULL || dl == 32) { + if (p != NULL && dl == 32) { u = vbe64dec(p + 24); /* * If the size is non-zero AND we are the top VDP From nils.goroll at uplex.de Tue Apr 10 14:18:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Apr 2018 14:18:10 +0000 (UTC) Subject: [master] 17f4c18 Add a basic stack overflow detection heuristic Message-ID: <20180410141810.AE75764A1F@lists.varnish-cache.org> commit 17f4c18ee8d1f25d77c41168da7274e7331815b7 Author: Nils Goroll Date: Tue Apr 10 16:12:24 2018 +0200 Add a basic stack overflow detection heuristic Yes, I know, this is far from perfect, but it does not require any changes outside the signal handler and will hopefully simplify error report handling significantly. Ref #2643 diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index ad7e312..fe8cdb3 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -235,17 +235,36 @@ child_signal_handler(int s, siginfo_t *si, void *c) { char buf[1024]; struct sigaction sa; + struct req *req; + const char *a, *p, *info = NULL; (void)c; - /* Don't come back */ memset(&sa, 0, sizeof sa); sa.sa_handler = SIG_DFL; (void)sigaction(SIGSEGV, &sa, NULL); (void)sigaction(SIGABRT, &sa, NULL); - bprintf(buf, "Signal %d (%s) received at %p si_code %d", - s, strsignal(s), si->si_addr, si->si_code); + while (s == SIGSEGV) { + req = THR_GetRequest(); + if (req == NULL || req->wrk == NULL) + break; + a = TRUST_ME(si->si_addr); + p = TRUST_ME(req->wrk); + p += sizeof *req->wrk; + // rough safe estimate - top of stack + if (a > p + cache_param->wthread_stacksize) + break; + if (a < p - 2 * cache_param->wthread_stacksize) + break; + info = "\nTHIS PROBABLY IS A STACK OVERFLOW - " + "check thread_pool_stack parameter"; + break; + } + bprintf(buf, "Signal %d (%s) received at %p si_code %d%s", + s, strsignal(s), si->si_addr, si->si_code, + info ? info : ""); + VAS_Fail(__func__, __FILE__, __LINE__, diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 2a9c75d..2c0ad15 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -96,6 +96,7 @@ WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv) static void WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) { + // child_signal_handler stack overflow check uses struct worker addr struct worker *w, ww; struct VSC_main ds; unsigned char ws[thread_workspace]; diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc index 4b05a8f..c70158e 100644 --- a/bin/varnishtest/tests/c00057.vtc +++ b/bin/varnishtest/tests/c00057.vtc @@ -39,6 +39,8 @@ client c1 { expect_close } -run +varnish v1 -cliexpect "STACK OVERFLOW" "panic.show" + varnish v1 -cliok "panic.clear" # Also check without the handler @@ -51,3 +53,31 @@ client c1 { } -run varnish v1 -expectexit 0x20 + +#################### + +varnish v2 \ + -arg "-p feature=+no_coredump" \ + -arg "-p vcc_allow_inline_c=true" \ + -vcl+backend { + + C{ + #include + }C + + sub vcl_recv { C{ + int *i = (void *)VRT_GetHdr; + *i = 42; + }C } +} -start + +client c2 -connect ${v2_sock} { + txreq + expect_close +} -run + +varnish v2 -cliexpect "Segmentation fault" "panic.show" + +varnish v2 -cliok "panic.clear" + +varnish v2 -expectexit 0x20 From nils.goroll at uplex.de Tue Apr 10 14:39:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Apr 2018 14:39:09 +0000 (UTC) Subject: [master] ca1c6d3 Also expect special Solar-ish uppercause Faults Message-ID: <20180410143909.D8BE0650AD@lists.varnish-cache.org> commit ca1c6d39b9ed29587d262e56127699d1ecd9dd84 Author: Nils Goroll Date: Tue Apr 10 16:37:59 2018 +0200 Also expect special Solar-ish uppercause Faults Ref #2643 diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc index c70158e..f33a8bd 100644 --- a/bin/varnishtest/tests/c00057.vtc +++ b/bin/varnishtest/tests/c00057.vtc @@ -76,7 +76,7 @@ client c2 -connect ${v2_sock} { expect_close } -run -varnish v2 -cliexpect "Segmentation fault" "panic.show" +varnish v2 -cliexpect "Segmentation [fF]ault" "panic.show" varnish v2 -cliok "panic.clear" From nils.goroll at uplex.de Tue Apr 10 15:13:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Apr 2018 15:13:10 +0000 (UTC) Subject: [master] 7f805f1 Use an anonymous struct to pass the argstruct to vmod functions Message-ID: <20180410151310.55E3065B26@lists.varnish-cache.org> commit 7f805f17e3cab6757bbd7ce370a19c8d9addd38f Author: Nils Goroll Date: Tue Apr 10 17:11:42 2018 +0200 Use an anonymous struct to pass the argstruct to vmod functions ... to save stack space Fixes #2643 diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 032a6e9..08977af 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -467,7 +467,6 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, const struct vjsn_val *vv, *vvp; const char *sa; char ssa[64]; - char ssa2[64]; int n; CAST_OBJ_NOTNULL(vv, priv, VJSN_VAL_MAGIC); @@ -481,10 +480,6 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, sa = vv->value; if (*sa == '\0') { sa = NULL; - } else { - bprintf(ssa, "args_%u", tl->unique++); - VSB_printf(tl->curproc->prologue, " %s %s;\n", sa, ssa); - sa = ssa; } vv = VTAILQ_NEXT(vv, list); SkipToken(tl, '('); @@ -572,27 +567,28 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, } if (sa != NULL) - e1 = vcc_mk_expr(rfmt, "%s(ctx%s,\v+(\n", cfunc, extra); + e1 = vcc_mk_expr(rfmt, "%s(ctx%s,\v+ &(%s){\n", + cfunc, extra, sa); else e1 = vcc_mk_expr(rfmt, "%s(ctx%s\v+", cfunc, extra); n = 0; VTAILQ_FOREACH_SAFE(fa, &head, list, fa2) { n++; - if (fa->optional) - VSB_printf(tl->curproc->prologue, - " %s.valid_%s = %d;\n", sa, fa->name, fa->avail); + if (fa->optional) { + bprintf(ssa, "\v1.valid_%s = %d,\n", + fa->name, fa->avail); + e1 = vcc_expr_edit(tl, e1->fmt, ssa, e1, NULL); + } if (fa->result == NULL && fa->type == ENUM && fa->val != NULL) vcc_do_enum(tl, fa, strlen(fa->val), fa->val); if (fa->result == NULL && fa->val != NULL) fa->result = vcc_mk_expr(fa->type, "%s", fa->val); if (fa->result != NULL && sa != NULL) { if (fa->name && *fa->name != '\0') - bprintf(ssa2, "\v1%s.%s = \v2,\n", - sa, fa->name); + bprintf(ssa, "\v1.%s = \v2,\n", fa->name); else - bprintf(ssa2, "\v1%s.arg%d = \v2,\n", - sa, n); - e1 = vcc_expr_edit(tl, e1->fmt, ssa2, e1, fa->result); + bprintf(ssa, "\v1.arg%d = \v2,\n", n); + e1 = vcc_expr_edit(tl, e1->fmt, ssa, e1, fa->result); } else if (fa->result != NULL) { e1 = vcc_expr_edit(tl, e1->fmt, "\v1,\n\v2", e1, fa->result); @@ -604,8 +600,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, free(fa); } if (sa != NULL) { - bprintf(ssa2, "\v1&%s\v-\n))", sa); - *e = vcc_expr_edit(tl, e1->fmt, ssa2, e1, NULL); + *e = vcc_expr_edit(tl, e1->fmt, "\v1\n})\v-", e1, NULL); } else { *e = vcc_expr_edit(tl, e1->fmt, "\v1\n)\v-", e1, NULL); } From phk at FreeBSD.org Wed Apr 11 07:20:15 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 11 Apr 2018 07:20:15 +0000 (UTC) Subject: [master] b5646ee Only check errno if write(2) failed. Message-ID: <20180411072015.E8D0DAD440@lists.varnish-cache.org> commit b5646ee4172f1e9b94fd60c9dafb114d2d6a5dad Author: Poul-Henning Kamp Date: Wed Apr 11 07:18:36 2018 +0000 Only check errno if write(2) failed. Hopefully this... Fixes #2605 diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index edb9907..c88fffd 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -271,7 +271,8 @@ http1_minimal_response(struct req *req, uint16_t status) if (wl > 0) req->acct.resp_hdrbytes += wl; if (wl != l) { - VTCP_Assert(1); + if (wl < 0) + VTCP_Assert(1); if (!req->doclose) req->doclose = SC_REM_CLOSE; return (-1); From phk at FreeBSD.org Wed Apr 11 08:29:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 11 Apr 2018 08:29:09 +0000 (UTC) Subject: [master] acd9cab No idea how I overlooked that wl was unsigned. Message-ID: <20180411082909.607A5AE930@lists.varnish-cache.org> commit acd9cab19f21a70504f13415687b5696f8428eaa Author: Poul-Henning Kamp Date: Wed Apr 11 08:27:43 2018 +0000 No idea how I overlooked that wl was unsigned. Doesn't change the logic however. diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index c88fffd..6506909 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -247,7 +247,7 @@ http1_reembark(struct worker *wrk, struct req *req) static int v_matchproto_(vtr_minimal_response_f) http1_minimal_response(struct req *req, uint16_t status) { - size_t wl, l; + ssize_t wl, l; char buf[80]; const char *reason; From hermunn at varnish-software.com Wed Apr 11 12:04:11 2018 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 11 Apr 2018 12:04:11 +0000 (UTC) Subject: [4.1] 50bfa24 Do not possibly underflow rlen Message-ID: <20180411120411.EBD81B338F@lists.varnish-cache.org> commit 50bfa24ea117ec232d541f1a4eec6b46081aeaa2 Author: Nils Goroll Date: Fri Oct 13 00:45:24 2017 +0200 Do not possibly underflow rlen for i < 0, rlen could underflow. We are safe because of the check for i < 0 further down, so this change is just a minor cleanup. Fixes #2444 diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index c688cd0..085d905 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -264,7 +264,7 @@ vbp_poke(struct vbp_target *vt) pfd->fd = s; rlen = 0; - do { + while (1) { pfd->events = POLLIN; pfd->revents = 0; tmo = (int)round((t_end - t_now) * 1e3); @@ -281,8 +281,10 @@ vbp_poke(struct vbp_target *vt) sizeof vt->resp_buf - rlen); else i = read(s, buf, sizeof buf); + if (i <= 0) + break; rlen += i; - } while (i > 0); + } VTCP_close(&s); From nils.goroll at uplex.de Wed Apr 11 13:40:14 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Apr 2018 13:40:14 +0000 (UTC) Subject: [master] 71c29e8 handle workspace overflow in V1F_Setup_Fetch() Message-ID: <20180411134014.327C3B5334@lists.varnish-cache.org> commit 71c29e869634a9b2931657e353365de26f2c988d Author: Nils Goroll Date: Wed Apr 11 12:08:09 2018 +0200 handle workspace overflow in V1F_Setup_Fetch() Ref #2645 but cannot be the cause because workspace is just not overflowed diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index 1a77021..27bf521 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -216,7 +216,11 @@ V1F_FetchRespHdr(struct busyobj *bo) assert(bo->vfc->resp == bo->beresp); if (bo->htc->body_status != BS_NONE && bo->htc->body_status != BS_ERROR) - (void)V1F_Setup_Fetch(bo->vfc, bo->htc); + if (V1F_Setup_Fetch(bo->vfc, bo->htc)) { + VSLb(bo->vsl, SLT_FetchError, "overflow"); + htc->doclose = SC_RX_OVERFLOW; + return (-1); + } return (0); } From phk at FreeBSD.org Wed Apr 11 19:35:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 11 Apr 2018 19:35:10 +0000 (UTC) Subject: [master] 4c3ef2d Report cursor position in screen-dump, and add "-expect-cursor" Message-ID: <20180411193510.5467361D33@lists.varnish-cache.org> commit 4c3ef2df74a14bc5c8592d9ac942597c2a41432f Author: Poul-Henning Kamp Date: Wed Apr 11 19:34:02 2018 +0000 Report cursor position in screen-dump, and add "-expect-cursor" diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index affa747..52736fd 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -55,10 +55,14 @@ process p4 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H" process p4 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n" process p4 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H" process p4 -write "111111112222222333333\x0d\x0a111111112" -process p4 -write "222222333333\x0d\x0a111111112222222333" -process p4 -write "333\x0d\x0a111111112222222333333\x0d\x0a\x1b[12" +process p4 -write "222222333333\x0d\x0a111111112222222333333 UTF8: " +process p4 -writehex {c2 a2 20 e2 82 ac 20 f0 9f 90 b0} +process p4 -write "\x0d\x0a111111112222222333333\x0d\x0a\x1b[12" process p4 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b" process p4 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a" -process p4 -need-bytes 252 -expect-text 3 1 "line3 <" -process p4 -screen_dump -stop +process p4 -need-bytes 270 -expect-text 3 1 "line3 <" +process p4 -expect-cursor 4 1 +process p4 -expect-cursor 4 0 +process p4 -expect-cursor 0 1 +process p4 -screen-dump -stop diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index be76323..a3f920d 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -205,9 +205,13 @@ static void term_screen_dump(const struct process *pp) { int i; + const teken_pos_t *pos; for (i = 0; i < pp->nlin; i++) vtc_dump(pp->vl, 3, "screen", pp->vram[i], pp->ncol); + pos = teken_get_cursor(pp->tek); + vtc_log(pp->vl, 3, "Cursor at line %d column %d", + pos->tp_row + 1, pos->tp_col + 1); } static void @@ -312,6 +316,23 @@ term_expect_text(struct process *pp, vtc_log(pp->vl, 4, "found expected text at %d,%d: '%s'", y, x, pat); } +static void +term_expect_cursor(struct process *pp, const char *lin, const char *col) +{ + int x, y; + const teken_pos_t *pos; + + pos = teken_get_cursor(pp->tek); + y = strtoul(lin, NULL, 0); + x = strtoul(col, NULL, 0); + if (y != 0 && (y-1) != pos->tp_row) + vtc_fatal(pp->vl, "Cursor on line %d (expected %d)", + pos->tp_row + 1, y); + if (x != 0 && (x-1) != pos->tp_col) + vtc_fatal(pp->vl, "Cursor in column %d (expected %d)", + pos->tp_col + 1, y); +} + /********************************************************************** * Allocate and initialize a process */ @@ -968,7 +989,15 @@ cmd_process(CMD_ARGS) av += 3; continue; } - if (!strcmp(*av, "-screen_dump")) { + if (!strcmp(*av, "-expect-cursor")) { + AN(av[1]); + AN(av[2]); + term_expect_cursor(p, av[1], av[2]); + av += 2; + continue; + } + if (!strcmp(*av, "-screen_dump") || + !strcmp(*av, "-screen-dump")) { term_screen_dump(p); continue; } From phk at FreeBSD.org Thu Apr 12 08:14:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 12 Apr 2018 08:14:13 +0000 (UTC) Subject: [master] ce0f13b Test form-feed Message-ID: <20180412081413.C19FDA29E8@lists.varnish-cache.org> commit ce0f13bbd59ee3e5f65d828e79b272d5d1b88f18 Author: Poul-Henning Kamp Date: Thu Apr 12 06:37:37 2018 +0000 Test form-feed diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index 52736fd..f5ef791 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -51,6 +51,8 @@ process p4 -hexdump {stty raw -echo; echo "*" ; sleep 2 ; cat} -start process p4 -expect-text 0 0 "*" +process p4 -write "\x1b[H\x1b[2Jzzzzzzz" +process p4 -write "\x0c1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H" process p4 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H" process p4 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n" process p4 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H" @@ -61,7 +63,7 @@ process p4 -write "\x0d\x0a111111112222222333333\x0d\x0a\x1b[12" process p4 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b" process p4 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a" -process p4 -need-bytes 270 -expect-text 3 1 "line3 <" +process p4 -need-bytes 310 -expect-text 3 1 "line3 <" process p4 -expect-cursor 4 1 process p4 -expect-cursor 4 0 process p4 -expect-cursor 0 1 From phk at FreeBSD.org Thu Apr 12 08:14:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 12 Apr 2018 08:14:13 +0000 (UTC) Subject: [master] 2c6f24d Clarify things for Coverity Message-ID: <20180412081413.E1DD8A29EA@lists.varnish-cache.org> commit 2c6f24d26ed4852963f2ff7a55029f6d765f3c3d Author: Poul-Henning Kamp Date: Thu Apr 12 08:07:00 2018 +0000 Clarify things for Coverity diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 73f42b8..663a363 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -388,6 +388,7 @@ haproxy_build_backends(const struct haproxy *h, const char *vsb_data) if (err != NULL) vtc_fatal(h->vl, "Create listen socket failed: %s", err); + assert(sock > 0); VTCP_myname(sock, addr, sizeof addr, port, sizeof port); bprintf(buf, "%s_%s", h->name, p); From hermunn at varnish-software.com Thu Apr 12 08:54:08 2018 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Thu, 12 Apr 2018 08:54:08 +0000 (UTC) Subject: [4.1] f108ad6 Adapt some test cases from master to avoid regression Message-ID: <20180412085408.D20C0A38D5@lists.varnish-cache.org> commit f108ad62d1ce2e3d1fd13c2462cf9b91765d5d4f Author: P?l Hermunn Johansen Date: Thu Apr 12 10:28:51 2018 +0200 Adapt some test cases from master to avoid regression Back port two VCC test cases from the master branch. The latter is from 4b48f886a5d9937fbd5b56a1560c755d by Nils Goroll diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index 990f39e..549f11c 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -127,3 +127,24 @@ varnish v1 -errvcl {Backend not found: 'c'} { } } } + +varnish v1 -errvcl {Regexp compilation error:} { + backend b { .host = "127.0.0.1"; } + sub vcl_recv { + if (req.url ~ "[a") {} + } +} + +varnish v1 -errvcl {Undefined acl foo} { + import ${vmod_directors}; + backend b { .host = "127.0.0.1"; } + + sub vcl_init { + new foo = directors.round_robin(); + } + sub vcl_recv { + if (client.ip ~ foo) { + return (synth(200)); + } + } +} From daghf at varnish-software.com Thu Apr 12 11:43:08 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Thu, 12 Apr 2018 11:43:08 +0000 (UTC) Subject: [master] ec0f819 Redo struct h2_sess allocation/workspace snapshot Message-ID: <20180412114308.7343AA6F62@lists.varnish-cache.org> commit ec0f8199f176c3df58bf2fc5f37a596b6caf2f20 Author: Dag Haavi Finstad Date: Thu Apr 12 13:42:30 2018 +0200 Redo struct h2_sess allocation/workspace snapshot Instead of allocating struct h2_sess out of the same workspace that is used as our rxbuffer, we rather keep it on the stack in h2_new_session. If the size of the initial h/2 request payload is close to half the size of the workspace, we would end up exhausting the workspace in HTC_RxInit after allocating our h2sess struct. Spotted by Federico G. Schwindt Fixes: #2619 diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 799d109..e34f824 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -93,12 +93,11 @@ h2_local_settings(struct h2_settings *h2s) * WS, VSL, HTC &c, but rather than implement all that stuff over, we * grab an actual struct req, and mirror the relevant fields into * struct h2_sess. - * To make things really incestuous, we allocate the h2_sess on - * the WS of that "Session ReQuest". */ static struct h2_sess * -h2_new_sess(const struct worker *wrk, struct sess *sp, struct req *srq) +h2_init_sess(const struct worker *wrk, struct sess *sp, + struct h2_sess *h2s, struct req *srq) { uintptr_t *up; struct h2_sess *h2; @@ -112,7 +111,7 @@ h2_new_sess(const struct worker *wrk, struct sess *sp, struct req *srq) if (srq == NULL) srq = Req_New(wrk, sp); AN(srq); - h2 = WS_Alloc(srq->ws, sizeof *h2); + h2 = h2s; AN(h2); INIT_OBJ(h2, H2_SESS_MAGIC); h2->srq = srq; @@ -331,9 +330,9 @@ h2_new_session(struct worker *wrk, void *arg) { struct req *req; struct sess *sp; + struct h2_sess h2s; struct h2_sess *h2; struct h2_req *r2, *r22; - uintptr_t wsp; int again; uint8_t settings[48]; size_t l; @@ -347,8 +346,8 @@ h2_new_session(struct worker *wrk, void *arg) assert (req->err_code == H2_PU_MARKER || req->err_code == H2_OU_MARKER); - h2 = h2_new_sess(wrk, sp, req->err_code == H2_PU_MARKER ? req : NULL); - wsp = WS_Snapshot(h2->ws); + h2 = h2_init_sess(wrk, sp, &h2s, + req->err_code == H2_PU_MARKER ? req : NULL); h2->req0 = h2_new_req(wrk, h2, 0, NULL); if (req->err_code == H2_OU_MARKER && !h2_ou_session(wrk, h2, req)) { @@ -359,6 +358,7 @@ h2_new_session(struct worker *wrk, void *arg) } assert(HTC_S_COMPLETE == H2_prism_complete(h2->htc)); HTC_RxPipeline(h2->htc, h2->htc->rxbuf_b + sizeof(H2_prism)); + WS_Reset(h2->ws, 0); HTC_RxInit(h2->htc, h2->ws); AN(h2->ws->r); VSLb(h2->vsl, SLT_Debug, "H2: Got pu PRISM"); @@ -380,7 +380,7 @@ h2_new_session(struct worker *wrk, void *arg) h2->cond = &wrk->cond; while (h2_rxframe(wrk, h2)) { - WS_Reset(h2->ws, wsp); + WS_Reset(h2->ws, 0); HTC_RxInit(h2->htc, h2->ws); if (WS_Overflowed(h2->ws)) { VSLb(h2->vsl, SLT_Debug, "H2: Empty Rx Workspace"); From fgsch at lodoss.net Fri Apr 13 09:31:15 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 13 Apr 2018 09:31:15 +0000 (UTC) Subject: [master] cffcca5 Plug some leaks Message-ID: <20180413093115.4974093D0D@lists.varnish-cache.org> commit cffcca5556cd06fec5304e7e20874338d4b5ffb3 Author: Federico G. Schwindt Date: Fri Apr 13 10:16:32 2018 +0100 Plug some leaks diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index d4e45c7..c6c5897 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -167,7 +167,7 @@ macro_undef(struct vtclog *vl, const char *instance, const char *name) VTAILQ_REMOVE(¯o_list, m, list); free(m->name); free(m->val); - free(m); + FREE_OBJ(m); } AZ(pthread_mutex_unlock(¯o_mtx)); } diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 663a363..8d4118f 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -56,7 +56,7 @@ struct haproxy { struct vtclog *vl; VTAILQ_ENTRY(haproxy) list; - char *filename; + const char *filename; struct vsb *args; int opt_worker; int opt_daemon; @@ -72,7 +72,7 @@ struct haproxy { int expect_signal; int its_dead_jim; - const char *cli_fn; + char *cli_fn; char *workdir; struct vsb *msgs; @@ -152,7 +152,7 @@ haproxy_new(const char *name) h->filename = getenv(HAPROXY_PROGRAM_ENV_VAR); if (h->filename == NULL) - REPLACE(h->filename, "haproxy"); + h->filename = "haproxy"; bprintf(buf, "${tmpdir}/%s", name); vsb = macro_expand(h->vl, buf); @@ -196,6 +196,9 @@ haproxy_delete(struct haproxy *h) free(h->name); free(h->workdir); + free(h->cli_fn); + free(h->cfg_fn); + free(h->pid_fn); VSB_destroy(&h->args); /* XXX: MEMLEAK (?) */ diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 0634d66..2c0b761 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -884,6 +884,7 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, } } else if (!strcmp(*av, "-bodylen")) { assert(body == nullbody); + free(body); body = synth_body(av[1], 0); bodylen = strlen(body); av++; @@ -895,13 +896,16 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, av++; } else if (!strcmp(*av, "-gziplen")) { assert(body == nullbody); + free(body); b = synth_body(av[1], 1); gzip_body(hp, b, &body, &bodylen); + free(b); VSB_printf(hp->vsb, "Content-Encoding: gzip%s", nl); // vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen); av++; } else if (!strcmp(*av, "-gzipbody")) { assert(body == nullbody); + free(body); gzip_body(hp, av[1], &body, &bodylen); VSB_printf(hp->vsb, "Content-Encoding: gzip%s", nl); // vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen); @@ -1911,7 +1915,8 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, free(hp->rxbuf); free(hp->rem_ip); free(hp->rem_port); - free(hp); + free(hp->rem_path); + FREE_OBJ(hp); return (retval); } diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index cbb3c07..735b747 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -191,6 +191,7 @@ logexp_delete(struct logexp *le) AZ(le->vslq); logexp_delete_tests(le); free(le->name); + free(le->vname); free(le->query); VSM_Destroy(&le->vsm); FREE_OBJ(le); diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index a3f920d..af6da00 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -386,6 +386,7 @@ process_new(const char *name) static void process_delete(struct process *p) { + int i; CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC); AZ(pthread_mutex_destroy(&p->mtx)); @@ -401,6 +402,10 @@ process_delete(struct process *p) * to the test's tmpdir. */ + for (i = 0; i < p->nlin; i++) + free(p->vram[i]); + free(p->vram); + /* XXX: MEMLEAK (?) */ FREE_OBJ(p); } From fgsch at lodoss.net Fri Apr 13 09:31:15 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 13 Apr 2018 09:31:15 +0000 (UTC) Subject: [master] 606ff7d Whitespace and style OCD Message-ID: <20180413093115.609A093D13@lists.varnish-cache.org> commit 606ff7dad63cbcbc02cac4f86ac307a1106c000d Author: Federico G. Schwindt Date: Fri Apr 13 10:23:21 2018 +0100 Whitespace and style OCD diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 25bdfda..c66b8a2 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -277,8 +277,7 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) if (++b->waiters == b->expected) { vtc_log(vl, 4, "Barrier(%s) wake %u", b->name, b->expected); AZ(pthread_cond_broadcast(&b->cond)); - } - else { + } else { vtc_log(vl, 4, "Barrier(%s) wait %u of %u", b->name, b->waiters, b->expected); AZ(pthread_cond_wait(&b->cond, &b->mtx)); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 2c0b761..db07d42 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1901,8 +1901,7 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, VTCP_hisname(sock, hp->rem_ip, VTCP_ADDRBUFSIZE, hp->rem_port, VTCP_PORTBUFSIZE); hp->rem_path = NULL; - } - else { + } else { strcpy(hp->rem_ip, "0.0.0.0"); strcpy(hp->rem_port, "0"); hp->rem_path = strdup(addr); diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index af6da00..b05d0ce 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -730,9 +730,8 @@ process_kill(struct process *p, const char *sig) if (kill(-pid, j) < 0) vtc_fatal(p->vl, "Failed to send signal %d (%s)", j, strerror(errno)); - else { + else vtc_log(p->vl, 4, "Sent signal %d", j); - } } /********************************************************************** diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index 987eb98..33e2aa6 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -283,7 +283,7 @@ Emit_UDS_Path(struct vcc *tl, const struct token *t_path, const char *errid) vcc_ErrWhere(tl, t_path); return; } - if (! S_ISSOCK(st.st_mode)) { + if (!S_ISSOCK(st.st_mode)) { VSB_printf(tl->sb, "%s: Not a socket:\n", errid); vcc_ErrWhere(tl, t_path); return; diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index 17b0da2..75dda13 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -200,7 +200,7 @@ dyn_uds_init(VRT_CTX, struct xyzzy_debug_dyn_uds *uds, VCL_STRING path) VRT_fail(ctx, "Cannot stat path %s: %s", path, strerror(errno)); return (-1); } - if (! S_ISSOCK(st.st_mode)) { + if (!S_ISSOCK(st.st_mode)) { VRT_fail(ctx, "%s is not a socket", path); return (-1); } diff --git a/lib/libvmod_unix/cred_compat.h b/lib/libvmod_unix/cred_compat.h index 843793a..cd4fd74 100644 --- a/lib/libvmod_unix/cred_compat.h +++ b/lib/libvmod_unix/cred_compat.h @@ -95,7 +95,7 @@ get_ids(int fd, uid_t *uid, gid_t *gid) priv_set_t *priv = NULL; errno = 0; - if (! priv_ineffect(PRIV_PROC_INFO)) { + if (!priv_ineffect(PRIV_PROC_INFO)) { priv = priv_proc_info; if (setppriv(PRIV_ON, PRIV_EFFECTIVE, priv)) return (CREDS_FAIL); diff --git a/lib/libvmod_unix/vmod_unix.c b/lib/libvmod_unix/vmod_unix.c index f51c108..59bbb2d 100644 --- a/lib/libvmod_unix/vmod_unix.c +++ b/lib/libvmod_unix/vmod_unix.c @@ -94,7 +94,7 @@ vmod_##func(VRT_CTX) \ } \ \ sp = get_sp(ctx); \ - if (! sp->listen_sock->uds) { \ + if (!sp->listen_sock->uds) { \ ERRNOTUDS(ctx); \ return (-1); \ } \ From fgsch at lodoss.net Fri Apr 13 09:51:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 13 Apr 2018 09:51:08 +0000 (UTC) Subject: [master] 5d026a3 Fix ASAN compilation under macaos Message-ID: <20180413095108.0DAA4943EF@lists.varnish-cache.org> commit 5d026a310b04e6a4a51cadbf69479e7011e98002 Author: Federico G. Schwindt Date: Fri Apr 13 10:49:27 2018 +0100 Fix ASAN compilation under macaos diff --git a/configure.ac b/configure.ac index 48ab568..3877a49 100644 --- a/configure.ac +++ b/configure.ac @@ -280,7 +280,11 @@ AC_ARG_ENABLE(msan, if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then SAN_CFLAGS="-D__SANITIZER=1 ${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -fPIC -fPIE -fno-omit-frame-pointer" - SAN_LDFLAGS="${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -pie" + SAN_LDFLAGS="${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS}" + save_CFLAGS="${CFLAGS}" + CFLAGS="${CFLAGS} -Werror=unused-command-line-argument" + AX_CHECK_LINK_FLAG([-pie], [SAN_LDFLAGS="${SAN_LDFLAGS} -pie"]) + CFLAGS="${save_CFLAGS}" case $CC in gcc*) SAN_CFLAGS="${SAN_CFLAGS} -fuse-ld=gold" From phk at FreeBSD.org Sat Apr 14 07:11:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 14 Apr 2018 07:11:13 +0000 (UTC) Subject: [master] 49f3387 Exercise the CONS25 code. Message-ID: <20180414071113.A7909B42A8@lists.varnish-cache.org> commit 49f3387c99f20f825eee39d4e10a2310c6d252c0 Author: Poul-Henning Kamp Date: Sat Apr 14 07:09:21 2018 +0000 Exercise the CONS25 code. diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index f5ef791..a79ceed 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -41,15 +41,14 @@ shell -exit 77 -expect {TEST _.vtc skipped} { exec varnishtest -v _.vtc || true } -process p1 "ps -lw | grep '[p][s]' ; tty ; sleep 1" -run -screen_dump +process p1 "ps -lw | grep '[p][s]' ; tty ; echo @" -start +process p1 -expect-text 0 0 {@} -screen_dump -wait -process p2 "stty -a ; sleep 1" -run -screen_dump +process p2 "stty -a ; echo '*'" -start +process p2 -expect-text 0 0 {*} -screen_dump -wait -process p3 "stty raw -echo ; stty -a ; sleep 1" -run -screen_dump - -process p4 -hexdump {stty raw -echo; echo "*" ; sleep 2 ; cat} -start - -process p4 -expect-text 0 0 "*" +process p4 -hexdump {stty raw -echo; stty -a ; echo "*" ; cat} -start +process p4 -expect-text 0 0 "*" -screen_dump process p4 -write "\x1b[H\x1b[2Jzzzzzzz" process p4 -write "\x0c1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H" @@ -67,4 +66,28 @@ process p4 -need-bytes 310 -expect-text 3 1 "line3 <" process p4 -expect-cursor 4 1 process p4 -expect-cursor 4 0 process p4 -expect-cursor 0 1 -process p4 -screen-dump -stop +process p4 -screen-dump + +# Exercise CONS25 mode +process p4 -write "\x1b[=1T" +process p4 -write "\x1b[8z" +process p4 -write "\x1b[0x" +process p4 -write "\x1b[=1A" +process p4 -write "\x1b[=1;2B" +process p4 -write "\x1b[=1;2;3C" +process p4 -write "\x1b[=1F" +process p4 -write "\x1b[=1G" +process p4 -write "\x1b[=1S" +process p4 -writehex {0c 08 40 0d 0a 08} + +process p4 -expect-text 1 1 "@" +process p4 -expect-cursor 1 80 +process p4 -writehex "0c 41 0e 42 0f" +process p4 -expect-text 1 1 "A" +process p4 -expect-text 0 0 "B" +process p4 -write "\x1b[=0T" + +process p4 -writehex "0c 0a 0d 43 0a 08 08 0e 44 0f" + +process p4 -expect-text 3 1 "C" +process p4 -expect-text 4 1 "D" From fgsch at lodoss.net Sat Apr 14 09:32:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 14 Apr 2018 09:32:11 +0000 (UTC) Subject: [master] 091e29b Plug a few more leaks Message-ID: <20180414093211.2F7BAB6B9C@lists.varnish-cache.org> commit 091e29badc49e8c22589be17d748f8d337ae1086 Author: Federico G. Schwindt Date: Sat Apr 14 08:56:29 2018 +0100 Plug a few more leaks diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 6776e88..8ad4a5a 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -96,7 +96,7 @@ void cmd_server_gen_haproxy_conf(struct vsb *vsb); void vtc_loginit(char *buf, unsigned buflen); struct vtclog *vtc_logopen(const char *id); -void vtc_logclose(struct vtclog *vl); +void vtc_logclose(void *arg); void vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...) v_printflike_(3, 4); void vtc_fatal(struct vtclog *vl, const char *, ...) diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index c66b8a2..116efdd 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -137,7 +137,7 @@ barrier_sock_thread(void *priv) AZ(pthread_mutex_lock(&b->mtx)); vl = vtc_logopen(b->name); - AN(vl); + pthread_cleanup_push(vtc_logclose, vl); sock = VTCP_listen_on("127.0.0.1:0", NULL, b->expected, &err); if (sock < 0) { @@ -219,7 +219,7 @@ barrier_sock_thread(void *priv) macro_undef(vl, b->name, "sock"); closefd(&sock); free(conns); - + pthread_cleanup_pop(1); return (NULL); } diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 0a0cb6d..4b2f608 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -207,6 +207,7 @@ client_thread(void *priv) AN(*c->connect); vl = vtc_logopen(c->name); + pthread_cleanup_push(vtc_logclose, vl); vsb = macro_expand(vl, c->connect); AN(vsb); @@ -236,6 +237,7 @@ client_thread(void *priv) } vtc_log(vl, 2, "Ending"); VSB_destroy(&vsb); + pthread_cleanup_pop(1); return (NULL); } diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index db07d42..edfe796 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1865,6 +1865,21 @@ const struct cmds http_cmds[] = { { NULL, NULL } }; +static void +http_process_cleanup(void *arg) +{ + struct http *hp = arg; + + if (hp->h2) + stop_h2(hp); + VSB_destroy(&hp->vsb); + free(hp->rxbuf); + free(hp->rem_ip); + free(hp->rem_port); + free(hp->rem_path); + FREE_OBJ(hp); +} + int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, const char *addr) @@ -1906,16 +1921,10 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, strcpy(hp->rem_port, "0"); hp->rem_path = strdup(addr); } + pthread_cleanup_push(http_process_cleanup, hp); parse_string(spec, http_cmds, hp, vl); - if (hp->h2) - stop_h2(hp); retval = hp->fd; - VSB_destroy(&hp->vsb); - free(hp->rxbuf); - free(hp->rem_ip); - free(hp->rem_port); - free(hp->rem_path); - FREE_OBJ(hp); + pthread_cleanup_pop(1); return (retval); } diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 8f917d4..2c458f3 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -2541,6 +2541,7 @@ static void stream_delete(struct stream *s) { CHECK_OBJ_NOTNULL(s, STREAM_MAGIC); + free(s->body); free(s->spec); free(s->name); FREE_OBJ(s); diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index 72108b1..d7ceefe 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -90,8 +90,9 @@ vtc_logopen(const char *id) } void -vtc_logclose(struct vtclog *vl) +vtc_logclose(void *arg) { + struct vtclog *vl = arg; CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC); if (pthread_getspecific(log_key) == vl) diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 735b747..add0f1f 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -194,6 +194,7 @@ logexp_delete(struct logexp *le) free(le->vname); free(le->query); VSM_Destroy(&le->vsm); + vtc_logclose(le->vl); FREE_OBJ(le); } diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 71ef165..8da7c56 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -240,8 +240,10 @@ tst_cb(const struct vev *ve, int what) jp->tst->filename, ecode ? "skipped" : "passed", t); } - if (jp->evt != NULL) + if (jp->evt != NULL) { VEV_Stop(vb, jp->evt); + free(jp->evt); + } FREE_OBJ(jp); return (1); diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 1307cd5..ca8c6a5 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -232,6 +232,7 @@ server_thread(void *priv) assert(s->sock >= 0); vl = vtc_logopen(s->name); + pthread_cleanup_push(vtc_logclose, vl); vtc_log(vl, 2, "Started on %s", s->listen); for (i = 0; i < s->repeat; i++) { @@ -255,6 +256,7 @@ server_thread(void *priv) VTCP_close(&fd); } vtc_log(vl, 2, "Ending"); + pthread_cleanup_pop(1); return (NULL); } @@ -288,6 +290,7 @@ server_dispatch_wrk(void *priv) assert(s->sock < 0); vl = vtc_logopen(s->name); + pthread_cleanup_push(vtc_logclose, vl); fd = s->fd; @@ -299,6 +302,7 @@ server_dispatch_wrk(void *priv) vtc_fatal(vl, "Shutdown failed: %s", strerror(errno)); VTCP_close(&s->fd); vtc_log(vl, 2, "Ending"); + pthread_cleanup_pop(1); return (NULL); } @@ -317,7 +321,8 @@ server_dispatch_thread(void *priv) assert(s->sock >= 0); vl = vtc_logopen(s->name); - AN(vl); + pthread_cleanup_push(vtc_logclose, vl); + vtc_log(vl, 2, "Dispatch started on %s", s->listen); while (1) { @@ -335,6 +340,7 @@ server_dispatch_thread(void *priv) s2->run = 1; AZ(pthread_create(&s2->tp, NULL, server_dispatch_wrk, s2)); } + pthread_cleanup_pop(1); NEEDLESS(return(NULL)); } From fgsch at lodoss.net Sat Apr 14 09:32:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 14 Apr 2018 09:32:11 +0000 (UTC) Subject: [master] e737210 Polish Message-ID: <20180414093211.423ACB6B9E@lists.varnish-cache.org> commit e73721038af4897bc6a3c4dbfafce6486716cd08 Author: Federico G. Schwindt Date: Sat Apr 14 08:56:56 2018 +0100 Polish diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 981c5bf..7a427d5 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -462,8 +462,8 @@ mgt_reap_child(void) if (ev_poker != NULL) { VEV_Stop(mgt_evb, ev_poker); free(ev_poker); + ev_poker = NULL; } - ev_poker = NULL; /* Stop the listener */ if (ev_listen != NULL) { diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 2c458f3..b8857b3 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -293,8 +293,7 @@ clean_frame(struct frame **f) if ((*f)->type == TYPE_GOAWAY) free((*f)->md.goaway.debug); free((*f)->data); - free(*f); - *f = NULL; + FREE_OBJ(*f); } static void diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index b05d0ce..44103f4 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -396,16 +396,16 @@ process_delete(struct process *p) free(p->out); free(p->err); + for (i = 0; i < p->nlin; i++) + free(p->vram[i]); + free(p->vram); + /* * We do not delete the directory, it may contain useful stdout * and stderr files. They will be deleted on account of belonging * to the test's tmpdir. */ - for (i = 0; i < p->nlin; i++) - free(p->vram[i]); - free(p->vram); - /* XXX: MEMLEAK (?) */ FREE_OBJ(p); } From fgsch at lodoss.net Sat Apr 14 10:26:09 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 14 Apr 2018 10:26:09 +0000 (UTC) Subject: [master] 02072a9 Try to appease Sun CC Message-ID: <20180414102609.B9B6AB7ACB@lists.varnish-cache.org> commit 02072a9eaa8bcf765793fa2cf3b4bed3b0aa5e93 Author: Federico G. Schwindt Date: Sat Apr 14 11:24:08 2018 +0100 Try to appease Sun CC diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index ca8c6a5..04199de 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -340,7 +340,7 @@ server_dispatch_thread(void *priv) s2->run = 1; AZ(pthread_create(&s2->tp, NULL, server_dispatch_wrk, s2)); } - pthread_cleanup_pop(1); + NEEDLESS(pthread_cleanup_pop(1)); NEEDLESS(return(NULL)); } From fgsch at lodoss.net Sat Apr 14 10:49:09 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 14 Apr 2018 10:49:09 +0000 (UTC) Subject: [master] 0fe5128 Just hide this whole thing from Sun CC for now Message-ID: <20180414104909.14FC1B814F@lists.varnish-cache.org> commit 0fe51283c27ab03ee8a93bf9699ea6fbabad846b Author: Federico G. Schwindt Date: Sat Apr 14 11:48:18 2018 +0100 Just hide this whole thing from Sun CC for now diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 04199de..2943058 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -321,7 +321,9 @@ server_dispatch_thread(void *priv) assert(s->sock >= 0); vl = vtc_logopen(s->name); +#if !defined(__SUNPRO_C) pthread_cleanup_push(vtc_logclose, vl); +#endif vtc_log(vl, 2, "Dispatch started on %s", s->listen); @@ -340,7 +342,9 @@ server_dispatch_thread(void *priv) s2->run = 1; AZ(pthread_create(&s2->tp, NULL, server_dispatch_wrk, s2)); } - NEEDLESS(pthread_cleanup_pop(1)); +#if !defined(__SUNPRO_C) + pthread_cleanup_pop(1); +#endif NEEDLESS(return(NULL)); } From fgsch at lodoss.net Sat Apr 14 19:08:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 14 Apr 2018 19:08:11 +0000 (UTC) Subject: [master] 6b944c3 Rework to avoid UB Message-ID: <20180414190811.594A794C2E@lists.varnish-cache.org> commit 6b944c3aa88d48ad0191967ac3c66532df209836 Author: Federico G. Schwindt Date: Sat Apr 14 20:03:46 2018 +0100 Rework to avoid UB Fixes #2617 diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 8797c92..6c3e2ce 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -669,8 +669,9 @@ http_GetHdrField(const struct http *hp, const char *hdr, ssize_t http_GetContentLength(const struct http *hp) { - ssize_t cl, cll; + ssize_t cl; const char *b; + unsigned n; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -680,11 +681,13 @@ http_GetContentLength(const struct http *hp) if (!vct_isdigit(*b)) return (-2); for (; vct_isdigit(*b); b++) { - cll = cl; + if (cl > (SSIZE_MAX / 10)) + return (-2); cl *= 10; - cl += *b - '0'; - if (cll != cl / 10) + n = *b - '0'; + if (cl > (SSIZE_MAX - n)) return (-2); + cl += n; } while (vct_islws(*b)) b++; From fgsch at lodoss.net Sat Apr 14 19:34:07 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 14 Apr 2018 19:34:07 +0000 (UTC) Subject: [master] ff7922b Add test for #2617 Message-ID: <20180414193407.951409544F@lists.varnish-cache.org> commit ff7922b799f5c02b5cc257caebee504873d94eaa Author: Federico G. Schwindt Date: Sat Apr 14 20:19:24 2018 +0100 Add test for #2617 diff --git a/bin/varnishtest/tests/r02617.vtc b/bin/varnishtest/tests/r02617.vtc new file mode 100644 index 0000000..78d9d2a --- /dev/null +++ b/bin/varnishtest/tests/r02617.vtc @@ -0,0 +1,16 @@ +varnishtest "Test undefined-behaviour: signed integer overflow" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + +} -start + +client c1 { + txreq -hdr "Content-Length: 9223372036854775808" + rxresp + expect resp.status == 400 +} -run From phk at FreeBSD.org Mon Apr 16 05:34:15 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 16 Apr 2018 05:34:15 +0000 (UTC) Subject: [master] 932ed4a More Teken coverage Message-ID: <20180416053415.518C495FD8@lists.varnish-cache.org> commit 932ed4af2a26b9df3aa0639b16ec3b15b816c509 Author: Poul-Henning Kamp Date: Mon Apr 16 05:26:48 2018 +0000 More Teken coverage diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index a79ceed..868fa54 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -57,7 +57,13 @@ process p4 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n" process p4 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H" process p4 -write "111111112222222333333\x0d\x0a111111112" process p4 -write "222222333333\x0d\x0a111111112222222333333 UTF8: " -process p4 -writehex {c2 a2 20 e2 82 ac 20 f0 9f 90 b0} +process p4 -writehex {c2 a2 20} +process p4 -writehex {e2 82 ac 20} +process p4 -writehex {f0 90 80 80 20} +process p4 -writehex {f0 9f 90 b0 20} +process p4 -writehex {f0 a0 80 80 20} +process p4 -writehex {f0 b0 80 80 20} +process p4 -write "\x1b[22;24;25;27;30;47m" process p4 -write "\x0d\x0a111111112222222333333\x0d\x0a\x1b[12" process p4 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b" process p4 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a" @@ -68,13 +74,15 @@ process p4 -expect-cursor 4 0 process p4 -expect-cursor 0 1 process p4 -screen-dump -# Exercise CONS25 mode +# Also exercise CONS25 mode process p4 -write "\x1b[=1T" +process p4 -write "\x1b[=2T" process p4 -write "\x1b[8z" process p4 -write "\x1b[0x" process p4 -write "\x1b[=1A" process p4 -write "\x1b[=1;2B" process p4 -write "\x1b[=1;2;3C" +process p4 -write "\x1b[=1;2;3;4C" process p4 -write "\x1b[=1F" process p4 -write "\x1b[=1G" process p4 -write "\x1b[=1S" @@ -91,3 +99,8 @@ process p4 -writehex "0c 0a 0d 43 0a 08 08 0e 44 0f" process p4 -expect-text 3 1 "C" process p4 -expect-text 4 1 "D" +process p4 -write "\x1b[2T" +process p4 -expect-text 5 1 "C" +process p4 -expect-text 6 1 "D" +process p4 -write "\x1b[3S" +process p4 -expect-text 3 1 "D" diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 44103f4..c182ed3 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -346,6 +346,30 @@ term_expect_cursor(struct process *pp, const char *lin, const char *col) VSB_destroy(&vsb); \ } while (0) +static void +process_coverage(struct process *p) +{ + const teken_attr_t *a; + teken_pos_t pos; + int fg, bg; + + // Code-Coverage of Teken + + (void)teken_get_sequence(p->tek, TKEY_UP); + (void)teken_get_sequence(p->tek, TKEY_F1); + (void)teken_256to8(0); + (void)teken_256to16(0); + a = teken_get_defattr(p->tek); + teken_set_defattr(p->tek, a); + a = teken_get_curattr(p->tek); + teken_set_curattr(p->tek, a); + (void)teken_get_winsize(p->tek); + pos.tp_row = 0; + pos.tp_col = 8; + teken_set_cursor(p->tek, &pos); + teken_get_defattr_cons25(p->tek, &fg, &bg); +} + static struct process * process_new(const char *name) { @@ -374,6 +398,7 @@ process_new(const char *name) VTAILQ_INSERT_TAIL(&processes, p, list); teken_init(p->tek, &process_teken_func, p); term_resize(p, 24, 80); + process_coverage(p); return (p); } From fgsch at lodoss.net Mon Apr 16 21:27:13 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 16 Apr 2018 21:27:13 +0000 (UTC) Subject: [master] f8cdec1 Tighten up checks Message-ID: <20180416212713.397A5AFBAC@lists.varnish-cache.org> commit f8cdec1d0253708d401dce03b32bc08c81f2c6b8 Author: Federico G. Schwindt Date: Mon Apr 16 22:15:37 2018 +0100 Tighten up checks diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 9772ac8..4bef5e7 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -243,6 +243,9 @@ VTIM_parse(const char *p) int hour = 0, min = 0, sec = 0; int d, leap; + if (p == NULL || *p == '\0') + FAIL(); + while (*p == ' ') p++; diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index bb2be44..ba10fa5 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -194,8 +194,6 @@ vmod_time(VRT_CTX, VCL_STRING p, VCL_TIME d) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (p == NULL) - return (d); r = VTIM_parse(p); if (r) return (r); diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c index 201d743..1bb003e 100644 --- a/lib/libvmod_std/vmod_std_fileread.c +++ b/lib/libvmod_std/vmod_std_fileread.c @@ -90,6 +90,9 @@ vmod_fileread(VRT_CTX, struct vmod_priv *priv, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); AN(priv); + if (file_name == NULL) + return (NULL); + if (priv->priv != NULL) { CAST_OBJ_NOTNULL(frf, priv->priv, CACHED_FILE_MAGIC); if (!strcmp(file_name, frf->file_name)) From fgsch at lodoss.net Mon Apr 16 21:27:13 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 16 Apr 2018 21:27:13 +0000 (UTC) Subject: [master] 12610fd More coverage Message-ID: <20180416212713.505A0AFBB9@lists.varnish-cache.org> commit 12610fd5f70185141d92c47ecf3cbd19375a7dcc Author: Federico G. Schwindt Date: Mon Apr 16 22:17:25 2018 +0100 More coverage diff --git a/bin/varnishtest/tests/m00014.vtc b/bin/varnishtest/tests/m00014.vtc index 6d60def..f3272a6 100644 --- a/bin/varnishtest/tests/m00014.vtc +++ b/bin/varnishtest/tests/m00014.vtc @@ -9,40 +9,50 @@ varnish v1 -vcl+backend { import std; sub vcl_deliver { - set resp.http.url = std.querysort(req.url); + set resp.http.url = std.querysort(req.http.url); } } -start client c1 { - txreq -url "/foo/bar?t=0&b=0&p=0&c=5" + txreq -hdr "url: /foo/bar?t=0&b=0&p=0&c=5" rxresp expect resp.http.url == "/foo/bar?b=0&c=5&p=0&t=0" delay .1 - txreq -url "/foo/bar?coa=0&co=0" + txreq -hdr "url: /foo/bar?coa=0&co=0" rxresp expect resp.http.url == "/foo/bar?co=0&coa=0" delay .1 - txreq -url "/foo/bar?a=0&&&&&" + txreq -hdr "url: /foo/bar?a=0&&&&&" rxresp expect resp.http.url == "/foo/bar?a=0" - txreq -url "/foo/bar?&a=0&&&&&z&w&x&" + txreq -hdr "url: /foo/bar?&a=0&&&&&z&w&x&" rxresp expect resp.http.url == "/foo/bar?a=0&w&x&z" delay .1 - txreq -url "/foo/bar?&" + txreq -hdr "url: /foo/bar?&" rxresp expect resp.http.url == "/foo/bar?" delay .1 - txreq -url "/foo/bar" + txreq -hdr "url: /foo/bar?t=0" + rxresp + expect resp.http.url == "/foo/bar?t=0" + + delay .1 + + txreq -hdr "url: /foo/bar" rxresp expect resp.http.url == "/foo/bar" + + txreq + rxresp + expect resp.http.url == "" } -run diff --git a/bin/varnishtest/tests/m00018.vtc b/bin/varnishtest/tests/m00018.vtc index 33d68f3..f41faf2 100644 --- a/bin/varnishtest/tests/m00018.vtc +++ b/bin/varnishtest/tests/m00018.vtc @@ -1,6 +1,6 @@ varnishtest "Test std.substr" -server s1 -repeat 2 { +server s1 { rxreq txresp } -start @@ -9,16 +9,16 @@ varnish v1 -vcl+backend { import std; sub vcl_deliver { - set resp.http.sub = std.strstr(req.url, "b"); + set resp.http.sub1 = std.strstr(req.http.one, "b"); + set resp.http.sub2 = std.strstr(req.http.two, "b"); + set resp.http.sub3 = std.strstr(req.http.unset, "b"); } } -start client c1 { - txreq -url "/foobar" + txreq -hdr "one: foobar" -hdr "two: quux" rxresp - expect resp.http.sub == "bar" - - txreq -url "/quux" - rxresp - expect resp.http.sub == "" + expect resp.http.sub1 == "bar" + expect resp.http.sub2 == "" + expect resp.http.sub3 == "" } -run diff --git a/bin/varnishtest/tests/m00020.vtc b/bin/varnishtest/tests/m00020.vtc index 8fe5c83..eb2eb0f 100644 --- a/bin/varnishtest/tests/m00020.vtc +++ b/bin/varnishtest/tests/m00020.vtc @@ -127,4 +127,7 @@ client c1 { expect resp.http.x-date != "Wed, 29 Feb 2012 00:00:00 GMT" delay .1 + txreq + rxresp + expect resp.http.x-date != } -run diff --git a/bin/varnishtest/tests/m00026.vtc b/bin/varnishtest/tests/m00026.vtc index 8c5f717..3659eb3 100644 --- a/bin/varnishtest/tests/m00026.vtc +++ b/bin/varnishtest/tests/m00026.vtc @@ -11,6 +11,7 @@ varnish v1 -vcl { sub vcl_synth { set resp.http.X-PATH = std.getenv("PATH"); + set resp.http.X-unset = std.getenv(req.http.unset); } } -start @@ -18,4 +19,5 @@ client c1 { txreq rxresp expect resp.http.X-PATH ~ "^/" + expect resp.http.X-unset == "" } -run From phk at FreeBSD.org Tue Apr 17 06:58:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Apr 2018 06:58:09 +0000 (UTC) Subject: [master] f260415 Add a read-only beresp.filters variable which doesn't return that yet. Message-ID: <20180417065809.A3A7CB9E06@lists.varnish-cache.org> commit f26041594540d65d69693b22ef7ca0ae6db78249 Author: Poul-Henning Kamp Date: Mon Apr 16 15:36:16 2018 +0000 Add a read-only beresp.filters variable which doesn't return that yet. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 582f70b..e1d835f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -388,6 +388,7 @@ struct busyobj { struct worker *wrk; struct vfp_ctx *vfc; + char *filter_list; struct ws ws[1]; uintptr_t ws_bo; diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 86df664..bb55018 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -36,6 +36,7 @@ #include "common/heritage.h" #include "vcl.h" +#include "vct.h" #include "cache_director.h" #include "vrt_obj.h" @@ -913,3 +914,26 @@ HTTP_VAR(req) HTTP_VAR(resp) HTTP_VAR(bereq) HTTP_VAR(beresp) + +/*--------------------------------------------------------------------*/ + +VCL_STRING +VRT_r_beresp_filters(VRT_CTX) +{ + const char *p; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + if (ctx->bo->filter_list != NULL) + return(ctx->bo->filter_list); +#if 1 + p = "Not Yet"; +#else + p = VBF_Get_Filter_List(ctx->bo); + WS_Release(ctx->bo->ws, strlen(p) + 1); +#endif + while(vct_isspace(*p)) + p++; + return (p); +} + diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index 868fa54..ec84187 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -72,7 +72,7 @@ process p4 -need-bytes 310 -expect-text 3 1 "line3 <" process p4 -expect-cursor 4 1 process p4 -expect-cursor 4 0 process p4 -expect-cursor 0 1 -process p4 -screen-dump +process p4 -screen-dump # Also exercise CONS25 mode process p4 -write "\x1b[=1T" diff --git a/bin/varnishtest/tests/e00023.vtc b/bin/varnishtest/tests/e00023.vtc index 7d4aeaa..08517c6 100644 --- a/bin/varnishtest/tests/e00023.vtc +++ b/bin/varnishtest/tests/e00023.vtc @@ -37,7 +37,9 @@ server s1 { varnish v1 -vcl+backend { sub vcl_backend_response { + set beresp.http.filter1 = beresp.filters; set beresp.do_esi = true; + set beresp.http.filter2 = beresp.filters; } } -start diff --git a/bin/varnishtest/tests/e00028.vtc b/bin/varnishtest/tests/e00028.vtc index 9a5fe16..bc25d68 100644 --- a/bin/varnishtest/tests/e00028.vtc +++ b/bin/varnishtest/tests/e00028.vtc @@ -36,12 +36,14 @@ server s1 { varnish v1 -arg "-p feature=+esi_disable_xml_check" -vcl+backend { sub vcl_backend_response { + set beresp.http.filters1 = beresp.filters; set beresp.do_esi = true; if (bereq.url ~ "/baz") { set beresp.do_gzip = true; } elif (bereq.url ~ "/foo(bar|baz|qux)") { set beresp.do_gunzip = true; } + set beresp.http.filters2 = beresp.filters; } } -start diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 6df3635..f29a9ed 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -895,6 +895,14 @@ beresp.storage_hint ``VCL <= 4.0`` Hint to Varnish that you want to save this object to a particular storage backend. +beresp.filters + + Type: STRING + + Readable from: vcl_backend_response + + List of VFP filters the beresp.body will be pulled through. + obj ~~~ From phk at FreeBSD.org Tue Apr 17 06:58:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Apr 2018 06:58:09 +0000 (UTC) Subject: [master] 766bd96 Yet more Teken coverage Message-ID: <20180417065809.B7085B9E09@lists.varnish-cache.org> commit 766bd96227a30dc8722d4fa492e8e926ba08e9d6 Author: Poul-Henning Kamp Date: Tue Apr 17 06:57:07 2018 +0000 Yet more Teken coverage diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index ec84187..9ba4290 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -63,7 +63,7 @@ process p4 -writehex {f0 90 80 80 20} process p4 -writehex {f0 9f 90 b0 20} process p4 -writehex {f0 a0 80 80 20} process p4 -writehex {f0 b0 80 80 20} -process p4 -write "\x1b[22;24;25;27;30;47m" +process p4 -write "\x1b[22;24;25;27;30;47;49;97;107m" process p4 -write "\x0d\x0a111111112222222333333\x0d\x0a\x1b[12" process p4 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b" process p4 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a" @@ -104,3 +104,17 @@ process p4 -expect-text 5 1 "C" process p4 -expect-text 6 1 "D" process p4 -write "\x1b[3S" process p4 -expect-text 3 1 "D" + +process p4 -write "\x1b[4;200H%" +process p4 -expect-text 4 80 "%" + +process p4 -write "\x1b[7;7H\x09X\x09Y\x09Z\x1b[2ZW\x1b[2Ew\x1b[F*" + +process p4 -expect-text 7 17 "W" +process p4 -expect-text 9 1 "w" +process p4 -expect-text 8 1 "*" + +process p4 -write "\x1b[10;4HABCDEFGHIJKLMN" +process p4 -write "\x1b[8G\x1b[2X>" +process p4 -expect-text 10 8 ">" +process p4 -screen-dump From phk at FreeBSD.org Tue Apr 17 09:01:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Apr 2018 09:01:11 +0000 (UTC) Subject: [master] 3f5b741 Max 8 numerical arguments, but test overflow while at it. Message-ID: <20180417090111.73323632E0@lists.varnish-cache.org> commit 3f5b74174fbf6f6419367edcbd7fb15f50f831a6 Author: Poul-Henning Kamp Date: Tue Apr 17 09:00:26 2018 +0000 Max 8 numerical arguments, but test overflow while at it. diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index 9ba4290..d9fce12 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -64,6 +64,8 @@ process p4 -writehex {f0 9f 90 b0 20} process p4 -writehex {f0 a0 80 80 20} process p4 -writehex {f0 b0 80 80 20} process p4 -write "\x1b[22;24;25;27;30;47;49;97;107m" +process p4 -write "\x1b[22;24;25;27;30m" +process p4 -write "\x1b[47;49;97;107m" process p4 -write "\x0d\x0a111111112222222333333\x0d\x0a\x1b[12" process p4 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b" process p4 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a" From daghf at varnish-software.com Tue Apr 17 11:57:09 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 17 Apr 2018 11:57:09 +0000 (UTC) Subject: [master] e8d8266 Test for MAX_FRAME_SIZE in h2_frame_complete Message-ID: <20180417115709.824119180C@lists.varnish-cache.org> commit e8d8266412b900f87b5602f883c0639e27f9ca84 Author: Dag Haavi Finstad Date: Tue Apr 17 13:53:26 2018 +0200 Test for MAX_FRAME_SIZE in h2_frame_complete Return HTC_S_OVERFLOW if exceeded. This is a quick-fix for #2624, where we close the connection if a client exceeds MAX_FRAME_SIZE. More proper handling with draining the rest of the frame for the specific frames types that allow it (rfc7540 section 4.2) to come later. Fixes: #2624 diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 39d97e6..6a0c92b 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -814,14 +814,18 @@ h2_req_fail(struct req *req, enum sess_close reason) static enum htc_status_e v_matchproto_(htc_complete_f) h2_frame_complete(struct http_conn *htc) { + struct h2_sess *h2; int l; unsigned u; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); + CAST_OBJ_NOTNULL(h2, htc->priv, H2_SESS_MAGIC); l = htc->rxbuf_e - htc->rxbuf_b; if (l < 9) return (HTC_S_MORE); u = vbe32dec(htc->rxbuf_b) >> 8; + if (l > h2->local_settings.max_frame_size + 9) + return (HTC_S_OVERFLOW); if (l < u + 9) // XXX: Only for !DATA frames return (HTC_S_MORE); return (HTC_S_COMPLETE); @@ -1032,6 +1036,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) /* FALLTHROUGH */ default: + /* XXX: HTC_S_OVERFLOW / FRAME_SIZE_ERROR handling */ Lck_Lock(&h2->sess->mtx); VSLb(h2->vsl, SLT_Debug, "H2: No frame (hs=%d)", hs); h2->error = H2CE_NO_ERROR; diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index e34f824..b2ab70f 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -349,6 +349,8 @@ h2_new_session(struct worker *wrk, void *arg) h2 = h2_init_sess(wrk, sp, &h2s, req->err_code == H2_PU_MARKER ? req : NULL); h2->req0 = h2_new_req(wrk, h2, 0, NULL); + AZ(h2->htc->priv); + h2->htc->priv = h2; if (req->err_code == H2_OU_MARKER && !h2_ou_session(wrk, h2, req)) { assert(h2->refcnt == 1); From phk at FreeBSD.org Tue Apr 17 12:40:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Apr 2018 12:40:13 +0000 (UTC) Subject: [master] f78342e Test Teken responses Message-ID: <20180417124013.6B51192690@lists.varnish-cache.org> commit f78342e32ec6f3085f15a63f7294a6da26f8637e Author: Poul-Henning Kamp Date: Tue Apr 17 12:38:44 2018 +0000 Test Teken responses diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index d9fce12..842a5c0 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -116,7 +116,30 @@ process p4 -expect-text 7 17 "W" process p4 -expect-text 9 1 "w" process p4 -expect-text 8 1 "*" -process p4 -write "\x1b[10;4HABCDEFGHIJKLMN" +process p4 -write "\x1b[10;4HABCDEFGHIJKLMN\x1b(A#$%\x1b)A" process p4 -write "\x1b[8G\x1b[2X>" process p4 -expect-text 10 8 ">" process p4 -screen-dump + +# Safely test responses +process p5 -hexdump "stty raw -echo; echo '*' ; tr '\x02\x1b' '\x1b@'" -start +process p5 -expect-text 0 0 "*" +process p5 -write "\x02[3;1HA\x02[5n" +process p5 -expect-text 3 0 "n" +process p5 -write "\x02[4;1HB\x02[6n" +process p5 -expect-text 4 0 "R" +process p5 -write "\x02[5;1HC\x02[15n" +process p5 -expect-text 5 0 "n" +process p5 -write "\x02[6;1HD\x02[25n" +process p5 -expect-text 6 0 "n" +process p5 -write "\x02[7;1HE\x02[26n" +process p5 -expect-text 7 0 "n" +process p5 -write "\x02[8;1HF\x02[?26n" +process p5 -expect-text 8 0 "n" +process p5 -write "\x02[9;1HG\x02Pfutfutfut\x01" +process p5 -write "\x02[10;1HH\x02]futfutfut\x01" +process p5 -write "\x02[11;1HI\x02[>c" +process p5 -expect-text 11 0 "c" +process p5 -write "\x02[24;1HW" +process p5 -expect-text 0 0 "W" +process p5 -screen-dump From phk at FreeBSD.org Tue Apr 17 17:04:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Apr 2018 17:04:11 +0000 (UTC) Subject: [master] 9537895 Trust it to linux to break even tr(1). Message-ID: <20180417170411.3027A97A49@lists.varnish-cache.org> commit 9537895517cdd6c8e6145ab5920d9bcb8cf7bc02 Author: Poul-Henning Kamp Date: Tue Apr 17 16:46:10 2018 +0000 Trust it to linux to break even tr(1). Supress terminal responses by default and hope they look right. diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc index 842a5c0..9316a5c 100644 --- a/bin/varnishtest/tests/a00000.vtc +++ b/bin/varnishtest/tests/a00000.vtc @@ -121,25 +121,16 @@ process p4 -write "\x1b[8G\x1b[2X>" process p4 -expect-text 10 8 ">" process p4 -screen-dump -# Safely test responses -process p5 -hexdump "stty raw -echo; echo '*' ; tr '\x02\x1b' '\x1b@'" -start -process p5 -expect-text 0 0 "*" -process p5 -write "\x02[3;1HA\x02[5n" -process p5 -expect-text 3 0 "n" -process p5 -write "\x02[4;1HB\x02[6n" -process p5 -expect-text 4 0 "R" -process p5 -write "\x02[5;1HC\x02[15n" -process p5 -expect-text 5 0 "n" -process p5 -write "\x02[6;1HD\x02[25n" -process p5 -expect-text 6 0 "n" -process p5 -write "\x02[7;1HE\x02[26n" -process p5 -expect-text 7 0 "n" -process p5 -write "\x02[8;1HF\x02[?26n" -process p5 -expect-text 8 0 "n" -process p5 -write "\x02[9;1HG\x02Pfutfutfut\x01" -process p5 -write "\x02[10;1HH\x02]futfutfut\x01" -process p5 -write "\x02[11;1HI\x02[>c" -process p5 -expect-text 11 0 "c" -process p5 -write "\x02[24;1HW" -process p5 -expect-text 0 0 "W" -process p5 -screen-dump +# Test responses +process p4 -write "\x1b[3;1HA\x1b[5n" +process p4 -write "\x1b[4;1HB\x1b[6n" +process p4 -write "\x1b[5;1HC\x1b[15n" +process p4 -write "\x1b[6;1HD\x1b[25n" +process p4 -write "\x1b[7;1HE\x1b[26n" +process p4 -write "\x1b[8;1HF\x1b[?26n" +process p4 -write "\x1b[9;1HG\x1bPfutfutfut\x01" +process p4 -write "\x1b[10;1HH\x1b]futfutfut\x01" +process p4 -write "\x1b[11;1HI\x1b[>c" +process p4 -write "\x1b[24;1HW" +process p4 -expect-text 24 1 "W" +process p4 -screen-dump diff --git a/bin/varnishtest/tests/a00001.vtc b/bin/varnishtest/tests/a00001.vtc index dfca677..b6628d8 100644 --- a/bin/varnishtest/tests/a00001.vtc +++ b/bin/varnishtest/tests/a00001.vtc @@ -2,7 +2,7 @@ varnishtest "Test Teken terminal emulator" feature cmd "vttest --version 2>&1 | grep -q Usage" -process p4 {vttest} -start +process p4 {vttest} -ansi-response -start process p4 -expect-text 21 11 "Enter choice number (0 - 12):" process p4 -screen_dump diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index c182ed3..7ab093b 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -88,6 +88,7 @@ struct process { int nlin; int ncol; + int ansi_response; char **vram; teken_t tek[1]; }; @@ -174,10 +175,12 @@ term_respond(void *priv, const void *p, size_t l) CAST_OBJ_NOTNULL(pp, priv, PROCESS_MAGIC); vtc_dump(pp->vl, 4, "term_response", p, l); - r = write(pp->fd_term, p, l); - if (r != l) - vtc_fatal(pp->vl, "Could not write to process: %s", - strerror(errno)); + if (pp->ansi_response) { + r = write(pp->fd_term, p, l); + if (r != l) + vtc_fatal(pp->vl, "Could not write to process: %s", + strerror(errno)); + } } static void @@ -1010,6 +1013,10 @@ cmd_process(CMD_ARGS) process_wait(p); continue; } + if (!strcmp(*av, "-ansi-response")) { + p->ansi_response = 1; + continue; + } if (!strcmp(*av, "-expect-text")) { AN(av[1]); AN(av[2]); From phk at FreeBSD.org Tue Apr 17 22:33:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Apr 2018 22:33:14 +0000 (UTC) Subject: [master] 44ed24b Make the individual VFP->init()'s responsible for the objflags. Message-ID: <20180417223314.2EEFFA4303@lists.varnish-cache.org> commit 44ed24b635f5aff5a833c128edc75588d6c1de19 Author: Poul-Henning Kamp Date: Tue Apr 17 21:00:17 2018 +0000 Make the individual VFP->init()'s responsible for the objflags. diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 906c2ba..5411df2 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -156,6 +156,7 @@ vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) ALLOC_OBJ(vef, VEF_MAGIC); if (vef == NULL) return (VFP_ERROR); + vc->obj_flags |= OF_GZIPED | OF_CHGGZIP | OF_ESIPROC; vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E"); vef->vep = VEP_Init(vc, vc->req, vfp_vep_callback, vef); vef->ibuf_sz = cache_param->gzip_buffer; @@ -229,6 +230,7 @@ vfp_esi_init(struct vfp_ctx *vc, struct vfp_entry *vfe) ALLOC_OBJ(vef, VEF_MAGIC); if (vef == NULL) return (VFP_ERROR); + vc->obj_flags |= OF_ESIPROC; vef->vep = VEP_Init(vc, vc->req, NULL, NULL); vfe->priv1 = vef; return (VFP_OK); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 27e7e70..aacbc69 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -503,6 +503,7 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) static int vbf_figure_out_vfp(struct busyobj *bo) { + int is_gzip, is_gunzip; /* * The VCL variables beresp.do_g[un]zip tells us how we want the @@ -543,26 +544,26 @@ vbf_figure_out_vfp(struct busyobj *bo) if (!cache_param->http_gzip_support) bo->do_gzip = bo->do_gunzip = 0; - bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); - bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); - assert(bo->is_gzip == 0 || bo->is_gunzip == 0); + is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); + is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); + assert(is_gzip == 0 || is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ - if (bo->do_gunzip && !bo->is_gzip) + if (bo->do_gunzip && !is_gzip) bo->do_gunzip = 0; /* We wont gzip unless if it already is gzip'ed */ - if (bo->do_gzip && !bo->is_gunzip) + if (bo->do_gzip && !is_gunzip) bo->do_gzip = 0; /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); - if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) + if (bo->do_gunzip || (is_gzip && bo->do_esi)) if (VFP_Push(bo->vfc, &VFP_gunzip) == NULL) return (-1); - if (bo->do_esi && (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))) + if (bo->do_esi && (bo->do_gzip || (is_gzip && !bo->do_gunzip))) return (VFP_Push(bo->vfc, &VFP_esi_gzip) == NULL ? -1 : 0); if (bo->do_esi) @@ -571,7 +572,7 @@ vbf_figure_out_vfp(struct busyobj *bo) if (bo->do_gzip) return (VFP_Push(bo->vfc, &VFP_gzip) == NULL ? -1 : 0); - if (bo->is_gzip && !bo->do_gunzip) + if (is_gzip && !bo->do_gunzip) return (VFP_Push(bo->vfc, &VFP_testgunzip) == NULL ? -1 : 0); return (0); @@ -614,14 +615,10 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) return (F_STP_ERROR); } - if (bo->do_esi) - ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1); - - if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) - ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1); - - if (bo->do_gzip || bo->do_gunzip) - ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1); +#define OBJ_FLAG(U, l, v) \ + if (bo->vfc->obj_flags & OF_##U) \ + ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_##U, 1); +#include "tbl/obj_attr.h" if (!(bo->fetch_objcore->flags & OC_F_PASS) && http_IsStatus(bo->beresp, 200) && ( diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 4745836..f6dae16 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -82,6 +82,7 @@ struct vfp_ctx { struct vfp_entry_s vfp; struct vfp_entry *vfp_nxt; + unsigned obj_flags; }; enum vfp_status VFP_Suck(struct vfp_ctx *, void *p, ssize_t *lp); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 72a86dd..2c6e9ce 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -446,13 +446,18 @@ vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) if (http_GetHdr(vc->resp, H_Content_Encoding, NULL)) return (VFP_NULL); vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1); + vc->obj_flags |= OF_GZIPED | OF_CHGGZIP; } else { if (!http_HdrIs(vc->resp, H_Content_Encoding, "gzip")) return (VFP_NULL); - if (vfe->vfp == &VFP_gunzip) + if (vfe->vfp == &VFP_gunzip) { vg = VGZ_NewGunzip(vc->wrk->vsl, vfe->vfp->priv1); - else + vc->obj_flags &= ~OF_GZIPED; + vc->obj_flags |= OF_CHGGZIP; + } else { vg = VGZ_NewTestGunzip(vc->wrk->vsl, vfe->vfp->priv1); + vc->obj_flags |= OF_GZIPED; + } } if (vg == NULL) return (VFP_ERROR); diff --git a/include/tbl/bo_flags.h b/include/tbl/bo_flags.h index 5757699..88b8eeb 100644 --- a/include/tbl/bo_flags.h +++ b/include/tbl/bo_flags.h @@ -36,8 +36,6 @@ BO_FLAG(do_gunzip, 1, 1, "") BO_FLAG(do_stream, 1, 1, "") BO_FLAG(do_pass, 0, 0, "") BO_FLAG(uncacheable, 0, 0, "") -BO_FLAG(is_gzip, 0, 0, "") -BO_FLAG(is_gunzip, 0, 0, "") BO_FLAG(was_304, 1, 0, "") BO_FLAG(is_bgfetch, 0, 0, "") #undef BO_FLAG From phk at FreeBSD.org Tue Apr 17 22:33:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Apr 2018 22:33:14 +0000 (UTC) Subject: [master] 9f742a0 Let VPF's handle their own 206 policy. Message-ID: <20180417223314.41AF2A4306@lists.varnish-cache.org> commit 9f742a0a4558e0795390ad35f32c1e66ee3baeaa Author: Poul-Henning Kamp Date: Tue Apr 17 21:26:34 2018 +0000 Let VPF's handle their own 206 policy. diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 5411df2..acc1104 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -153,6 +153,11 @@ vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->req, HTTP_MAGIC); CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); + if (http_GetStatus(vc->resp) == 206) { + VSLb(vc->wrk->vsl, SLT_VCL_Error, + "Attempted ESI on partial (206) response"); + return (VFP_ERROR); + } ALLOC_OBJ(vef, VEF_MAGIC); if (vef == NULL) return (VFP_ERROR); @@ -227,6 +232,11 @@ vfp_esi_init(struct vfp_ctx *vc, struct vfp_entry *vfe) CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->req, HTTP_MAGIC); + if (http_GetStatus(vc->resp) == 206) { + VSLb(vc->wrk->vsl, SLT_VCL_Error, + "Attempted ESI on partial (206) response"); + return (VFP_ERROR); + } ALLOC_OBJ(vef, VEF_MAGIC); if (vef == NULL) return (VFP_ERROR); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index aacbc69..ee6091d 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -516,22 +516,8 @@ vbf_figure_out_vfp(struct busyobj *bo) * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip * - * On partial responses (206 on pass), we fail if do_esi is - * requested because it could leak partial esi-directives, and - * ignore gzipery, because it makes no sense. - * */ - if (http_GetStatus(bo->beresp) == 206) { - if (bo->do_esi) { - VSLb(bo->vsl, SLT_VCL_Error, - "beresp.do_esi on partial response"); - return (-1); - } - bo->do_gzip = bo->do_gunzip = 0; - return (0); - } - /* No body -> done */ if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) { diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 2c6e9ce..3694c0a 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -442,6 +442,13 @@ vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); + /* + * G(un)zip makes no sence on partial responses, but since + * it is an pure 1:1 transform, we can just ignore it. + */ + if (http_GetStatus(vc->resp) == 206) + return (VFP_NULL); + if (vfe->vfp == &VFP_gzip) { if (http_GetHdr(vc->resp, H_Content_Encoding, NULL)) return (VFP_NULL); diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index fe8cdb3..c77cca9 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -102,7 +102,7 @@ THR_SetName(const char *name) AZ(pthread_setspecific(name_key, name)); #if defined(HAVE_PTHREAD_SET_NAME_NP) - (void)pthread_set_name_np(pthread_self(), name); + pthread_set_name_np(pthread_self(), name); #elif defined(HAVE_PTHREAD_SETNAME_NP) #if defined(__APPLE__) (void)pthread_setname_np(name); From phk at FreeBSD.org Tue Apr 17 22:33:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Apr 2018 22:33:14 +0000 (UTC) Subject: [master] 662a552 Constify Message-ID: <20180417223314.596FEA430A@lists.varnish-cache.org> commit 662a552d6a2884d227ae513b3b0367710bb29ee3 Author: Poul-Henning Kamp Date: Tue Apr 17 22:28:32 2018 +0000 Constify diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index e1d835f..02d0420 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -388,7 +388,7 @@ struct busyobj { struct worker *wrk; struct vfp_ctx *vfc; - char *filter_list; + const char *filter_list; struct ws ws[1]; uintptr_t ws_bo; @@ -751,7 +751,7 @@ unsigned WS_Reserve(struct ws *ws, unsigned bytes); unsigned WS_ReserveLumps(struct ws *ws, size_t sz); void WS_MarkOverflow(struct ws *ws); void WS_Release(struct ws *ws, unsigned bytes); -void WS_ReleaseP(struct ws *ws, char *ptr); +void WS_ReleaseP(struct ws *ws, const char *ptr); void WS_Assert(const struct ws *ws); void WS_Reset(struct ws *ws, uintptr_t); void *WS_Alloc(struct ws *ws, unsigned bytes); diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index a58f657..3ec5ec0 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -275,7 +275,7 @@ WS_Release(struct ws *ws, unsigned bytes) } void -WS_ReleaseP(struct ws *ws, char *ptr) +WS_ReleaseP(struct ws *ws, const char *ptr) { WS_Assert(ws); DSL(DBG_WORKSPACE, 0, "WS_ReleaseP(%p, %p (%zd))", ws, ptr, ptr - ws->f); From phk at FreeBSD.org Tue Apr 17 22:33:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Apr 2018 22:33:14 +0000 (UTC) Subject: [master] 12e191c Switch to the list format for VFP's, now visible in beresp.filters Message-ID: <20180417223314.72089A430E@lists.varnish-cache.org> commit 12e191c27c84805269f43a39a77af8cf673721fe Author: Poul-Henning Kamp Date: Tue Apr 17 22:31:44 2018 +0000 Switch to the list format for VFP's, now visible in beresp.filters diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index ee6091d..424ccbc 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -36,6 +36,7 @@ #include "hash/hash_slinger.h" #include "storage/storage.h" #include "vcl.h" +#include "vct.h" #include "vtim.h" /*-------------------------------------------------------------------- @@ -500,9 +501,12 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) /*-------------------------------------------------------------------- */ -static int -vbf_figure_out_vfp(struct busyobj *bo) +static void +vbf_default_filter_list(const struct busyobj *bo, struct vsb *vsb) { + + int do_gzip = bo->do_gzip; + int do_gunzip = bo->do_gunzip; int is_gzip, is_gunzip; /* @@ -515,52 +519,125 @@ vbf_figure_out_vfp(struct busyobj *bo) * "Content-Encoding: gzip" --> object is gzip'ed. * no Content-Encoding --> object is not gzip'ed. * anything else --> do nothing wrt gzip - * */ /* No body -> done */ - if (bo->htc->body_status == BS_NONE || - bo->htc->content_length == 0) { - http_Unset(bo->beresp, H_Content_Encoding); - bo->do_gzip = bo->do_gunzip = 0; - bo->do_stream = 0; - return (0); - } + if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) + return; if (!cache_param->http_gzip_support) - bo->do_gzip = bo->do_gunzip = 0; + do_gzip = do_gunzip = 0; is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); assert(is_gzip == 0 || is_gunzip == 0); /* We won't gunzip unless it is gzip'ed */ - if (bo->do_gunzip && !is_gzip) - bo->do_gunzip = 0; + if (do_gunzip && !is_gzip) + do_gunzip = 0; /* We wont gzip unless if it already is gzip'ed */ - if (bo->do_gzip && !is_gunzip) - bo->do_gzip = 0; + if (do_gzip && !is_gunzip) + do_gzip = 0; /* But we can't do both at the same time */ - assert(bo->do_gzip == 0 || bo->do_gunzip == 0); + assert(do_gzip == 0 || do_gunzip == 0); - if (bo->do_gunzip || (is_gzip && bo->do_esi)) - if (VFP_Push(bo->vfc, &VFP_gunzip) == NULL) - return (-1); + if (do_gunzip || (is_gzip && bo->do_esi)) + VSB_cat(vsb, " gunzip"); - if (bo->do_esi && (bo->do_gzip || (is_gzip && !bo->do_gunzip))) - return (VFP_Push(bo->vfc, &VFP_esi_gzip) == NULL ? -1 : 0); + if (bo->do_esi && (do_gzip || (is_gzip && !do_gunzip))) { + VSB_cat(vsb, " esi_gzip"); + return; + } - if (bo->do_esi) - return (VFP_Push(bo->vfc, &VFP_esi) == NULL ? -1 : 0); + if (bo->do_esi) { + VSB_cat(vsb, " esi"); + return; + } + + if (do_gzip) + VSB_cat(vsb, " gzip"); - if (bo->do_gzip) - return (VFP_Push(bo->vfc, &VFP_gzip) == NULL ? -1 : 0); + if (is_gzip && !do_gunzip) + VSB_cat(vsb, " testgunzip"); +} - if (is_gzip && !bo->do_gunzip) - return (VFP_Push(bo->vfc, &VFP_testgunzip) == NULL ? -1 : 0); +const char * +VBF_Get_Filter_List(struct busyobj *bo) +{ + unsigned u; + struct vsb vsb[1]; + + u = WS_Reserve(bo->ws, 0); + if (u == 0) { + WS_Release(bo->ws, 0); + WS_MarkOverflow(bo->ws); + return (NULL); + } + AN(VSB_new(vsb, bo->ws->f, u, VSB_FIXEDLEN)); + vbf_default_filter_list(bo, vsb); + if (VSB_finish(vsb)) { + WS_Release(bo->ws, 0); + WS_MarkOverflow(bo->ws); + return (NULL); + } + WS_Release(bo->ws, VSB_len(vsb) + 1); + return (VSB_data(vsb) + 1); +} + +static const struct vfp *vfplist[] = { + &VFP_testgunzip, + &VFP_gunzip, + &VFP_gzip, + &VFP_esi, + &VFP_esi_gzip, + NULL, +}; + +static int +vbf_figure_out_vfp(struct busyobj *bo) +{ + const char *p, *q; + const struct vfp **vp; + int l; + + /* No body -> done */ + if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) { + http_Unset(bo->beresp, H_Content_Encoding); + bo->do_gzip = bo->do_gunzip = 0; + bo->do_stream = 0; + bo->filter_list = ""; + return (0); + } + + if (bo->filter_list == NULL) + bo->filter_list = VBF_Get_Filter_List(bo); + + VSLb(bo->vsl, SLT_Debug, "Filters <%s>", bo->filter_list); + + for (p = bo->filter_list; *p; p = q) { + if (vct_isspace(*p)) { + q = p + 1; + continue; + } + for (q = p; *q; q++) + if (vct_isspace(*q)) + break; + for(vp = vfplist; *vp != NULL; vp++) { + l = strlen((*vp)->name); + if (l != q - p) + continue; + if (!memcmp(p, (*vp)->name, l)) + break; + } + if (*vp == NULL) + return (VFP_Error(bo->vfc, + "Filter '%.*s' not found", (int)(q-p), p)); + if (VFP_Push(bo->vfc, *vp) == NULL) + return (-1); + } return (0); } diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index e05e87c..397703a 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -201,6 +201,7 @@ enum vbf_fetch_mode_e { }; void VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, struct objcore *oldoc, enum vbf_fetch_mode_e); +const char *VBF_Get_Filter_List(struct busyobj *); /* cache_fetch_proc.c */ void VFP_Init(void); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index bb55018..5e56b3a 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -36,7 +36,6 @@ #include "common/heritage.h" #include "vcl.h" -#include "vct.h" #include "cache_director.h" #include "vrt_obj.h" @@ -920,20 +919,12 @@ HTTP_VAR(beresp) VCL_STRING VRT_r_beresp_filters(VRT_CTX) { - const char *p; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); if (ctx->bo->filter_list != NULL) return(ctx->bo->filter_list); -#if 1 - p = "Not Yet"; -#else - p = VBF_Get_Filter_List(ctx->bo); - WS_Release(ctx->bo->ws, strlen(p) + 1); -#endif - while(vct_isspace(*p)) - p++; - return (p); + /* We do not set bo->filter_list yet, things might still change */ + return (VBF_Get_Filter_List(ctx->bo)); } From phk at FreeBSD.org Wed Apr 18 07:56:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 18 Apr 2018 07:56:11 +0000 (UTC) Subject: [master] 8d04330 Add SLT_Filters, shuffle code more naturally. Message-ID: <20180418075611.8B9B2B0110@lists.varnish-cache.org> commit 8d043303bf1ffaa1e7120e1bf733d0738dc51642 Author: Poul-Henning Kamp Date: Wed Apr 18 07:17:56 2018 +0000 Add SLT_Filters, shuffle code more naturally. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 424ccbc..1706c98 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -36,7 +36,6 @@ #include "hash/hash_slinger.h" #include "storage/storage.h" #include "vcl.h" -#include "vct.h" #include "vtim.h" /*-------------------------------------------------------------------- @@ -504,10 +503,10 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) static void vbf_default_filter_list(const struct busyobj *bo, struct vsb *vsb) { - + const char *p; int do_gzip = bo->do_gzip; int do_gunzip = bo->do_gunzip; - int is_gzip, is_gunzip; + int is_gzip = 0, is_gunzip = 0; /* * The VCL variables beresp.do_g[un]zip tells us how we want the @@ -528,9 +527,10 @@ vbf_default_filter_list(const struct busyobj *bo, struct vsb *vsb) if (!cache_param->http_gzip_support) do_gzip = do_gunzip = 0; - is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); - is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); - assert(is_gzip == 0 || is_gunzip == 0); + if (http_GetHdr(bo->beresp, H_Content_Encoding, &p)) + is_gzip = !strcasecmp(p, "gzip"); + else + is_gunzip = 1; /* We won't gunzip unless it is gzip'ed */ if (do_gunzip && !is_gzip) @@ -563,7 +563,6 @@ vbf_default_filter_list(const struct busyobj *bo, struct vsb *vsb) VSB_cat(vsb, " testgunzip"); } - const char * VBF_Get_Filter_List(struct busyobj *bo) { @@ -587,60 +586,6 @@ VBF_Get_Filter_List(struct busyobj *bo) return (VSB_data(vsb) + 1); } -static const struct vfp *vfplist[] = { - &VFP_testgunzip, - &VFP_gunzip, - &VFP_gzip, - &VFP_esi, - &VFP_esi_gzip, - NULL, -}; - -static int -vbf_figure_out_vfp(struct busyobj *bo) -{ - const char *p, *q; - const struct vfp **vp; - int l; - - /* No body -> done */ - if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) { - http_Unset(bo->beresp, H_Content_Encoding); - bo->do_gzip = bo->do_gunzip = 0; - bo->do_stream = 0; - bo->filter_list = ""; - return (0); - } - - if (bo->filter_list == NULL) - bo->filter_list = VBF_Get_Filter_List(bo); - - VSLb(bo->vsl, SLT_Debug, "Filters <%s>", bo->filter_list); - - for (p = bo->filter_list; *p; p = q) { - if (vct_isspace(*p)) { - q = p + 1; - continue; - } - for (q = p; *q; q++) - if (vct_isspace(*q)) - break; - for(vp = vfplist; *vp != NULL; vp++) { - l = strlen((*vp)->name); - if (l != q - p) - continue; - if (!memcmp(p, (*vp)->name, l)) - break; - } - if (*vp == NULL) - return (VFP_Error(bo->vfc, - "Filter '%.*s' not found", (int)(q-p), p)); - if (VFP_Push(bo->vfc, *vp) == NULL) - return (-1); - } - return (0); -} - static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { @@ -652,7 +597,17 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) assert(wrk->handling == VCL_RET_DELIVER); - if (vbf_figure_out_vfp(bo)) { + /* No body -> done */ + if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0) { + http_Unset(bo->beresp, H_Content_Encoding); + bo->do_gzip = bo->do_gunzip = 0; + bo->do_stream = 0; + bo->filter_list = ""; + } else if (bo->filter_list == NULL) { + bo->filter_list = VBF_Get_Filter_List(bo); + } + + if (VFP_FilterList(bo->vfc, bo->filter_list)) { (bo)->htc->doclose = SC_OVERLOAD; VDI_Finish((bo)->wrk, bo); return (F_STP_ERROR); diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 1413f1b..667c713 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -35,6 +35,8 @@ #include "cache_filter.h" #include "vcli_serve.h" +#include "vct.h" + static unsigned fetchfrag; /*-------------------------------------------------------------------- @@ -198,6 +200,7 @@ VFP_Suck(struct vfp_ctx *vc, void *p, ssize_t *lp) /*-------------------------------------------------------------------- */ + struct vfp_entry * VFP_Push(struct vfp_ctx *vc, const struct vfp *vfp) { @@ -221,6 +224,52 @@ VFP_Push(struct vfp_ctx *vc, const struct vfp *vfp) } /*-------------------------------------------------------------------- + */ + +static const struct vfp *vfplist[] = { + &VFP_testgunzip, + &VFP_gunzip, + &VFP_gzip, + &VFP_esi, + &VFP_esi_gzip, + NULL, +}; + +int +VFP_FilterList(struct vfp_ctx *vc, const char *fl) +{ + const char *p, *q; + const struct vfp **vp; + int l; + + VSLb(vc->wrk->vsl, SLT_Filters, "%s", fl); + + for (p = fl; *p; p = q) { + if (vct_isspace(*p)) { + q = p + 1; + continue; + } + for (q = p; *q; q++) + if (vct_isspace(*q)) + break; + for(vp = vfplist; *vp != NULL; vp++) { + l = strlen((*vp)->name); + if (l != q - p) + continue; + if (!memcmp(p, (*vp)->name, l)) + break; + } + if (*vp == NULL) + return (VFP_Error(vc, + "Filter '%.*s' not found", (int)(q-p), p)); + if (VFP_Push(vc, *vp) == NULL) + return (-1); + } + return (0); +} + + +/*-------------------------------------------------------------------- * Debugging aids */ diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index f6dae16..789a68c 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -88,6 +88,7 @@ struct vfp_ctx { enum vfp_status VFP_Suck(struct vfp_ctx *, void *p, ssize_t *lp); enum vfp_status VFP_Error(struct vfp_ctx *, const char *fmt, ...) v_printflike_(2, 3); +int VFP_FilterList(struct vfp_ctx *, const char *); /* Deliver processors ------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 5e56b3a..0ab6286 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -927,4 +927,3 @@ VRT_r_beresp_filters(VRT_CTX) /* We do not set bo->filter_list yet, things might still change */ return (VBF_Get_Filter_List(ctx->bo)); } - diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 89a422a..88106bc 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -573,6 +573,10 @@ SLTM(HitMiss, 0, "Hit for miss object in cache.", "\n" ) +SLTM(Filters, 0, "Body filters", + "List of filters applied to the body" +) + #undef NODEF_NOTICE #undef SLTM From phk at FreeBSD.org Wed Apr 18 07:56:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 18 Apr 2018 07:56:11 +0000 (UTC) Subject: [master] ebad1cf Make beresp.filters writable, and test it. Message-ID: <20180418075611.9EC5FB0113@lists.varnish-cache.org> commit ebad1cff8a221ef11010f25384b0eaa40c175899 Author: Poul-Henning Kamp Date: Wed Apr 18 07:54:56 2018 +0000 Make beresp.filters writable, and test it. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 1706c98..b316cec 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -540,9 +540,6 @@ vbf_default_filter_list(const struct busyobj *bo, struct vsb *vsb) if (do_gzip && !is_gunzip) do_gzip = 0; - /* But we can't do both at the same time */ - assert(do_gzip == 0 || do_gunzip == 0); - if (do_gunzip || (is_gzip && bo->do_esi)) VSB_cat(vsb, " gunzip"); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 0ab6286..52adc63 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -927,3 +927,21 @@ VRT_r_beresp_filters(VRT_CTX) /* We do not set bo->filter_list yet, things might still change */ return (VBF_Get_Filter_List(ctx->bo)); } + +VCL_VOID +VRT_l_beresp_filters(VRT_CTX, const char *str, ...) +{ + va_list ap; + const char *b; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + va_start(ap, str); + b = VRT_String(ctx->bo->ws, NULL, str, ap); + va_end(ap); + if (b == NULL) { + WS_MarkOverflow(ctx->bo->ws); + return; + } + ctx->bo->filter_list = b; +} diff --git a/bin/varnishtest/tests/g00003.vtc b/bin/varnishtest/tests/g00003.vtc index 1ba77ba..74bfa58 100644 --- a/bin/varnishtest/tests/g00003.vtc +++ b/bin/varnishtest/tests/g00003.vtc @@ -27,6 +27,12 @@ server s1 { expect req.http.accept-encoding == txresp -hdr "Vary: Accept-Encoding" \ -body "keep plain real" + + rxreq + expect req.url == "/filters" + expect req.http.accept-encoding == + txresp -bodylen 78 + } -start varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { @@ -36,6 +42,9 @@ varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { if (bereq.url == "/foobar") { set beresp.do_gzip = true; } + if (bereq.url == "/filters") { + set beresp.filters = "gzip gunzip gzip gunzip gzip"; + } } } -start @@ -62,6 +71,8 @@ client c1 { expect resp.bodylen == 43 } -run +varnish v1 -vsl_catchup + varnish v1 -expect n_gzip == 1 varnish v1 -expect n_gunzip == 2 varnish v1 -expect n_test_gunzip == 0 @@ -80,3 +91,13 @@ client c1 { expect resp.http.content-encoding == expect resp.body == "keep plain real" } -run + +varnish v1 -vsl_catchup + +client c1 { + txreq -url /filters + rxresp + expect resp.http.content-encoding == "gzip" + gunzip + expect resp.bodylen == 78 +} -run diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index f29a9ed..c1b765c 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -901,6 +901,8 @@ beresp.filters Readable from: vcl_backend_response + Writable from: vcl_backend_response + List of VFP filters the beresp.body will be pulled through. obj From nils.goroll at uplex.de Wed Apr 18 09:38:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 18 Apr 2018 09:38:10 +0000 (UTC) Subject: [master] 896060b fix status code typo Message-ID: <20180418093810.B6BADB1FE7@lists.varnish-cache.org> commit 896060bd164a1e73b519a61edfc5badfc9b72791 Author: Nils Goroll Date: Tue Apr 17 08:14:29 2018 +0200 fix status code typo Thank you to @martin-uplex for the review diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst index d8c9861..690f337 100644 --- a/doc/sphinx/users-guide/vcl-built-in-subs.rst +++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst @@ -440,7 +440,7 @@ based on the response status and the response headers ``Age``, * If present and valid, the value of the ``Age`` header is effectively deduced from all ttl calculations. -* For status codes 200, 203, 204, 300, 401, 304, 404, 410 and 414: +* For status codes 200, 203, 204, 300, 301, 304, 404, 410 and 414: * If ``Cache-Control`` contains an ``s-maxage`` or ``max-age`` field (in that order of preference), the ttl is set to the respective From nils.goroll at uplex.de Wed Apr 18 09:56:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 18 Apr 2018 09:56:09 +0000 (UTC) Subject: [master] 948de55 promote resp.body / beresp.body syntax a bit more Message-ID: <20180418095609.E2CD0B255B@lists.varnish-cache.org> commit 948de551d855879ccedcac52e15784b7f0abf7ff Author: Nils Goroll Date: Wed Apr 18 11:54:31 2018 +0200 promote resp.body / beresp.body syntax a bit more Should we plan to phase out synthetic() ? diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc index 42c96fe..29fbe9e 100644 --- a/bin/varnishtest/tests/b00017.vtc +++ b/bin/varnishtest/tests/b00017.vtc @@ -9,7 +9,7 @@ varnish v1 -vcl { } sub vcl_synth { - synthetic("Custom vcl_synth's synth output"); + set resp.body = "Custom vcl_synth's body"; return (deliver); } } -start @@ -18,8 +18,9 @@ client c1 { txreq -url "/" rxresp expect resp.status == 888 - expect resp.bodylen == 31 expect resp.http.connection != close + expect resp.bodylen == 23 + expect resp.body == "Custom vcl_synth's body" } -run varnish v1 -expect s_synth == 1 diff --git a/bin/varnishtest/tests/c00063.vtc b/bin/varnishtest/tests/c00063.vtc index b5c2cfe..5b0a61e 100644 --- a/bin/varnishtest/tests/c00063.vtc +++ b/bin/varnishtest/tests/c00063.vtc @@ -8,7 +8,7 @@ varnish v1 -vcl { set beresp.ttl = 1s; set beresp.grace = 3s; set beresp.http.foobar = "BLA" + bereq.xid; - synthetic(beresp.http.foobar); + set beresp.body = beresp.http.foobar; return (deliver); } } -start @@ -24,9 +24,11 @@ client c1 { txreq rxresp expect resp.http.foobar == "BLA1002" + expect resp.body == "BLA1002" delay 3 txreq rxresp expect resp.http.foobar == "BLA1004" + expect resp.body == "BLA1004" } -run diff --git a/bin/varnishtest/tests/c00068.vtc b/bin/varnishtest/tests/c00068.vtc index 313ca9b..88a47b6 100644 --- a/bin/varnishtest/tests/c00068.vtc +++ b/bin/varnishtest/tests/c00068.vtc @@ -37,7 +37,7 @@ varnish v1 -vcl+backend { return (restart); } else { set resp.http.restarts = req.restarts; - synthetic(req.restarts); + set resp.body = req.restarts; } } return (deliver); diff --git a/bin/varnishtest/tests/m00004.vtc b/bin/varnishtest/tests/m00004.vtc index f554a01..c418b8f 100644 --- a/bin/varnishtest/tests/m00004.vtc +++ b/bin/varnishtest/tests/m00004.vtc @@ -42,8 +42,8 @@ varnish v1 -vcl+backend { } sub vcl_synth { - synthetic("Response was served by " - + std.fileread("${tmpdir}/m00004_file_four")); + set resp.body = "Response was served by " + + std.fileread("${tmpdir}/m00004_file_four"); return(deliver); } } -start @@ -111,8 +111,8 @@ varnish v1 -vcl+backend { } sub vcl_synth { - synthetic("Response was served by " - + std.fileread("${tmpdir}/m00004_file_four")); + set resp.body = "Response was served by " + + std.fileread("${tmpdir}/m00004_file_four"); return(deliver); } } diff --git a/bin/varnishtest/tests/r01688.vtc b/bin/varnishtest/tests/r01688.vtc index fd0373b..bb4c92d 100644 --- a/bin/varnishtest/tests/r01688.vtc +++ b/bin/varnishtest/tests/r01688.vtc @@ -18,7 +18,8 @@ varnish v1 -vcl+backend { sub vcl_synth { if (resp.status == 998) { - synthetic("this is the body of an included synthetic response"); + set resp.body = "this is the body of an " + + "included synthetic response"; return(deliver); } } diff --git a/bin/varnishtest/tests/r01838.vtc b/bin/varnishtest/tests/r01838.vtc index ce1f20b..077851f 100644 --- a/bin/varnishtest/tests/r01838.vtc +++ b/bin/varnishtest/tests/r01838.vtc @@ -14,7 +14,7 @@ varnish v1 -vcl+backend { sub vcl_synth { if (resp.status == 998) { - synthetic("synthetic body"); + set resp.body = "synthetic body"; return (deliver); } } diff --git a/bin/varnishtest/tests/r02429.vtc b/bin/varnishtest/tests/r02429.vtc index 81e42f0..3eb1b6e 100644 --- a/bin/varnishtest/tests/r02429.vtc +++ b/bin/varnishtest/tests/r02429.vtc @@ -6,7 +6,7 @@ server s1 { varnish v1 -arg "-s Transient=file,${tmpdir}/_.file,10m" -vcl+backend { sub vcl_backend_error { - synthetic("foo"); + set beresp.body = "foo"; return (deliver); } } -start diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index d2263dc..9137757 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -467,8 +467,10 @@ hash_data(input) synthetic(STRING) ~~~~~~~~~~~~~~~~~ - Prepare a synthetic response body containing the *STRING*. Available in - ``vcl_synth`` and ``vcl_backend_error``. + Prepare a synthetic response body containing the *STRING*. Available + in ``vcl_synth`` and ``vcl_backend_error``. + + Identical to ``set resp.body`` / ``set beresp.body``. .. list above comes from struct action_table[] in vcc_action.c. From phk at FreeBSD.org Wed Apr 18 15:38:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 18 Apr 2018 15:38:14 +0000 (UTC) Subject: [master] b59a733 Reset filter_list on retry. Message-ID: <20180418153814.9BF87B89EF@lists.varnish-cache.org> commit b59a73375f20adff3a61b6e1a16be0d673254753 Author: Poul-Henning Kamp Date: Wed Apr 18 15:37:11 2018 +0000 Reset filter_list on retry. Spotted by: slink diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index b316cec..35e7d25 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -240,6 +240,7 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo) bo->storage = NULL; bo->do_esi = 0; bo->do_stream = 1; + bo->filter_list = NULL; // XXX: BereqEnd + BereqAcct ? VSL_ChgId(bo->vsl, "bereq", "retry", VXID_Get(wrk, VSL_BACKENDMARKER)); From nils.goroll at uplex.de Thu Apr 19 06:43:14 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 19 Apr 2018 06:43:14 +0000 (UTC) Subject: [master] 4ac5e49 don't panic for ws overflow in VBF_Get_Filter_List() Message-ID: <20180419064314.67CCFA2F61@lists.varnish-cache.org> commit 4ac5e49f558096a68861ef2bfd975727ec364922 Author: Nils Goroll Date: Thu Apr 19 08:41:50 2018 +0200 don't panic for ws overflow in VBF_Get_Filter_List() diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 35e7d25..07259f6 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -605,7 +605,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) bo->filter_list = VBF_Get_Filter_List(bo); } - if (VFP_FilterList(bo->vfc, bo->filter_list)) { + if (bo->filter_list == NULL || + VFP_FilterList(bo->vfc, bo->filter_list)) { (bo)->htc->doclose = SC_OVERLOAD; VDI_Finish((bo)->wrk, bo); return (F_STP_ERROR); From nils.goroll at uplex.de Thu Apr 19 07:44:12 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 19 Apr 2018 07:44:12 +0000 (UTC) Subject: [master] 05c74b9 vtc sweep through tight backend workspace conditions Message-ID: <20180419074412.D5748A41BF@lists.varnish-cache.org> commit 05c74b992b3ab5a2595afc1351c423515fe8f6e6 Author: Nils Goroll Date: Thu Apr 19 09:40:23 2018 +0200 vtc sweep through tight backend workspace conditions Ref: #2645 diff --git a/bin/varnishtest/tests/r02645.vtc b/bin/varnishtest/tests/r02645.vtc new file mode 100644 index 0000000..d665737 --- /dev/null +++ b/bin/varnishtest/tests/r02645.vtc @@ -0,0 +1,25 @@ +varnishtest "sweep through tight backend workspace conditions" + +server s1 -repeat 100 { + rxreq + send "HTTP/1.1 200 OK\r\nTransfer-encoding: chunked\r\n\r\n00000004\r\n1234\r\n00000000\r\n\r\n" +} -start + +varnish v1 -vcl+backend { + import vtc; + import std; + sub vcl_recv { + return (pass); + } + sub vcl_backend_fetch { + vtc.workspace_alloc(backend, -4 * + (std.integer(bereq.xid, 1002) - 1000) / 2); + } +} -start + +client c1 -repeat 100 { + txreq -url "/" + # some responses will fail (503), some won't. All we care + # about here is the fact that we don't panic + rxresp +} -run From phk at FreeBSD.org Thu Apr 19 07:45:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 19 Apr 2018 07:45:13 +0000 (UTC) Subject: [master] af71e47 Add an internal API to register/remove a VFP Message-ID: <20180419074513.900A1A425A@lists.varnish-cache.org> commit af71e4796a6ab513c3d0820abbb575dbe0d7ae25 Author: Poul-Henning Kamp Date: Wed Apr 18 20:39:15 2018 +0000 Add an internal API to register/remove a VFP diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 667c713..5c5e94a 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -226,21 +226,52 @@ VFP_Push(struct vfp_ctx *vc, const struct vfp *vfp) /*-------------------------------------------------------------------- */ -static const struct vfp *vfplist[] = { - &VFP_testgunzip, - &VFP_gunzip, - &VFP_gzip, - &VFP_esi, - &VFP_esi_gzip, - NULL, +struct vfp_filter { + unsigned magic; +#define VFP_FILTER_MAGIC 0xd40894e9 + const struct vfp *filter; + int nlen; + VTAILQ_ENTRY(vfp_filter) list; }; +static VTAILQ_HEAD(,vfp_filter) vfp_filters = + VTAILQ_HEAD_INITIALIZER(vfp_filters); + +void +VFP_AddFilter(const struct vfp *filter) +{ + struct vfp_filter *vp; + + VTAILQ_FOREACH(vp, &vfp_filters, list) { + assert(vp->filter != filter); + assert(strcasecmp(vp->filter->name, filter->name)); + } + ALLOC_OBJ(vp, VFP_FILTER_MAGIC); + AN(vp); + vp->filter = filter; + vp->nlen = strlen(filter->name); + VTAILQ_INSERT_TAIL(&vfp_filters, vp, list); +} + +void +VFP_RemoveFilter(const struct vfp *filter) +{ + struct vfp_filter *vp; + + VTAILQ_FOREACH(vp, &vfp_filters, list) { + if (vp->filter == filter) + break; + } + AN(vp); + VTAILQ_REMOVE(&vfp_filters, vp, list); + FREE_OBJ(vp); +} + int VFP_FilterList(struct vfp_ctx *vc, const char *fl) { const char *p, *q; - const struct vfp **vp; - int l; + const struct vfp_filter *vp; VSLb(vc->wrk->vsl, SLT_Filters, "%s", fl); @@ -252,23 +283,21 @@ VFP_FilterList(struct vfp_ctx *vc, const char *fl) for (q = p; *q; q++) if (vct_isspace(*q)) break; - for(vp = vfplist; *vp != NULL; vp++) { - l = strlen((*vp)->name); - if (l != q - p) + VTAILQ_FOREACH(vp, &vfp_filters, list) { + if (vp->nlen != q - p) continue; - if (!memcmp(p, (*vp)->name, l)) + if (!memcmp(p, vp->filter->name, vp->nlen)) break; } - if (*vp == NULL) + if (vp == NULL) return (VFP_Error(vc, "Filter '%.*s' not found", (int)(q-p), p)); - if (VFP_Push(vc, *vp) == NULL) + if (VFP_Push(vc, vp->filter) == NULL) return (-1); } return (0); } - /*-------------------------------------------------------------------- * Debugging aids */ @@ -295,4 +324,9 @@ VFP_Init(void) { CLI_AddFuncs(debug_cmds); + VFP_AddFilter(&VFP_testgunzip); + VFP_AddFilter(&VFP_gunzip); + VFP_AddFilter(&VFP_gzip); + VFP_AddFilter(&VFP_esi); + VFP_AddFilter(&VFP_esi_gzip); } diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 789a68c..924782e 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -89,6 +89,8 @@ enum vfp_status VFP_Suck(struct vfp_ctx *, void *p, ssize_t *lp); enum vfp_status VFP_Error(struct vfp_ctx *, const char *fmt, ...) v_printflike_(2, 3); int VFP_FilterList(struct vfp_ctx *, const char *); +void VFP_AddFilter(const struct vfp *); +void VFP_RemoveFilter(const struct vfp *); /* Deliver processors ------------------------------------------------*/ From phk at FreeBSD.org Thu Apr 19 07:45:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 19 Apr 2018 07:45:13 +0000 (UTC) Subject: [master] 4687245 cache_vcl.c was getting too long, split along lines of "maintain list of VCLs" vs. "Do things with VCLs on list" Message-ID: <20180419074513.AB134A425E@lists.varnish-cache.org> commit 468724563de4de1cad2a876c24bc1f5944ed6160 Author: Poul-Henning Kamp Date: Thu Apr 19 07:43:55 2018 +0000 cache_vcl.c was getting too long, split along lines of "maintain list of VCLs" vs. "Do things with VCLs on list" diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 1e8640e..9329f36 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -44,6 +44,7 @@ varnishd_SOURCES = \ cache/cache_tcp_pool.c \ cache/cache_vary.c \ cache/cache_vcl.c \ + cache/cache_vcl_vrt.c \ cache/cache_vrt.c \ cache/cache_vrt_priv.c \ cache/cache_vrt_re.c \ @@ -125,6 +126,7 @@ noinst_HEADERS = \ cache/cache_pool.h \ cache/cache_tcp_pool.h \ cache/cache_transport.h \ + cache/cache_vcl.h \ cache/cache_vgz.h \ common/heritage.h \ common/vsmw.h \ diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 30adf54..abc519f 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -42,49 +42,15 @@ #include "vcl.h" #include "cache_director.h" -#include "cache_backend.h" +#include "cache_vcl.h" #include "vcli_serve.h" -static const char * const VCL_TEMP_INIT = "init"; -static const char * const VCL_TEMP_COLD = "cold"; -static const char * const VCL_TEMP_WARM = "warm"; -static const char * const VCL_TEMP_BUSY = "busy"; -static const char * const VCL_TEMP_COOLING = "cooling"; -static const char * const VCL_TEMP_LABEL = "label"; - -/* - * NB: The COOLING temperature is neither COLD nor WARM. - * And LABEL is not a temperature, it's a different kind of VCL. - */ -#define VCL_WARM(v) ((v)->temp == VCL_TEMP_WARM || (v)->temp == VCL_TEMP_BUSY) -#define VCL_COLD(v) ((v)->temp == VCL_TEMP_INIT || (v)->temp == VCL_TEMP_COLD) - -struct vcl { - unsigned magic; -#define VCL_MAGIC 0x214188f2 - VTAILQ_ENTRY(vcl) list; - void *dlh; - const struct VCL_conf *conf; - char state[8]; - char *loaded_name; - unsigned busy; - unsigned discard; - const char *temp; - pthread_rwlock_t temp_rwl; - VTAILQ_HEAD(,director) director_list; - VTAILQ_HEAD(,vclref) ref_list; - int nrefs; - struct vcl *label; - int nlabels; -}; - -struct vclref { - unsigned magic; -#define VCLREF_MAGIC 0x47fb6848 - const struct vcl *vcl; - VTAILQ_ENTRY(vclref) list; - char desc[32]; -}; +const char * const VCL_TEMP_INIT = "init"; +const char * const VCL_TEMP_COLD = "cold"; +const char * const VCL_TEMP_WARM = "warm"; +const char * const VCL_TEMP_BUSY = "busy"; +const char * const VCL_TEMP_COOLING = "cooling"; +const char * const VCL_TEMP_LABEL = "label"; /* * XXX: Presently all modifications to this list happen from the @@ -93,8 +59,8 @@ struct vclref { static VTAILQ_HEAD(, vcl) vcl_head = VTAILQ_HEAD_INITIALIZER(vcl_head); -static struct lock vcl_mtx; -static struct vcl *vcl_active; /* protected by vcl_mtx */ +struct lock vcl_mtx; +struct vcl *vcl_active; /* protected by vcl_mtx */ static struct vrt_ctx ctx_cli; static unsigned handling_cli; @@ -171,7 +137,7 @@ vcl_send_event(VRT_CTX, enum vcl_event_e ev) /*--------------------------------------------------------------------*/ -static struct vcl * +struct vcl * vcl_find(const char *name) { struct vcl *vcl; @@ -225,37 +191,7 @@ VCL_Panic(struct vsb *vsb, const struct vcl *vcl) /*--------------------------------------------------------------------*/ -const char * -VCL_Return_Name(unsigned r) -{ - - switch (r) { -#define VCL_RET_MAC(l, U, B) \ - case VCL_RET_##U: \ - return(#l); -#include "tbl/vcl_returns.h" - default: - return (NULL); - } -} - -const char * -VCL_Method_Name(unsigned m) -{ - - switch (m) { -#define VCL_MET_MAC(func, upper, typ, bitmap) \ - case VCL_MET_##upper: \ - return (#upper); -#include "tbl/vcl_returns.h" - default: - return (NULL); - } -} - -/*--------------------------------------------------------------------*/ - -static void +void vcl_get(struct vcl **vcc, struct vcl *vcl) { AN(vcc); @@ -281,114 +217,8 @@ vcl_get(struct vcl **vcc, struct vcl *vcl) AZ(errno=pthread_rwlock_unlock(&(*vcc)->temp_rwl)); } -void -VCL_Refresh(struct vcl **vcc) -{ - if (*vcc == vcl_active) - return; - if (*vcc != NULL) - VCL_Rel(vcc); /* XXX: optimize locking */ - - while (vcl_active == NULL) - (void)usleep(100000); - - vcl_get(vcc, NULL); -} - -void -VCL_Ref(struct vcl *vcl) -{ - - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); - assert(!VCL_COLD(vcl)); - AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - Lck_Lock(&vcl_mtx); - assert(vcl->busy > 0); - vcl->busy++; - Lck_Unlock(&vcl_mtx); -} - -void -VCL_Rel(struct vcl **vcc) -{ - struct vcl *vcl; - - AN(*vcc); - vcl = *vcc; - *vcc = NULL; - - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - Lck_Lock(&vcl_mtx); - assert(vcl->busy > 0); - vcl->busy--; - /* - * We do not garbage collect discarded VCL's here, that happens - * in VCL_Poll() which is called from the CLI thread. - */ - Lck_Unlock(&vcl_mtx); -} - /*--------------------------------------------------------------------*/ -int -VCL_AddDirector(struct vcl *vcl, struct director *d, const char *vcl_name) -{ - struct vsb *vsb; - - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - AN(d->destroy); - - vsb = VSB_new_auto(); - AN(vsb); - VSB_printf(vsb, "%s.%s", VCL_Name(vcl), vcl_name); - AZ(VSB_finish(vsb)); - REPLACE((d->display_name), VSB_data(vsb)); - VSB_destroy(&vsb); - - AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); - if (vcl->temp == VCL_TEMP_COOLING) { - AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - return (1); - } - - Lck_Lock(&vcl_mtx); - VTAILQ_INSERT_TAIL(&vcl->director_list, d, vcl_list); - d->vcl = vcl; - Lck_Unlock(&vcl_mtx); - - if (VCL_WARM(vcl)) - /* Only when adding backend to already warm VCL */ - VDI_Event(d, VCL_EVENT_WARM); - else if (vcl->temp != VCL_TEMP_INIT) - WRONG("Dynamic Backends can only be added to warm VCLs"); - AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - - return (0); -} - -void -VCL_DelDirector(struct director *d) -{ - struct vcl *vcl; - - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - vcl = d->vcl; - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - Lck_Lock(&vcl_mtx); - VTAILQ_REMOVE(&vcl->director_list, d, vcl_list); - Lck_Unlock(&vcl_mtx); - - AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); - if (VCL_WARM(vcl)) - VDI_Event(d, VCL_EVENT_COLD); - AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - AN(d->destroy); - REPLACE(d->display_name, NULL); - d->destroy(d); -} - static int vcl_iterdir(struct cli *cli, const char *pat, const struct vcl *vcl, vcl_be_func *func, void *priv) @@ -572,153 +402,6 @@ VCL_TestLoad(const char *fn) /*--------------------------------------------------------------------*/ -struct director * -VCL_DefaultDirector(const struct vcl *vcl) -{ - - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC); - return (*vcl->conf->default_director); -} - -const char * -VCL_Name(const struct vcl *vcl) -{ - - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - return (vcl->loaded_name); -} - -const struct vrt_backend_probe * -VCL_DefaultProbe(const struct vcl *vcl) -{ - - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC); - return (vcl->conf->default_probe); -} - -/*-------------------------------------------------------------------- - * VRT apis relating to VCL's as VCLS. - */ - -void -VRT_count(VRT_CTX, unsigned u) -{ - - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); - assert(u < ctx->vcl->conf->nref); - if (ctx->vsl != NULL) - VSLb(ctx->vsl, SLT_VCL_trace, "%s %u %u.%u.%u", - ctx->vcl->loaded_name, u, ctx->vcl->conf->ref[u].source, - ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); - else - VSL(SLT_VCL_trace, 0, "%s %u %u.%u.%u", - ctx->vcl->loaded_name, u, ctx->vcl->conf->ref[u].source, - ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); -} - -VCL_VCL -VRT_vcl_get(VRT_CTX, const char *name) -{ - VCL_VCL vcl; - - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - vcl = vcl_find(name); - AN(vcl); - Lck_Lock(&vcl_mtx); - vcl->nrefs++; - Lck_Unlock(&vcl_mtx); - return (vcl); -} - -void -VRT_vcl_rel(VRT_CTX, VCL_VCL vcl) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(vcl); - Lck_Lock(&vcl_mtx); - vcl->nrefs--; - Lck_Unlock(&vcl_mtx); -} - -void -VRT_vcl_select(VRT_CTX, VCL_VCL vcl) -{ - struct req *req = ctx->req; - - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - VCL_Rel(&req->vcl); - vcl_get(&req->vcl, vcl); - /* XXX: better logging */ - VSLb(ctx->req->vsl, SLT_Debug, "Now using %s VCL", vcl->loaded_name); -} - -struct vclref * -VRT_ref_vcl(VRT_CTX, const char *desc) -{ - struct vcl *vcl; - struct vclref* ref; - - ASSERT_CLI(); - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(desc); - AN(*desc); - - vcl = ctx->vcl; - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - assert(VCL_WARM(vcl)); - - ALLOC_OBJ(ref, VCLREF_MAGIC); - AN(ref); - ref->vcl = vcl; - bprintf(ref->desc, "%s", desc); - - Lck_Lock(&vcl_mtx); - VTAILQ_INSERT_TAIL(&vcl->ref_list, ref, list); - vcl->nrefs++; - Lck_Unlock(&vcl_mtx); - - return (ref); -} - -void -VRT_rel_vcl(VRT_CTX, struct vclref **refp) -{ - struct vcl *vcl; - struct vclref *ref; - - AN(refp); - ref = *refp; - *refp = NULL; - - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ref, VCLREF_MAGIC); - - vcl = ctx->vcl; - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - assert(vcl == ref->vcl); - - /* NB: A VCL may be released by a VMOD at any time, but it must happen - * after a warmup and before the end of a cooldown. The release may or - * may not happen while the same thread holds the temperature lock, so - * instead we check that all references are gone in VCL_Nuke. - */ - - Lck_Lock(&vcl_mtx); - assert(!VTAILQ_EMPTY(&vcl->ref_list)); - VTAILQ_REMOVE(&vcl->ref_list, ref, list); - vcl->nrefs--; - /* No garbage collection here, for the same reasons as in VCL_Rel. */ - Lck_Unlock(&vcl_mtx); - - FREE_OBJ(ref); -} - -/*--------------------------------------------------------------------*/ - static void vcl_print_refs(VRT_CTX) { @@ -1100,92 +783,6 @@ vcl_cli_show(struct cli *cli, const char * const *av, void *priv) } } -/*-------------------------------------------------------------------- - * Method functions to call into VCL programs. - * - * Either the request or busyobject must be specified, but not both. - * The workspace argument is where random VCL stuff gets space from. - */ - -static void -vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, - void *specific, unsigned method, vcl_func_f *func) -{ - uintptr_t aws; - struct vsl_log *vsl = NULL; - struct vrt_ctx ctx; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - INIT_OBJ(&ctx, VRT_CTX_MAGIC); - if (req != NULL) { - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(req->vcl, VCL_MAGIC); - vsl = req->vsl; - ctx.vcl = req->vcl; - ctx.http_req = req->http; - ctx.http_req_top = req->top->http; - ctx.http_resp = req->resp; - ctx.req = req; - ctx.sp = req->sp; - ctx.now = req->t_prev; - ctx.ws = req->ws; - } - if (bo != NULL) { - if (req) - assert(method == VCL_MET_PIPE); - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(bo->vcl, VCL_MAGIC); - vsl = bo->vsl; - ctx.vcl = bo->vcl; - ctx.http_bereq = bo->bereq; - ctx.http_beresp = bo->beresp; - ctx.bo = bo; - ctx.sp = bo->sp; - ctx.now = bo->t_prev; - ctx.ws = bo->ws; - } - assert(ctx.now != 0); - ctx.syntax = ctx.vcl->conf->syntax; - ctx.vsl = vsl; - ctx.specific = specific; - ctx.method = method; - wrk->handling = 0; - ctx.handling = &wrk->handling; - aws = WS_Snapshot(wrk->aws); - wrk->cur_method = method; - wrk->seen_methods |= method; - AN(vsl); - VSLb(vsl, SLT_VCL_call, "%s", VCL_Method_Name(method)); - func(&ctx); - VSLb(vsl, SLT_VCL_return, "%s", VCL_Return_Name(wrk->handling)); - wrk->cur_method |= 1; // Magic marker - if (wrk->handling == VCL_RET_FAIL) - wrk->stats->vcl_fail++; - - /* - * VCL/Vmods are not allowed to make permanent allocations from - * wrk->aws, but they can reserve and return from it. - */ - assert(aws == WS_Snapshot(wrk->aws)); -} - -#define VCL_MET_MAC(func, upper, typ, bitmap) \ -void \ -VCL_##func##_method(struct vcl *vcl, struct worker *wrk, \ - struct req *req, struct busyobj *bo, void *specific) \ -{ \ - \ - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); \ - CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC); \ - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); \ - vcl_call_method(wrk, req, bo, specific, \ - VCL_MET_ ## upper, vcl->conf->func##_func); \ - AN((1U << wrk->handling) & bitmap); \ -} - -#include "tbl/vcl_returns.h" - /*--------------------------------------------------------------------*/ static struct cli_proto vcl_cmds[] = { diff --git a/bin/varnishd/cache/cache_vcl.h b/bin/varnishd/cache/cache_vcl.h new file mode 100644 index 0000000..87433af --- /dev/null +++ b/bin/varnishd/cache/cache_vcl.h @@ -0,0 +1,77 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2016 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +struct vcl { + unsigned magic; +#define VCL_MAGIC 0x214188f2 + VTAILQ_ENTRY(vcl) list; + void *dlh; + const struct VCL_conf *conf; + char state[8]; + char *loaded_name; + unsigned busy; + unsigned discard; + const char *temp; + pthread_rwlock_t temp_rwl; + VTAILQ_HEAD(,director) director_list; + VTAILQ_HEAD(,vclref) ref_list; + int nrefs; + struct vcl *label; + int nlabels; +}; + +struct vclref { + unsigned magic; +#define VCLREF_MAGIC 0x47fb6848 + const struct vcl *vcl; + VTAILQ_ENTRY(vclref) list; + char desc[32]; +}; + +extern struct lock vcl_mtx; +extern struct vcl *vcl_active; /* protected by vcl_mtx */ +struct vcl *vcl_find(const char *); +void vcl_get(struct vcl **, struct vcl *); + +extern const char * const VCL_TEMP_INIT; +extern const char * const VCL_TEMP_COLD; +extern const char * const VCL_TEMP_WARM; +extern const char * const VCL_TEMP_BUSY; +extern const char * const VCL_TEMP_COOLING; +extern const char * const VCL_TEMP_LABEL; + +/* + * NB: The COOLING temperature is neither COLD nor WARM. + * And LABEL is not a temperature, it's a different kind of VCL. + */ +#define VCL_WARM(v) ((v)->temp == VCL_TEMP_WARM || (v)->temp == VCL_TEMP_BUSY) +#define VCL_COLD(v) ((v)->temp == VCL_TEMP_INIT || (v)->temp == VCL_TEMP_COLD) + + diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c new file mode 100644 index 0000000..484c14b --- /dev/null +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -0,0 +1,415 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2016 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include "config.h" + +#include +#include +#include + +#include "cache_varnishd.h" + +#include "vcl.h" + +#include "cache_director.h" +#include "cache_vcl.h" + +/*--------------------------------------------------------------------*/ + +const char * +VCL_Return_Name(unsigned r) +{ + + switch (r) { +#define VCL_RET_MAC(l, U, B) \ + case VCL_RET_##U: \ + return(#l); +#include "tbl/vcl_returns.h" + default: + return (NULL); + } +} + +const char * +VCL_Method_Name(unsigned m) +{ + + switch (m) { +#define VCL_MET_MAC(func, upper, typ, bitmap) \ + case VCL_MET_##upper: \ + return (#upper); +#include "tbl/vcl_returns.h" + default: + return (NULL); + } +} + +/*--------------------------------------------------------------------*/ + +void +VCL_Refresh(struct vcl **vcc) +{ + if (*vcc == vcl_active) + return; + if (*vcc != NULL) + VCL_Rel(vcc); /* XXX: optimize locking */ + + while (vcl_active == NULL) + (void)usleep(100000); + + vcl_get(vcc, NULL); +} + +void +VCL_Ref(struct vcl *vcl) +{ + + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); + assert(!VCL_COLD(vcl)); + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); + Lck_Lock(&vcl_mtx); + assert(vcl->busy > 0); + vcl->busy++; + Lck_Unlock(&vcl_mtx); +} + +void +VCL_Rel(struct vcl **vcc) +{ + struct vcl *vcl; + + AN(*vcc); + vcl = *vcc; + *vcc = NULL; + + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + Lck_Lock(&vcl_mtx); + assert(vcl->busy > 0); + vcl->busy--; + /* + * We do not garbage collect discarded VCL's here, that happens + * in VCL_Poll() which is called from the CLI thread. + */ + Lck_Unlock(&vcl_mtx); +} + +/*--------------------------------------------------------------------*/ + +int +VCL_AddDirector(struct vcl *vcl, struct director *d, const char *vcl_name) +{ + struct vsb *vsb; + + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + AN(d->destroy); + + vsb = VSB_new_auto(); + AN(vsb); + VSB_printf(vsb, "%s.%s", VCL_Name(vcl), vcl_name); + AZ(VSB_finish(vsb)); + REPLACE((d->display_name), VSB_data(vsb)); + VSB_destroy(&vsb); + + AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); + if (vcl->temp == VCL_TEMP_COOLING) { + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); + return (1); + } + + Lck_Lock(&vcl_mtx); + VTAILQ_INSERT_TAIL(&vcl->director_list, d, vcl_list); + d->vcl = vcl; + Lck_Unlock(&vcl_mtx); + + if (VCL_WARM(vcl)) + /* Only when adding backend to already warm VCL */ + VDI_Event(d, VCL_EVENT_WARM); + else if (vcl->temp != VCL_TEMP_INIT) + WRONG("Dynamic Backends can only be added to warm VCLs"); + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); + + return (0); +} + +void +VCL_DelDirector(struct director *d) +{ + struct vcl *vcl; + + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + vcl = d->vcl; + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + Lck_Lock(&vcl_mtx); + VTAILQ_REMOVE(&vcl->director_list, d, vcl_list); + Lck_Unlock(&vcl_mtx); + + AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); + if (VCL_WARM(vcl)) + VDI_Event(d, VCL_EVENT_COLD); + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); + AN(d->destroy); + REPLACE(d->display_name, NULL); + d->destroy(d); +} + +/*--------------------------------------------------------------------*/ + +struct director * +VCL_DefaultDirector(const struct vcl *vcl) +{ + + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC); + return (*vcl->conf->default_director); +} + +const char * +VCL_Name(const struct vcl *vcl) +{ + + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + return (vcl->loaded_name); +} + +const struct vrt_backend_probe * +VCL_DefaultProbe(const struct vcl *vcl) +{ + + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC); + return (vcl->conf->default_probe); +} + +/*-------------------------------------------------------------------- + * VRT apis relating to VCL's as VCLS. + */ + +void +VRT_count(VRT_CTX, unsigned u) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); + CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); + assert(u < ctx->vcl->conf->nref); + if (ctx->vsl != NULL) + VSLb(ctx->vsl, SLT_VCL_trace, "%s %u %u.%u.%u", + ctx->vcl->loaded_name, u, ctx->vcl->conf->ref[u].source, + ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); + else + VSL(SLT_VCL_trace, 0, "%s %u %u.%u.%u", + ctx->vcl->loaded_name, u, ctx->vcl->conf->ref[u].source, + ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); +} + +VCL_VCL +VRT_vcl_get(VRT_CTX, const char *name) +{ + VCL_VCL vcl; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + vcl = vcl_find(name); + AN(vcl); + Lck_Lock(&vcl_mtx); + vcl->nrefs++; + Lck_Unlock(&vcl_mtx); + return (vcl); +} + +void +VRT_vcl_rel(VRT_CTX, VCL_VCL vcl) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(vcl); + Lck_Lock(&vcl_mtx); + vcl->nrefs--; + Lck_Unlock(&vcl_mtx); +} + +void +VRT_vcl_select(VRT_CTX, VCL_VCL vcl) +{ + struct req *req = ctx->req; + + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + VCL_Rel(&req->vcl); + vcl_get(&req->vcl, vcl); + /* XXX: better logging */ + VSLb(ctx->req->vsl, SLT_Debug, "Now using %s VCL", vcl->loaded_name); +} + +struct vclref * +VRT_ref_vcl(VRT_CTX, const char *desc) +{ + struct vcl *vcl; + struct vclref* ref; + + ASSERT_CLI(); + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(desc); + AN(*desc); + + vcl = ctx->vcl; + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + assert(VCL_WARM(vcl)); + + ALLOC_OBJ(ref, VCLREF_MAGIC); + AN(ref); + ref->vcl = vcl; + bprintf(ref->desc, "%s", desc); + + Lck_Lock(&vcl_mtx); + VTAILQ_INSERT_TAIL(&vcl->ref_list, ref, list); + vcl->nrefs++; + Lck_Unlock(&vcl_mtx); + + return (ref); +} + +void +VRT_rel_vcl(VRT_CTX, struct vclref **refp) +{ + struct vcl *vcl; + struct vclref *ref; + + AN(refp); + ref = *refp; + *refp = NULL; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ref, VCLREF_MAGIC); + + vcl = ctx->vcl; + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + assert(vcl == ref->vcl); + + /* NB: A VCL may be released by a VMOD at any time, but it must happen + * after a warmup and before the end of a cooldown. The release may or + * may not happen while the same thread holds the temperature lock, so + * instead we check that all references are gone in VCL_Nuke. + */ + + Lck_Lock(&vcl_mtx); + assert(!VTAILQ_EMPTY(&vcl->ref_list)); + VTAILQ_REMOVE(&vcl->ref_list, ref, list); + vcl->nrefs--; + /* No garbage collection here, for the same reasons as in VCL_Rel. */ + Lck_Unlock(&vcl_mtx); + + FREE_OBJ(ref); +} + +/*-------------------------------------------------------------------- + * Method functions to call into VCL programs. + * + * Either the request or busyobject must be specified, but not both. + * The workspace argument is where random VCL stuff gets space from. + */ + +static void +vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, + void *specific, unsigned method, vcl_func_f *func) +{ + uintptr_t aws; + struct vsl_log *vsl = NULL; + struct vrt_ctx ctx; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + INIT_OBJ(&ctx, VRT_CTX_MAGIC); + if (req != NULL) { + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req->vcl, VCL_MAGIC); + vsl = req->vsl; + ctx.vcl = req->vcl; + ctx.http_req = req->http; + ctx.http_req_top = req->top->http; + ctx.http_resp = req->resp; + ctx.req = req; + ctx.sp = req->sp; + ctx.now = req->t_prev; + ctx.ws = req->ws; + } + if (bo != NULL) { + if (req) + assert(method == VCL_MET_PIPE); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(bo->vcl, VCL_MAGIC); + vsl = bo->vsl; + ctx.vcl = bo->vcl; + ctx.http_bereq = bo->bereq; + ctx.http_beresp = bo->beresp; + ctx.bo = bo; + ctx.sp = bo->sp; + ctx.now = bo->t_prev; + ctx.ws = bo->ws; + } + assert(ctx.now != 0); + ctx.syntax = ctx.vcl->conf->syntax; + ctx.vsl = vsl; + ctx.specific = specific; + ctx.method = method; + wrk->handling = 0; + ctx.handling = &wrk->handling; + aws = WS_Snapshot(wrk->aws); + wrk->cur_method = method; + wrk->seen_methods |= method; + AN(vsl); + VSLb(vsl, SLT_VCL_call, "%s", VCL_Method_Name(method)); + func(&ctx); + VSLb(vsl, SLT_VCL_return, "%s", VCL_Return_Name(wrk->handling)); + wrk->cur_method |= 1; // Magic marker + if (wrk->handling == VCL_RET_FAIL) + wrk->stats->vcl_fail++; + + /* + * VCL/Vmods are not allowed to make permanent allocations from + * wrk->aws, but they can reserve and return from it. + */ + assert(aws == WS_Snapshot(wrk->aws)); +} + +#define VCL_MET_MAC(func, upper, typ, bitmap) \ +void \ +VCL_##func##_method(struct vcl *vcl, struct worker *wrk, \ + struct req *req, struct busyobj *bo, void *specific) \ +{ \ + \ + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); \ + CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC); \ + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); \ + vcl_call_method(wrk, req, bo, specific, \ + VCL_MET_ ## upper, vcl->conf->func##_func); \ + AN((1U << wrk->handling) & bitmap); \ +} + +#include "tbl/vcl_returns.h" From nils.goroll at uplex.de Thu Apr 19 07:53:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 19 Apr 2018 07:53:09 +0000 (UTC) Subject: [master] f5ce551 be paranoid when marking workspace overflows Message-ID: <20180419075309.AC3BDA45E4@lists.varnish-cache.org> commit f5ce55168a2d31eb24e7ea2a12036d1dc489235c Author: Nils Goroll Date: Thu Apr 19 09:47:20 2018 +0200 be paranoid when marking workspace overflows issue a membar to ensure that the overflow marker is written before a failing WS allocation returns and is thus visible from a different context. pan_ic() already has an implicit membar via the mutex lock. Ref #2645 diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 3ec5ec0..a6038de 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -31,6 +31,7 @@ #include "config.h" #include "cache_varnishd.h" +#include "vmb.h" #include @@ -115,6 +116,7 @@ WS_MarkOverflow(struct ws *ws) CHECK_OBJ_NOTNULL(ws, WS_MAGIC); ws->id[0] &= ~0x20; // cheesy toupper() + VWMB(); } static void From phk at FreeBSD.org Thu Apr 19 08:57:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 19 Apr 2018 08:57:13 +0000 (UTC) Subject: [master] 48a5737 Sync VJSN tests with Nicolas Seriots master list Message-ID: <20180419085713.5DD7CA5781@lists.varnish-cache.org> commit 48a573717aeef8fb0ff7bd40513cc3bab5d1a0ae Author: Poul-Henning Kamp Date: Thu Apr 19 08:15:19 2018 +0000 Sync VJSN tests with Nicolas Seriots master list diff --git a/lib/libvarnish/vjsn.c b/lib/libvarnish/vjsn.c index 8f08e74..d1b3408 100644 --- a/lib/libvarnish/vjsn.c +++ b/lib/libvarnish/vjsn.c @@ -520,40 +520,45 @@ vjsn_dump(const struct vjsn *js, FILE *fo) * * And run this python in test_parsing: - import glob - - def emit(fin): - if fin in skip: - return - x = bytearray(open(fin).read()) - if 0 in x: - return - if len(x) > 1000: - return - t = '\t"' - for i in x: - t += "\\x%02x" % i - if len(t) > 64: - print(t + '"') - t = '\t"' - print(t + '",') - - print("const char *good[] = {") - for f in glob.glob("y_*"): - emit(f) - print("\tNULL") - print("};") - - print("const char *bad[] = {") - for f in glob.glob("n_*"): - emit(f) - print("\tNULL") - print("};") - -*/ + import glob + + skip = {} + + def emit(fin): + if fin in skip: + return + x = bytearray(open(fin).read()) + if 0 in x: + return + if len(x) > 1000: + return + t = '\t"' + for i in x: + t += "\\x%02x" % i + if len(t) > 64: + print(t + '"') + t = '\t"' + print(t + '",') + + print("const char *good[] = {") + l = list(glob.glob("y_*")) + l.sort() + for f in l: + emit(f) + print("\tNULL") + print("};") + + print("const char *bad[] = {") + l = list(glob.glob("n_*")) + l.sort() + for f in l: + emit(f) + print("\tNULL") + print("};") + + */ static const char *good[] = { - "\x5b\x31\x32\x33\x65\x36\x35\x5d", "\x5b\x5b\x5d\x20\x20\x20\x5d", "\x5b\x22\x22\x5d", "\x5b\x5d", @@ -567,6 +572,7 @@ static const char *good[] = { "\x5b\x31\x2c\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c" "\x6c\x2c\x32\x5d", "\x5b\x32\x5d\x20", + "\x5b\x31\x32\x33\x65\x36\x35\x5d", "\x5b\x30\x65\x2b\x31\x5d", "\x5b\x30\x65\x31\x5d", "\x5b\x20\x34\x5d", @@ -620,10 +626,10 @@ static const char *good[] = { "\x34\x33\x35\x5c\x75\x30\x34\x33\x61\x5c\x75\x30\x34\x33\x65\x5c" "\x75\x30\x34\x33\x66\x5c\x75\x30\x34\x33\x30\x22\x20\x7d", "\x7b\x0a\x22\x61\x22\x3a\x20\x22\x62\x22\x0a\x7d", - "\x5b\x22\x5c\x75\x44\x38\x30\x31\x5c\x75\x64\x63\x33\x37\x22\x5d" - "", "\x5b\x22\x5c\x75\x30\x30\x36\x30\x5c\x75\x30\x31\x32\x61\x5c\x75" "\x31\x32\x41\x42\x22\x5d", + "\x5b\x22\x5c\x75\x44\x38\x30\x31\x5c\x75\x64\x63\x33\x37\x22\x5d" + "", "\x5b\x22\x5c\x75\x64\x38\x33\x64\x5c\x75\x64\x65\x33\x39\x5c\x75" "\x64\x38\x33\x64\x5c\x75\x64\x63\x38\x64\x22\x5d", "\x5b\x22\x5c\x22\x5c\x5c\x5c\x2f\x5c\x62\x5c\x66\x5c\x6e\x5c\x72" @@ -637,17 +643,17 @@ static const char *good[] = { "\x5b\x22\x5c\x75\x30\x30\x31\x32\x22\x5d", "\x5b\x22\x5c\x75\x46\x46\x46\x46\x22\x5d", "\x5b\x22\x61\x73\x64\x22\x5d", - "\x5b\x22\x6e\x65\x77\x5c\x75\x30\x30\x41\x30\x6c\x69\x6e\x65\x22" - "\x5d", "\x5b\x20\x22\x61\x73\x64\x22\x5d", "\x5b\x22\x5c\x75\x44\x42\x46\x46\x5c\x75\x44\x46\x46\x46\x22\x5d" "", + "\x5b\x22\x6e\x65\x77\x5c\x75\x30\x30\x41\x30\x6c\x69\x6e\x65\x22" + "\x5d", "\x5b\x22\xf4\x8f\xbf\xbf\x22\x5d", - "\x5b\x22\xf0\x9b\xbf\xbf\x22\x5d", "\x5b\x22\xef\xbf\xbf\x22\x5d", "\x5b\x22\x5c\x75\x30\x30\x30\x30\x22\x5d", "\x5b\x22\x5c\x75\x30\x30\x32\x63\x22\x5d", "\x5b\x22\xcf\x80\x22\x5d", + "\x5b\x22\xf0\x9b\xbf\xbf\x22\x5d", "\x5b\x22\x61\x73\x64\x20\x22\x5d", "\x22\x20\x22", "\x5b\x22\x5c\x75\x44\x38\x33\x34\x5c\x75\x44\x64\x31\x65\x22\x5d" @@ -668,12 +674,12 @@ static const char *good[] = { "", "\x5b\x22\x5c\x75\x44\x38\x33\x46\x5c\x75\x44\x46\x46\x45\x22\x5d" "", - "\x5b\x22\xe2\x82\xac\xf0\x9d\x84\x9e\x22\x5d", "\x5b\x22\x5c\x75\x32\x30\x30\x42\x22\x5d", "\x5b\x22\x5c\x75\x32\x30\x36\x34\x22\x5d", "\x5b\x22\x5c\x75\x46\x44\x44\x30\x22\x5d", "\x5b\x22\x5c\x75\x46\x46\x46\x45\x22\x5d", "\x5b\x22\x5c\x75\x30\x30\x32\x32\x22\x5d", + "\x5b\x22\xe2\x82\xac\xf0\x9d\x84\x9e\x22\x5d", "\x5b\x22\x61\x7f\x61\x22\x5d", "\x66\x61\x6c\x73\x65", "\x34\x32", @@ -698,9 +704,9 @@ static const char *bad[] = { "\x5b\x22\x78\x22\x5d\x5d", "\x5b\x22\x22\x2c\x5d", "\x5b\x22\x78\x22", - "\x5b\xff\x5d", "\x5b\x78", "\x5b\x33\x5b\x34\x5d\x5d", + "\x5b\xff\x5d", "\x5b\x31\x3a\x32\x5d", "\x5b\x2c\x5d", "\x5b\x2d\x5d", @@ -776,6 +782,7 @@ static const char *bad[] = { "\x7b\xf0\x9f\x87\xa8\xf0\x9f\x87\xad\x7d", "\x7b\x22\x61\x22\x3a\x22\x61\x22\x20\x31\x32\x33\x7d", "\x7b\x6b\x65\x79\x3a\x20\x27\x76\x61\x6c\x75\x65\x27\x7d", + "\x7b\x22\xb9\x22\x3a\x22\x30\x22\x2c\x7d", "\x7b\x22\x61\x22\x20\x62\x7d", "\x7b\x3a\x22\x62\x22\x7d", "\x7b\x22\x61\x22\x20\x22\x62\x22\x7d", @@ -783,7 +790,6 @@ static const char *bad[] = { "\x7b\x22\x61\x22", "\x7b\x31\x3a\x31\x7d", "\x7b\x39\x39\x39\x39\x45\x39\x39\x39\x39\x3a\x31\x7d", - "\x7b\x22\xb9\x22\x3a\x22\x30\x22\x2c\x7d", "\x7b\x6e\x75\x6c\x6c\x3a\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c\x6c\x3a" "\x6e\x75\x6c\x6c\x7d", "\x7b\x22\x69\x64\x22\x3a\x30\x2c\x2c\x2c\x2c\x2c\x7d", From phk at FreeBSD.org Thu Apr 19 12:45:19 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 19 Apr 2018 12:45:19 +0000 (UTC) Subject: [master] 716a307 We will have to maintain the filter list per VCL, so move the stuff there. Message-ID: <20180419124519.86353A9A9C@lists.varnish-cache.org> commit 716a307a750b3c2297984187b3ed1094ff94d50a Author: Poul-Henning Kamp Date: Thu Apr 19 11:47:12 2018 +0000 We will have to maintain the filter list per VCL, so move the stuff there. diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 5c5e94a..78f6055 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -224,81 +224,6 @@ VFP_Push(struct vfp_ctx *vc, const struct vfp *vfp) } /*-------------------------------------------------------------------- - */ - -struct vfp_filter { - unsigned magic; -#define VFP_FILTER_MAGIC 0xd40894e9 - const struct vfp *filter; - int nlen; - VTAILQ_ENTRY(vfp_filter) list; -}; - -static VTAILQ_HEAD(,vfp_filter) vfp_filters = - VTAILQ_HEAD_INITIALIZER(vfp_filters); - -void -VFP_AddFilter(const struct vfp *filter) -{ - struct vfp_filter *vp; - - VTAILQ_FOREACH(vp, &vfp_filters, list) { - assert(vp->filter != filter); - assert(strcasecmp(vp->filter->name, filter->name)); - } - ALLOC_OBJ(vp, VFP_FILTER_MAGIC); - AN(vp); - vp->filter = filter; - vp->nlen = strlen(filter->name); - VTAILQ_INSERT_TAIL(&vfp_filters, vp, list); -} - -void -VFP_RemoveFilter(const struct vfp *filter) -{ - struct vfp_filter *vp; - - VTAILQ_FOREACH(vp, &vfp_filters, list) { - if (vp->filter == filter) - break; - } - AN(vp); - VTAILQ_REMOVE(&vfp_filters, vp, list); - FREE_OBJ(vp); -} - -int -VFP_FilterList(struct vfp_ctx *vc, const char *fl) -{ - const char *p, *q; - const struct vfp_filter *vp; - - VSLb(vc->wrk->vsl, SLT_Filters, "%s", fl); - - for (p = fl; *p; p = q) { - if (vct_isspace(*p)) { - q = p + 1; - continue; - } - for (q = p; *q; q++) - if (vct_isspace(*q)) - break; - VTAILQ_FOREACH(vp, &vfp_filters, list) { - if (vp->nlen != q - p) - continue; - if (!memcmp(p, vp->filter->name, vp->nlen)) - break; - } - if (vp == NULL) - return (VFP_Error(vc, - "Filter '%.*s' not found", (int)(q-p), p)); - if (VFP_Push(vc, vp->filter) == NULL) - return (-1); - } - return (0); -} - -/*-------------------------------------------------------------------- * Debugging aids */ @@ -324,9 +249,4 @@ VFP_Init(void) { CLI_AddFuncs(debug_cmds); - VFP_AddFilter(&VFP_testgunzip); - VFP_AddFilter(&VFP_gunzip); - VFP_AddFilter(&VFP_gzip); - VFP_AddFilter(&VFP_esi); - VFP_AddFilter(&VFP_esi_gzip); } diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 924782e..65a0e65 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -89,8 +89,8 @@ enum vfp_status VFP_Suck(struct vfp_ctx *, void *p, ssize_t *lp); enum vfp_status VFP_Error(struct vfp_ctx *, const char *fmt, ...) v_printflike_(2, 3); int VFP_FilterList(struct vfp_ctx *, const char *); -void VFP_AddFilter(const struct vfp *); -void VFP_RemoveFilter(const struct vfp *); +void VFP_AddFilter(struct vcl *, const struct vfp *); +void VFP_RemoveFilter(struct vcl *, const struct vfp *); /* Deliver processors ------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index c77cca9..bfc76b4 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -351,6 +351,7 @@ child_main(int sigmagic, size_t altstksz) ObjInit(); VCL_Init(); + VCL_VRT_Init(); HTTP_Init(); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 397703a..072c526 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -405,6 +405,9 @@ typedef int vcl_be_func(struct cli *, struct director *, void *); int VCL_IterDirector(struct cli *, const char *, vcl_be_func *, void *); +/* cache_vcl_vrt.c */ +void VCL_VRT_Init(void); + /* cache_vrt.c */ void VRTPRIV_init(struct vrt_privs *privs); void VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index abc519f..c4d9ae1 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -526,6 +526,7 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx, XXXAN(vcl->loaded_name); VTAILQ_INIT(&vcl->director_list); VTAILQ_INIT(&vcl->ref_list); + VTAILQ_INIT(&vcl->vfps); vcl->temp = VCL_TEMP_INIT; diff --git a/bin/varnishd/cache/cache_vcl.h b/bin/varnishd/cache/cache_vcl.h index 87433af..f0789a8 100644 --- a/bin/varnishd/cache/cache_vcl.h +++ b/bin/varnishd/cache/cache_vcl.h @@ -26,8 +26,15 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * + * NB: This is a private .h file for cache_vcl*.c + * NB: No other code should include this file. + * */ +struct vfp_filter; + +VTAILQ_HEAD(vfp_filter_head, vfp_filter); + struct vcl { unsigned magic; #define VCL_MAGIC 0x214188f2 @@ -45,6 +52,7 @@ struct vcl { int nrefs; struct vcl *label; int nlabels; + struct vfp_filter_head vfps; }; struct vclref { diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index 484c14b..9c8e823 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -37,9 +37,11 @@ #include "cache_varnishd.h" #include "vcl.h" +#include "vct.h" #include "cache_director.h" #include "cache_vcl.h" +#include "cache_filter.h" /*--------------------------------------------------------------------*/ @@ -413,3 +415,98 @@ VCL_##func##_method(struct vcl *vcl, struct worker *wrk, \ } #include "tbl/vcl_returns.h" + +/*-------------------------------------------------------------------- + */ + +struct vfp_filter { + unsigned magic; +#define VFP_FILTER_MAGIC 0xd40894e9 + const struct vfp *filter; + int nlen; + VTAILQ_ENTRY(vfp_filter) list; +}; + +static struct vfp_filter_head vfp_filters = + VTAILQ_HEAD_INITIALIZER(vfp_filters); + +void +VFP_AddFilter(struct vcl *vcl, const struct vfp *filter) +{ + struct vfp_filter *vp; + struct vfp_filter_head *hd = &vfp_filters; + + VTAILQ_FOREACH(vp, hd, list) { + xxxassert(vp->filter != filter); + xxxassert(strcasecmp(vp->filter->name, filter->name)); + } + if (vcl != NULL) { + hd = &vcl->vfps; + VTAILQ_FOREACH(vp, hd, list) { + xxxassert(vp->filter != filter); + xxxassert(strcasecmp(vp->filter->name, filter->name)); + } + } + ALLOC_OBJ(vp, VFP_FILTER_MAGIC); + AN(vp); + vp->filter = filter; + vp->nlen = strlen(filter->name); + VTAILQ_INSERT_TAIL(hd, vp, list); +} + +void +VFP_RemoveFilter(struct vcl *vcl, const struct vfp *filter) +{ + struct vfp_filter *vp; + struct vfp_filter_head *hd = &vcl->vfps; + + AN(vcl); + VTAILQ_FOREACH(vp, hd, list) { + if (vp->filter == filter) + break; + } + XXXAN(vp); + VTAILQ_REMOVE(hd, vp, list); + FREE_OBJ(vp); +} + +int +VFP_FilterList(struct vfp_ctx *vc, const char *fl) +{ + const char *p, *q; + const struct vfp_filter *vp; + + VSLb(vc->wrk->vsl, SLT_Filters, "%s", fl); + + for (p = fl; *p; p = q) { + if (vct_isspace(*p)) { + q = p + 1; + continue; + } + for (q = p; *q; q++) + if (vct_isspace(*q)) + break; + VTAILQ_FOREACH(vp, &vfp_filters, list) { + if (vp->nlen != q - p) + continue; + if (!memcmp(p, vp->filter->name, vp->nlen)) + break; + } + if (vp == NULL) + return (VFP_Error(vc, + "Filter '%.*s' not found", (int)(q-p), p)); + if (VFP_Push(vc, vp->filter) == NULL) + return (-1); + } + return (0); +} + +void +VCL_VRT_Init(void) +{ + VFP_AddFilter(NULL, &VFP_testgunzip); + VFP_AddFilter(NULL, &VFP_gunzip); + VFP_AddFilter(NULL, &VFP_gzip); + VFP_AddFilter(NULL, &VFP_esi); + VFP_AddFilter(NULL, &VFP_esi_gzip); +} From phk at FreeBSD.org Thu Apr 19 12:45:19 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 19 Apr 2018 12:45:19 +0000 (UTC) Subject: [master] ad69f04 Rename functions to their new family Message-ID: <20180419124519.9AF2DA9A9F@lists.varnish-cache.org> commit ad69f04e05d77bdaa48bb05e0527eef66ffaf7a5 Author: Poul-Henning Kamp Date: Thu Apr 19 12:15:07 2018 +0000 Rename functions to their new family diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 07259f6..4e074c7 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -606,7 +606,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) } if (bo->filter_list == NULL || - VFP_FilterList(bo->vfc, bo->filter_list)) { + VCL_StackVFP(bo->vfc, bo->filter_list)) { (bo)->htc->doclose = SC_OVERLOAD; VDI_Finish((bo)->wrk, bo); return (F_STP_ERROR); diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 65a0e65..3192238 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -88,9 +88,8 @@ struct vfp_ctx { enum vfp_status VFP_Suck(struct vfp_ctx *, void *p, ssize_t *lp); enum vfp_status VFP_Error(struct vfp_ctx *, const char *fmt, ...) v_printflike_(2, 3); -int VFP_FilterList(struct vfp_ctx *, const char *); -void VFP_AddFilter(struct vcl *, const struct vfp *); -void VFP_RemoveFilter(struct vcl *, const struct vfp *); +void VRT_AddVFP(VRT_CTX, const struct vfp *); +void VRT_RemoveVFP(VRT_CTX, const struct vfp *); /* Deliver processors ------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 072c526..c859324 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -407,6 +407,7 @@ int VCL_IterDirector(struct cli *, const char *, vcl_be_func *, void *); /* cache_vcl_vrt.c */ void VCL_VRT_Init(void); +int VCL_StackVFP(struct vfp_ctx *, const char *); /* cache_vrt.c */ void VRTPRIV_init(struct vrt_privs *privs); diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index 9c8e823..c29e689 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -431,7 +431,7 @@ static struct vfp_filter_head vfp_filters = VTAILQ_HEAD_INITIALIZER(vfp_filters); void -VFP_AddFilter(struct vcl *vcl, const struct vfp *filter) +VRT_AddVFP(VRT_CTX, const struct vfp *filter) { struct vfp_filter *vp; struct vfp_filter_head *hd = &vfp_filters; @@ -440,8 +440,8 @@ VFP_AddFilter(struct vcl *vcl, const struct vfp *filter) xxxassert(vp->filter != filter); xxxassert(strcasecmp(vp->filter->name, filter->name)); } - if (vcl != NULL) { - hd = &vcl->vfps; + if (ctx != NULL) { + hd = &ctx->vcl->vfps; VTAILQ_FOREACH(vp, hd, list) { xxxassert(vp->filter != filter); xxxassert(strcasecmp(vp->filter->name, filter->name)); @@ -455,12 +455,11 @@ VFP_AddFilter(struct vcl *vcl, const struct vfp *filter) } void -VFP_RemoveFilter(struct vcl *vcl, const struct vfp *filter) +VRT_RemoveVFP(VRT_CTX, const struct vfp *filter) { struct vfp_filter *vp; - struct vfp_filter_head *hd = &vcl->vfps; + struct vfp_filter_head *hd = &ctx->vcl->vfps; - AN(vcl); VTAILQ_FOREACH(vp, hd, list) { if (vp->filter == filter) break; @@ -471,7 +470,7 @@ VFP_RemoveFilter(struct vcl *vcl, const struct vfp *filter) } int -VFP_FilterList(struct vfp_ctx *vc, const char *fl) +VCL_StackVFP(struct vfp_ctx *vc, const char *fl) { const char *p, *q; const struct vfp_filter *vp; @@ -504,9 +503,9 @@ VFP_FilterList(struct vfp_ctx *vc, const char *fl) void VCL_VRT_Init(void) { - VFP_AddFilter(NULL, &VFP_testgunzip); - VFP_AddFilter(NULL, &VFP_gunzip); - VFP_AddFilter(NULL, &VFP_gzip); - VFP_AddFilter(NULL, &VFP_esi); - VFP_AddFilter(NULL, &VFP_esi_gzip); + VRT_AddVFP(NULL, &VFP_testgunzip); + VRT_AddVFP(NULL, &VFP_gunzip); + VRT_AddVFP(NULL, &VFP_gzip); + VRT_AddVFP(NULL, &VFP_esi); + VRT_AddVFP(NULL, &VFP_esi_gzip); } From phk at FreeBSD.org Thu Apr 19 12:45:19 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 19 Apr 2018 12:45:19 +0000 (UTC) Subject: [master] c984e5d Look for VFP's also on the per-vcl list Message-ID: <20180419124519.B729CA9AA3@lists.varnish-cache.org> commit c984e5d39f6a8f4ad34aa80c8a4519b35bedde5c Author: Poul-Henning Kamp Date: Thu Apr 19 12:37:36 2018 +0000 Look for VFP's also on the per-vcl list diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 4e074c7..487807c 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -606,7 +606,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) } if (bo->filter_list == NULL || - VCL_StackVFP(bo->vfc, bo->filter_list)) { + VCL_StackVFP(bo->vfc, bo->vcl, bo->filter_list)) { (bo)->htc->doclose = SC_OVERLOAD; VDI_Finish((bo)->wrk, bo); return (F_STP_ERROR); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index c859324..bec82c1 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -407,7 +407,7 @@ int VCL_IterDirector(struct cli *, const char *, vcl_be_func *, void *); /* cache_vcl_vrt.c */ void VCL_VRT_Init(void); -int VCL_StackVFP(struct vfp_ctx *, const char *); +int VCL_StackVFP(struct vfp_ctx *, const struct vcl *, const char *); /* cache_vrt.c */ void VRTPRIV_init(struct vrt_privs *privs); diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index c29e689..36bdf3f 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -470,7 +470,7 @@ VRT_RemoveVFP(VRT_CTX, const struct vfp *filter) } int -VCL_StackVFP(struct vfp_ctx *vc, const char *fl) +VCL_StackVFP(struct vfp_ctx *vc, const struct vcl *vcl, const char *fl) { const char *p, *q; const struct vfp_filter *vp; @@ -491,6 +491,14 @@ VCL_StackVFP(struct vfp_ctx *vc, const char *fl) if (!memcmp(p, vp->filter->name, vp->nlen)) break; } + if (vp == NULL) { + VTAILQ_FOREACH(vp, &vcl->vfps, list) { + if (vp->nlen != q - p) + continue; + if (!memcmp(p, vp->filter->name, vp->nlen)) + break; + } + } if (vp == NULL) return (VFP_Error(vc, "Filter '%.*s' not found", (int)(q-p), p)); diff --git a/bin/varnishtest/tests/m00048.vtc b/bin/varnishtest/tests/m00048.vtc new file mode 100644 index 0000000..9f01bf5 --- /dev/null +++ b/bin/varnishtest/tests/m00048.vtc @@ -0,0 +1,21 @@ +varnishtest "VMOD vfp" + +server s1 { + rxreq + txresp -body "Ponto Facto, Caesar Transit!" +} -start + +varnish v1 -vcl+backend { + import debug; + + sub vcl_backend_response { + set beresp.filters = "rot13"; + } +} -start + +client c1 { + txreq + rxresp + expect resp.body == "Cbagb Snpgb, Pnrfne Genafvg!" +} -run + From phk at FreeBSD.org Thu Apr 19 12:45:19 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 19 Apr 2018 12:45:19 +0000 (UTC) Subject: [master] 4ad414e Add a proof-of-concept rot13 VFP Message-ID: <20180419124519.D0724A9AA7@lists.varnish-cache.org> commit 4ad414e218f16bdd5eb931f4fcc046d8f0d44c72 Author: Poul-Henning Kamp Date: Thu Apr 19 12:38:54 2018 +0000 Add a proof-of-concept rot13 VFP diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 0da45d8..5c2661e 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -36,6 +36,7 @@ #include #include "cache/cache_varnishd.h" +#include "cache/cache_filter.h" #include "vsa.h" #include "vtim.h" @@ -57,6 +58,37 @@ static pthread_mutex_t vsc_mtx = PTHREAD_MUTEX_INITIALIZER; static struct vsc_seg *vsc_seg = NULL; static struct VSC_debug *vsc = NULL; +/**********************************************************************/ + +static enum vfp_status v_matchproto_(vfp_pull_f) +xyzzy_rot13_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, + ssize_t *lp) +{ + enum vfp_status vp; + char *q; + ssize_t l; + + (void)vfe; + vp = VFP_Suck(vc, p, lp); + if (vp == VFP_ERROR) + return (vp); + q = p; + for (l = 0; l < *lp; l++, q++) { + if (*q >= 'A' && *q <= 'Z') + *q = (((*q - 'A') + 13) % 26) + 'A'; + if (*q >= 'a' && *q <= 'z') + *q = (((*q - 'a') + 13) % 26) + 'a'; + } + return (vp); +} + +static const struct vfp xyzzy_rot13 = { + .name = "rot13", + .pull = xyzzy_rot13_pull, +}; + +/**********************************************************************/ + VCL_STRING v_matchproto_(td_debug_author) xyzzy_author(VRT_CTX, VCL_ENUM person, VCL_ENUM someone) { @@ -241,6 +273,12 @@ event_load(VRT_CTX, struct vmod_priv *priv) AN(priv_vcl->foo); priv->priv = priv_vcl; priv->free = priv_vcl_free; + + /* + * NB: This is a proof of concept, until we decide what the real + * API should look like, do NOT do this anywhere else. + */ + VRT_AddVFP(ctx, &xyzzy_rot13); return (0); } @@ -319,6 +357,7 @@ event_function(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e) case VCL_EVENT_WARM: return (event_warm(ctx, priv)); case VCL_EVENT_COLD: return (event_cold(ctx, priv)); case VCL_EVENT_DISCARD: + VRT_RemoveVFP(ctx, &xyzzy_rot13); if (vsc) VSC_debug_Destroy(&vsc_seg); return (0); From hermunn at varnish-software.com Thu Apr 19 20:36:14 2018 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Thu, 19 Apr 2018 20:36:14 +0000 (UTC) Subject: [4.1] 3d85f73 Try to hotfix documentation builds on varnish-cache.org Message-ID: <20180419203614.3C597B4493@lists.varnish-cache.org> commit 3d85f7301cab6bbc3c8e2a43d0042462bde430ca Author: P?l Hermunn Johansen Date: Thu Apr 19 22:34:18 2018 +0200 Try to hotfix documentation builds on varnish-cache.org What can possibly go wrong? diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 41c02a3..4fa2ecf 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -7,7 +7,7 @@ SPHINXBUILD = sphinx-build -W -q -N PAPER = a4 BUILDDIR = build -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees -D latex_paper_size=$(PAPER) $(SPHINXOPTS) $(builddir) +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) $(builddir) .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest From phk at FreeBSD.org Fri Apr 20 10:22:19 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 20 Apr 2018 10:22:19 +0000 (UTC) Subject: [master] df4bbfa Whitespace ocd Message-ID: <20180420102219.081CF95405@lists.varnish-cache.org> commit df4bbfaf48d1ce9a6e689da3997b13bc8c5d7616 Author: Poul-Henning Kamp Date: Fri Apr 20 09:54:24 2018 +0000 Whitespace ocd diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index c1b765c..332f1c3 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -43,7 +43,7 @@ Without PROXY protocol:: client server remote local - v v + v v CLIENT ------------ VARNISHD @@ -71,18 +71,18 @@ local.endpoint ``VCL >= 4.1`` Type: STRING Readable from: client, backend - + The address of the '-a' socket the session was accepted on. If the argument was `-a foo=:81` this would be ":81" - + local.socket ``VCL >= 4.1`` Type: STRING Readable from: client, backend - + The name of the '-a' socket the session was accepted on. If the argument was `-a foo=:81` this would be "foo". @@ -99,7 +99,7 @@ remote.ip The IP address of the other end of the TCP connection. This can either be the clients IP, or the outgoing IP of a proxy server. - + If the connection is a UNIX domain socket, the value will be `0.0.0.0:0` @@ -109,7 +109,7 @@ client.ip Readable from: client, backend - + The client's IP address, either the same as `local.ip` or what the PROXY protocol told us. @@ -121,7 +121,7 @@ client.identity Writable from: client - + Identification of the client, used to load balance in the client director. Defaults to `client.ip` @@ -136,28 +136,28 @@ server.ip Readable from: client, backend - + The IP address of the socket on which the client connection was received, either the same as `server.ip` or what the PROXY protocol told us. - + server.hostname Type: STRING Readable from: all - + The host name of the server, as returned by the `gethostname(3)` system function. - + server.identity Type: STRING Readable from: all - + The identity of the server, as set by the `-i` parameter. If an `-i` parameter is not passed to varnishd, the return @@ -176,10 +176,10 @@ req Readable from: client - + The entire request HTTP data structure. Mostly useful for passing to VMODs. - + req.method @@ -189,9 +189,9 @@ req.method Writable from: client - + The request method (e.g. "GET", "HEAD", ...) - + req.hash @@ -199,11 +199,11 @@ req.hash Readable from: vcl_hit, vcl_miss, vcl_pass, vcl_purge, vcl_deliver - + The hash key of this request. Mostly useful for passing to VMODs, but can also be useful for debugging hit/miss status. - + req.url @@ -213,9 +213,9 @@ req.url Writable from: client - + The requested URL, for instance "/robots.txt". - + req.proto ``VCL <= 4.0`` @@ -227,7 +227,7 @@ req.proto ``VCL <= 4.0`` The HTTP protocol version used by the client, usually "HTTP/1.1" or "HTTP/2.0". - + req.proto ``VCL >= 4.1`` Type: STRING @@ -236,7 +236,7 @@ req.proto ``VCL >= 4.1`` The HTTP protocol version used by the client, usually "HTTP/1.1" or "HTTP/2.0". - + req.http.* @@ -248,12 +248,12 @@ req.http.* Unsetable from: client - + The headers of request, things like `req.http.date`. The RFCs allow multiple headers with the same name, and both `set` and `unset` will remove *all* headers with the name given. - + req.restarts @@ -261,9 +261,9 @@ req.restarts Readable from: client - + A count of how many times this request has been restarted. - + req.storage @@ -273,16 +273,16 @@ req.storage Writable from: client - + The storage backend to use to save this request body. - + req.esi_level Type: INT Readable from: client - + A count of how many levels of ESI requests we're currently at. req.ttl @@ -293,16 +293,16 @@ req.ttl Writable from: client - + Upper limit on the object age for cache lookups to return hit. - + Usage of req.ttl should be replaced with a check on obj.ttl in vcl_hit, returning miss when needed, but this currently hits bug #1799, so an additional workaround is required. - + Deprecated and scheduled for removal with varnish release 7. - + req.xid @@ -330,10 +330,10 @@ req.can_gzip Type: BOOL Readable from: client - + True if the client provided `gzip` or `x-gzip` in the `Accept-Encoding` header. - + req.backend_hint @@ -349,7 +349,7 @@ req.backend_hint or the director otherwise. When used in string context, returns the name of the director or backend, respectively. - + req.hash_ignore_busy @@ -365,7 +365,7 @@ req.hash_ignore_busy You only want to do this when you have two server looking up content sideways from each other to avoid deadlocks. - + req.hash_always_miss @@ -382,7 +382,7 @@ req.hash_always_miss This is useful to force-update the cache without invalidating existing entries in case the fetch fails. - + req_top.method @@ -393,39 +393,39 @@ req_top.method The request method of the top-level request in a tree of ESI requests. (e.g. "GET", "HEAD"). Identical to req.method in non-ESI requests. - + req_top.url Type: STRING Readable from: client - + The requested URL of the top-level request in a tree of ESI requests. Identical to req.url in non-ESI requests. - + req_top.http.* Type: HEADER Readable from: client - + HTTP headers of the top-level request in a tree of ESI requests. Identical to req.http. in non-ESI requests. - + req_top.proto Type: STRING Readable from: client - + HTTP protocol version of the top-level request in a tree of ESI requests. Identical to req.proto in non-ESI requests. - + bereq ~~~~~ @@ -445,25 +445,25 @@ bereq The entire backend request HTTP data structure. Mostly useful as argument to VMODs. - + bereq.xid Type: STRING Readable from: backend - + Unique ID of this request. - + bereq.retries Type: INT Readable from: backend - + A count of how many times this request has been retried. - + bereq.backend @@ -472,21 +472,21 @@ bereq.backend Readable from: vcl_pipe, backend Writable from: vcl_pipe, backend - + This is the backend or director we attempt to fetch from. When set to a director, reading this variable returns an actual backend if the director has resolved immediately, or the director otherwise. When used in string context, returns the name of the director or backend, respectively. - + bereq.body Type: BODY Unsetable from: vcl_backend_fetch - + The request body, only present on `pass` requests. Unset will also remove `bereq.http.Content-Length`. @@ -496,9 +496,9 @@ bereq.hash Type: BLOB Readable from: vcl_pipe, backend - + The hash key of this request, a copy of `req.hash`. - + bereq.method @@ -507,11 +507,11 @@ bereq.method Readable from: vcl_pipe, backend Writable from: vcl_pipe, backend - + The request type (e.g. "GET", "HEAD"). Regular (non-pipe, non-pass) fetches are always "GET" - + bereq.url @@ -522,7 +522,7 @@ bereq.url Writable from: vcl_pipe, backend The requested URL, copied from `req.url` - + bereq.proto ``VCL <= 4.0`` @@ -531,19 +531,19 @@ bereq.proto ``VCL <= 4.0`` Readable from: vcl_pipe, backend Writable from: vcl_pipe, backend - + The HTTP protocol version, "HTTP/1.1" unless a pass or pipe request has "HTTP/1.0" in `req.proto` - + bereq.proto ``VCL >= 4.1`` Type: STRING Readable from: vcl_pipe, backend - + The HTTP protocol version, "HTTP/1.1" unless a pass or pipe request has "HTTP/1.0" in `req.proto` - + bereq.http.* @@ -556,7 +556,7 @@ bereq.http.* Unsetable from: vcl_pipe, backend The headers to be sent to the backend. - + bereq.uncacheable @@ -564,10 +564,10 @@ bereq.uncacheable Readable from: backend - + Indicates whether this request is uncacheable due to a `pass` in the client side or a hit on an hit-for-pass object. - + bereq.connect_timeout @@ -576,10 +576,10 @@ bereq.connect_timeout Readable from: vcl_pipe, backend Writable from: vcl_pipe, backend - + The time in seconds to wait for a backend connection to be established. - + bereq.first_byte_timeout @@ -588,10 +588,10 @@ bereq.first_byte_timeout Readable from: backend Writable from: backend - + The time in seconds to wait getting the first byte back from the backend. Not available in pipe mode. - + bereq.between_bytes_timeout @@ -600,17 +600,17 @@ bereq.between_bytes_timeout Readable from: backend Writable from: backend - + The time in seconds to wait between each received byte from the backend. Not available in pipe mode. - + bereq.is_bgfetch Type: BOOL Readable from: backend - + True for fetches where the client got a hit on an object in grace, and this fetch was kicked of in the background to get a fresh copy. @@ -635,7 +635,7 @@ beresp.body Type: BODY Writable from: vcl_backend_error - + For producing a synthetic body. beresp.proto ``VCL <= 4.0`` @@ -647,7 +647,7 @@ beresp.proto ``VCL <= 4.0`` Writable from: vcl_backend_response, vcl_backend_error The HTTP protocol version the backend replied with. - + beresp.proto ``VCL >= 4.1`` Type: STRING @@ -655,7 +655,7 @@ beresp.proto ``VCL >= 4.1`` Readable from: vcl_backend_response, vcl_backend_error The HTTP protocol version the backend replied with. - + beresp.status @@ -666,7 +666,7 @@ beresp.status Writable from: vcl_backend_response, vcl_backend_error The HTTP status code returned by the server. - + Status codes on the form XXYZZ can be set where XXYZZ is less than 65536 and Y is [1...9]. Only YZZ will be sent back to clients. @@ -709,7 +709,7 @@ beresp.do_esi Set it to true to parse the object for ESI directives. Will only be honored if req.esi is true. - + beresp.do_stream @@ -768,7 +768,7 @@ beresp.was_304 Readable from: vcl_backend_response, vcl_backend_error - + When `true` this indicates that we got a 304 response to our conditional fetch from the backend and turned that into `beresp.status = 200` @@ -782,14 +782,14 @@ beresp.uncacheable Writable from: vcl_backend_response, vcl_backend_error Inherited from bereq.uncacheable, see there. - + Setting this variable makes the object uncacheable. This may may produce a hit-for-miss object in the cache. - + Clearing the variable has no effect and will log the warning "Ignoring attempt to reset beresp.uncacheable". - + beresp.ttl @@ -800,7 +800,7 @@ beresp.ttl Writable from: vcl_backend_response, vcl_backend_error The object's remaining time to live, in seconds. - + beresp.age @@ -809,7 +809,7 @@ beresp.age Readable from: vcl_backend_response, vcl_backend_error The age of the object. - + beresp.grace @@ -820,7 +820,7 @@ beresp.grace Writable from: vcl_backend_response, vcl_backend_error Set to a period to enable grace. - + beresp.keep @@ -831,13 +831,13 @@ beresp.keep Writable from: vcl_backend_response, vcl_backend_error Set to a period to enable conditional backend requests. - + The keep time is cache lifetime in addition to the ttl. - + Objects with ttl expired but with keep time left may be used to issue conditional (If-Modified-Since / If-None-Match) requests to the backend to refresh them. - + beresp.backend @@ -849,7 +849,7 @@ beresp.backend was set to a director, this will be the backend selected by the director. When used in string context, returns its name. - + beresp.backend.name @@ -859,7 +859,7 @@ beresp.backend.name Name of the backend this response was fetched from. Same as beresp.backend. - + beresp.backend.ip ``VCL <= 4.0`` @@ -877,7 +877,7 @@ beresp.storage Writable from: vcl_backend_response, vcl_backend_error - + The storage backend to use to save this object. beresp.storage_hint ``VCL <= 4.0`` @@ -888,10 +888,10 @@ beresp.storage_hint ``VCL <= 4.0`` Writable from: vcl_backend_response, vcl_backend_error - + Deprecated since varnish 5.1 and discontinued since VCL 4.1 (varnish 6.0). Use beresp.storage instead. - + Hint to Varnish that you want to save this object to a particular storage backend. @@ -904,7 +904,7 @@ beresp.filters Writable from: vcl_backend_response List of VFP filters the beresp.body will be pulled through. - + obj ~~~ @@ -917,7 +917,7 @@ obj.proto Readable from: vcl_hit The HTTP protocol version stored in the object. - + obj.status @@ -925,9 +925,9 @@ obj.status Readable from: vcl_hit - + The HTTP status code stored in the object. - + obj.reason @@ -935,9 +935,9 @@ obj.reason Readable from: vcl_hit - + The HTTP reason phrase stored in the object. - + obj.hits @@ -945,11 +945,11 @@ obj.hits Readable from: vcl_hit, vcl_deliver - + The count of cache-hits on this object. In `vcl_deliver` a value of 0 indicates a cache miss. - + obj.http.* @@ -958,7 +958,7 @@ obj.http.* Readable from: vcl_hit The HTTP headers stored in the object. - + obj.ttl @@ -967,7 +967,7 @@ obj.ttl Readable from: vcl_hit, vcl_deliver The object's remaining time to live, in seconds. - + obj.age @@ -976,7 +976,7 @@ obj.age Readable from: vcl_hit, vcl_deliver The age of the object. - + obj.grace @@ -985,7 +985,7 @@ obj.grace Readable from: vcl_hit, vcl_deliver The object's grace period in seconds. - + obj.keep @@ -994,7 +994,7 @@ obj.keep Readable from: vcl_hit, vcl_deliver The object's keep period in seconds. - + obj.uncacheable @@ -1004,7 +1004,7 @@ obj.uncacheable Whether the object is uncacheable (pass, hit-for-pass or hit-for-miss). - + obj.storage @@ -1013,7 +1013,7 @@ obj.storage Readable from: vcl_hit, vcl_deliver The storage backend where this object is stored. - + resp ~~~~ @@ -1070,14 +1070,14 @@ resp.status Writable from: vcl_deliver, vcl_synth The HTTP status code that will be returned. - + Assigning a HTTP standardized code to resp.status will also set resp.reason to the corresponding status message. - + resp.status 200 will get changed into 304 by core code after a return(deliver) from vcl_deliver for conditional requests to cached content if validation succeeds. - + resp.reason @@ -1088,7 +1088,7 @@ resp.reason Writable from: vcl_deliver, vcl_synth The HTTP status message that will be returned. - + resp.http.* @@ -1100,7 +1100,7 @@ resp.http.* Unsetable from: vcl_deliver, vcl_synth - + The HTTP headers that will be returned. resp.do_esi ``VCL >= 4.1`` @@ -1116,7 +1116,7 @@ resp.do_esi ``VCL >= 4.1`` This can be used to selectively disable ESI processing, even though ESI parsing happened during fetch. This is useful when Varnish caches peer with each other. - + resp.is_streaming @@ -1126,7 +1126,7 @@ resp.is_streaming Returns true when the response will be streamed while being fetched from the backend. - + Special variables ~~~~~~~~~~~~~~~~~ @@ -1137,7 +1137,7 @@ now Readable from: all - + The current time, in seconds since the UNIX epoch. When converted to STRING in expressions it returns @@ -1169,10 +1169,10 @@ storage..free_space Readable from: client, backend - + Free space available in the named stevedore. Only available for the malloc stevedore. - + storage..used_space @@ -1180,10 +1180,10 @@ storage..used_space Readable from: client, backend - + Used space in the named stevedore. Only available for the malloc stevedore. - + storage..happy @@ -1191,7 +1191,7 @@ storage..happy Readable from: client, backend - + Health status for the named stevedore. Not available in any of the current stevedores. - + From phk at FreeBSD.org Fri Apr 20 10:22:19 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 20 Apr 2018 10:22:19 +0000 (UTC) Subject: [master] b8c80cc Minor FlexeLinting Message-ID: <20180420102219.1FE2495408@lists.varnish-cache.org> commit b8c80ccb9c17b49ceab9f31de54cbd2799b9ad80 Author: Poul-Henning Kamp Date: Fri Apr 20 10:14:08 2018 +0000 Minor FlexeLinting diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 78f6055..d9db19f 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -35,8 +35,6 @@ #include "cache_filter.h" #include "vcli_serve.h" -#include "vct.h" - static unsigned fetchfrag; /*-------------------------------------------------------------------- diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c index 6315d95..f233c5f 100644 --- a/lib/libvarnish/vus.c +++ b/lib/libvarnish/vus.c @@ -121,7 +121,7 @@ VUS_connect(const char *path, int msec) if (msec != 0) (void)VTCP_nonblocking(s); - i = connect(s, (const struct sockaddr *)&uds, sl); + i = connect(s, (const void*)&uds, sl); if (i == 0) return (s); if (errno != EINPROGRESS) { diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index 75dda13..5fa8833 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -262,7 +262,7 @@ xyzzy_dyn_uds__fini(struct xyzzy_debug_dyn_uds **udsp) free(uds->vcl_name); AZ(pthread_mutex_destroy(&uds->mtx)); FREE_OBJ(uds); - udsp = NULL; + *udsp = NULL; } VCL_BACKEND v_matchproto_(td_debug_dyn_uds_backend) From phk at FreeBSD.org Fri Apr 20 10:22:19 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 20 Apr 2018 10:22:19 +0000 (UTC) Subject: [master] ffeec82 Whitespace OCD Message-ID: <20180420102219.39A329540E@lists.varnish-cache.org> commit ffeec8263ca39652d7b3d4f329c715463def97c6 Author: Poul-Henning Kamp Date: Fri Apr 20 10:19:19 2018 +0000 Whitespace OCD diff --git a/lib/libvarnish/vjsn.c b/lib/libvarnish/vjsn.c index d1b3408..4ba0cd4 100644 --- a/lib/libvarnish/vjsn.c +++ b/lib/libvarnish/vjsn.c @@ -520,42 +520,41 @@ vjsn_dump(const struct vjsn *js, FILE *fo) * * And run this python in test_parsing: - import glob - - skip = {} - - def emit(fin): - if fin in skip: - return - x = bytearray(open(fin).read()) - if 0 in x: - return - if len(x) > 1000: - return - t = '\t"' - for i in x: - t += "\\x%02x" % i - if len(t) > 64: - print(t + '"') - t = '\t"' - print(t + '",') - - print("const char *good[] = {") - l = list(glob.glob("y_*")) - l.sort() - for f in l: - emit(f) - print("\tNULL") - print("};") - - print("const char *bad[] = {") - l = list(glob.glob("n_*")) - l.sort() - for f in l: - emit(f) - print("\tNULL") - print("};") - + import glob + + skip = {} + + def emit(fin): + if fin in skip: + return + x = bytearray(open(fin).read()) + if 0 in x: + return + if len(x) > 1000: + return + t = '\t"' + for i in x: + t += "\\x%02x" % i + if len(t) > 64: + print(t + '"') + t = '\t"' + print(t + '",') + + print("const char *good[] = {") + l = list(glob.glob("y_*")) + l.sort() + for f in l: + emit(f) + print("\tNULL") + print("};") + + print("const char *bad[] = {") + l = list(glob.glob("n_*")) + l.sort() + for f in l: + emit(f) + print("\tNULL") + print("};") */ static const char *good[] = { From phk at FreeBSD.org Fri Apr 20 10:22:19 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 20 Apr 2018 10:22:19 +0000 (UTC) Subject: [master] 1bb0688 Scope the scope-enum, so it doesn't clash with "VCL" type. Message-ID: <20180420102219.5323995412@lists.varnish-cache.org> commit 1bb0688c9a744e3f89fe0624f58c1ba0d4347867 Author: Poul-Henning Kamp Date: Fri Apr 20 10:19:46 2018 +0000 Scope the scope-enum, so it doesn't clash with "VCL" type. diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 168f15a..1d6877e 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -68,10 +68,10 @@ */ enum vmod_directors_shard_param_scope { _SCOPE_INVALID = 0, - VMOD, - VCL, - TASK, - STACK + SCOPE_VMOD, + SCOPE_VCL, + SCOPE_TASK, + SCOPE_STACK }; struct vmod_directors_shard_param; @@ -101,7 +101,7 @@ static const struct vmod_directors_shard_param shard_param_default = { .key = 0, .vcl_name = "builtin defaults", .defaults = NULL, - .scope = VMOD, + .scope = SCOPE_VMOD, .mask = _arg_mask_param, .by = BY_HASH, @@ -747,7 +747,7 @@ vmod_shard_param__init(VRT_CTX, ALLOC_OBJ(p, VMOD_SHARD_SHARD_PARAM_MAGIC); AN(p); p->vcl_name = vcl_name; - p->scope = VCL; + p->scope = SCOPE_VCL; p->defaults = &shard_param_default; *pp = p; @@ -778,7 +778,7 @@ shard_param_stack(struct vmod_directors_shard_param *p, AN(p); INIT_OBJ(p, VMOD_SHARD_SHARD_PARAM_MAGIC); p->vcl_name = who; - p->scope = STACK; + p->scope = SCOPE_STACK; p->defaults = pa; return (p); @@ -808,7 +808,7 @@ shard_param_task(VRT_CTX, const void *id, if (task->priv) { p = task->priv; CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); - assert(p->scope == TASK); + assert(p->scope = SCOPE_TASK); /* XXX VSL(SLT_Debug, 0, "shard_param_task(id %p, pa %p) = %p (found, ws=%p)", @@ -825,9 +825,9 @@ shard_param_task(VRT_CTX, const void *id, task->priv = p; INIT_OBJ(p, VMOD_SHARD_SHARD_PARAM_MAGIC); p->vcl_name = pa->vcl_name; - p->scope = TASK; + p->scope = SCOPE_TASK; - if (id == pa || pa->scope != VCL) + if (id == pa || pa->scope != SCOPE_VCL) p->defaults = pa; else p->defaults = shard_param_task(ctx, pa, pa); From nils.goroll at uplex.de Fri Apr 20 11:55:15 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 20 Apr 2018 11:55:15 +0000 (UTC) Subject: [master] 40555a0 fix VBF_Get_Filter_List for the no filter case Message-ID: <20180420115515.B7123971C5@lists.varnish-cache.org> commit 40555a04d44e3750634619f0479d344c1f2d1346 Author: Nils Goroll Date: Fri Apr 20 13:46:03 2018 +0200 fix VBF_Get_Filter_List for the no filter case If vbf_default_filter_list() returned without anything written to the vsb (for no body or length 0), we would return as the filter list whatever existed at ws->f + 1 prior to calling VBF_Get_Filter_List (for example the string logged with std.log()). Consequently, backend requests would fail with FetchError Filter '...' not found diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 487807c..f0b5fcf 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -580,8 +580,12 @@ VBF_Get_Filter_List(struct busyobj *bo) WS_MarkOverflow(bo->ws); return (NULL); } - WS_Release(bo->ws, VSB_len(vsb) + 1); - return (VSB_data(vsb) + 1); + if (VSB_len(vsb)) { + WS_Release(bo->ws, VSB_len(vsb) + 1); + return (VSB_data(vsb) + 1); + } + WS_Release(bo->ws, 0); + return (""); } static enum fetch_step From nils.goroll at uplex.de Fri Apr 20 12:15:14 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 20 Apr 2018 12:15:14 +0000 (UTC) Subject: [master] 6a70f4a fix lost workspace overflow markers Message-ID: <20180420121514.D0912977E7@lists.varnish-cache.org> commit 6a70f4aeb5bf6047affa77307522b0cf6f2dd7ba Author: Nils Goroll Date: Fri Apr 20 14:10:32 2018 +0200 fix lost workspace overflow markers As WS_Reset() clears the overflow marker, the correct order for resetting and marking an overflow is WS_Reset(); WS_MarkOverflow(); Fixes the last seemingly obscure bit of #2645 diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 52adc63..90d9e0d 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -411,8 +411,8 @@ VRT_l_beresp_storage_hint(VRT_CTX, const char *str, ...) if (p == NULL) { VSLb(ctx->vsl, SLT_LostHeader, "storage_hint"); - WS_MarkOverflow(ctx->ws); WS_Reset(ctx->ws, sn); + WS_MarkOverflow(ctx->ws); return; } diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index fdf1227..8651993 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -154,8 +154,8 @@ vmod_log(VRT_CTX, const char *fmt, ...) va_end(ap); if (p == NULL) { - WS_MarkOverflow(ctx->ws); WS_Reset(ctx->ws, sn); + WS_MarkOverflow(ctx->ws); return; } From nils.goroll at uplex.de Fri Apr 20 12:15:14 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 20 Apr 2018 12:15:14 +0000 (UTC) Subject: [master] 2e6178c Revert "be paranoid when marking workspace overflows" Message-ID: <20180420121514.E827E977EB@lists.varnish-cache.org> commit 2e6178c0b24ee025a5f4e0e02a644b6086f29973 Author: Nils Goroll Date: Fri Apr 20 14:13:08 2018 +0200 Revert "be paranoid when marking workspace overflows" No need to remain paranoid when the actual root case is dead simple (see previous commit) This reverts commit f5ce55168a2d31eb24e7ea2a12036d1dc489235c. Ref: #2645 diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index a6038de..3ec5ec0 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -31,7 +31,6 @@ #include "config.h" #include "cache_varnishd.h" -#include "vmb.h" #include @@ -116,7 +115,6 @@ WS_MarkOverflow(struct ws *ws) CHECK_OBJ_NOTNULL(ws, WS_MAGIC); ws->id[0] &= ~0x20; // cheesy toupper() - VWMB(); } static void From fgsch at lodoss.net Sat Apr 21 07:28:22 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 21 Apr 2018 07:28:22 +0000 (UTC) Subject: [master] 3e57151 Add a fuzzer for the ESI parser Message-ID: <20180421072822.1F1F2A0571@lists.varnish-cache.org> commit 3e57151e5e97aeea8834e21da6f924a5bc42c731 Author: Federico G. Schwindt Date: Sat Apr 21 08:17:41 2018 +0100 Add a fuzzer for the ESI parser This includes building the fuzzer to avoid code rot, but exercising the fuzzer will be done elsewhere. diff --git a/.gitignore b/.gitignore index 74f73ae..cdba66f 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,9 @@ cscope.*out /tools/vt_key /tools/vt_key.pub +# fuzzers +/bin/varnishd/esi_parse_fuzzer + # Coverity output /cov-int /myproject.tgz diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 9329f36..2556501 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -194,6 +194,16 @@ vhp_decode_test_CFLAGS = @SAN_CFLAGS@ \ vhp_decode_test_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.a +noinst_PROGRAMS += esi_parse_fuzzer +esi_parse_fuzzer_SOURCES = \ + cache/cache_esi_parse.c \ + fuzzers/esi_parse_fuzzer.c +esi_parse_fuzzer_CFLAGS = \ + @SAN_CFLAGS@ -DNOT_IN_A_VMOD -DTEST_DRIVER -include config.h +esi_parse_fuzzer_LDADD = \ + $(top_builddir)/lib/libvarnish/libvarnish.a \ + $(top_builddir)/lib/libvgz/libvgz.a + TESTS = vhp_table_test vhp_decode_test # diff --git a/bin/varnishd/fuzzers/esi_parse_fuzzer.c b/bin/varnishd/fuzzers/esi_parse_fuzzer.c new file mode 100644 index 0000000..056e636 --- /dev/null +++ b/bin/varnishd/fuzzers/esi_parse_fuzzer.c @@ -0,0 +1,121 @@ +/*- + * Copyright (c) 2018 Varnish Software AS + * All rights reserved. + * + * Author: Federico G. Schwindt + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * ESI parser fuzzer. + */ + +#include +#include +#include + +#include "cache/cache.h" +#include "cache/cache_vgz.h" /* enum vgz_flag */ +#include "cache/cache_esi.h" +#include "cache/cache_filter.h" /* struct vfp_ctx */ +#include "common/common_param.h" /* struct params */ + +#include "VSC_main.h" +#include "vfil.h" +#include "vsb.h" + +int LLVMFuzzerTestOneInput(const uint8_t *, size_t); + +struct VSC_main *VSC_C_main; +struct params *cache_param; + +void +VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) +{ + (void)vsl; + (void)tag; + (void)fmt; +} + +void * +WS_Alloc(struct ws *ws, unsigned bytes) +{ + (void)ws; + return (calloc(1, bytes)); +} + +int +LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) +{ + struct VSC_main __VSC_C_main; + struct params __cache_param; + struct http req = { .magic = HTTP_MAGIC }; + struct http resp = { .magic = HTTP_MAGIC }; + struct vfp_ctx vc = { .magic = VFP_CTX_MAGIC }; + struct vep_state *vep; + struct vsb *vsb; + struct worker wrk; + txt hd[HTTP_HDR_URL + 1]; + + if (size < 1) + return (0); + + VSC_C_main = &__VSC_C_main; + cache_param = &__cache_param; + + /* Zero out the esi feature bits for now */ + memset(&__cache_param, 0, sizeof(__cache_param)); + + /* Setup req */ + req.hd = hd; + req.hd[HTTP_HDR_URL].b = "/"; + + /* Setup vc */ + vc.wrk = &wrk; + vc.resp = &resp; + + vep = VEP_Init(&vc, &req, NULL, NULL); + AN(vep); + VEP_Parse(vep, (const char *)data, size); + vsb = VEP_Finish(vep); + if (vsb != NULL) + VSB_destroy(&vsb); + free(vep); + + return (0); +} + +#if defined(TEST_DRIVER) +int +main(int argc, char **argv) +{ + size_t len; + char *buf; + int i; + + for (i = 1; i < argc; i++) { + buf = VFIL_readfile(NULL, argv[i], &len); + AN(buf); + LLVMFuzzerTestOneInput(buf, len); + free(buf); + } +} +#endif From fgsch at lodoss.net Sat Apr 21 07:46:10 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 21 Apr 2018 07:46:10 +0000 (UTC) Subject: [master] 0604dba Fix previous Message-ID: <20180421074610.4ECA5A0B01@lists.varnish-cache.org> commit 0604dbad922ed9bb95f21e47e117e9386082f772 Author: Federico G. Schwindt Date: Sat Apr 21 08:41:11 2018 +0100 Fix previous diff --git a/bin/varnishd/fuzzers/esi_parse_fuzzer.c b/bin/varnishd/fuzzers/esi_parse_fuzzer.c index 056e636..0a372b7 100644 --- a/bin/varnishd/fuzzers/esi_parse_fuzzer.c +++ b/bin/varnishd/fuzzers/esi_parse_fuzzer.c @@ -107,14 +107,14 @@ LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) int main(int argc, char **argv) { - size_t len; + ssize_t len; char *buf; int i; for (i = 1; i < argc; i++) { buf = VFIL_readfile(NULL, argv[i], &len); AN(buf); - LLVMFuzzerTestOneInput(buf, len); + LLVMFuzzerTestOneInput((uint8_t *)buf, len); free(buf); } } From fgsch at lodoss.net Sat Apr 21 07:54:09 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 21 Apr 2018 07:54:09 +0000 (UTC) Subject: [master] 268bdfc Bump osx image Message-ID: <20180421075409.9336FA0F13@lists.varnish-cache.org> commit 268bdfc01dee25cebb3b1330c6fea795c1f4a561 Author: Federico G. Schwindt Date: Sat Apr 21 08:52:22 2018 +0100 Bump osx image diff --git a/.travis.yml b/.travis.yml index 2ff305c..8a741f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ matrix: compiler: clang env: CLANG=6.0 SAN_FLAGS="--enable-asan --enable-ubsan" - os: osx - osx_image: xcode9.2 + osx_image: xcode9.3 compiler: clang allow_failures: - os: osx From nils.goroll at uplex.de Sun Apr 22 10:05:19 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 22 Apr 2018 12:05:19 +0200 Subject: [master] 3e57151 Add a fuzzer for the ESI parser In-Reply-To: <20180421072822.1F1F2A0571@lists.varnish-cache.org> References: <20180421072822.1F1F2A0571@lists.varnish-cache.org> Message-ID: <943e5739-3cde-90e6-0a69-751ad2f35ab0@uplex.de> On 21/04/18 09:28, Federico G. Schwindt wrote: > This includes building the fuzzer to avoid code rot, but exercising > the fuzzer will be done elsewhere. This sounds like a cool thing with the potential to be really helpful :) Can you talk about the "elsewhere"? Thx, Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Sun Apr 22 10:25:39 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 22 Apr 2018 12:25:39 +0200 Subject: [master] 3e57151 Add a fuzzer for the ESI parser In-Reply-To: <943e5739-3cde-90e6-0a69-751ad2f35ab0@uplex.de> References: <20180421072822.1F1F2A0571@lists.varnish-cache.org> <943e5739-3cde-90e6-0a69-751ad2f35ab0@uplex.de> Message-ID: <79a5e6da-8007-54af-e92d-3961252eb68c@uplex.de> -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA256 BTW - the issue below has likely caused /tmp on the zone to run full, in which case we see all kinds of unrelated other errors (cleared daily by a restart) http://varnish-cache.org/vtest/bugs_distcheck.html#268bdfc 793 @ SunOS i86pc 5.11 1.04 32_sun12.4 uplex 792 @ SunOS i86pc 5.11 1.04 64_sun12.4 uplex CC varnishd-VSC_main.o CC varnishd-VSC_mempool.o CC varnishd-VSC_mgt.o CC varnishd-VSC_sma.o CC varnishd-VSC_smf.o CC varnishd-VSC_smu.o CC varnishd-VSC_vbe.o echo '/*' > builtin_vcl.c echo ' * NB: This file is machine generated, DO NOT EDIT!' >> builtin_vcl.c echo ' *' >> builtin_vcl.c echo ' * Edit builtin.vcl instead and run make' >> builtin_vcl.c CCLD vhp_table_test echo ' *' >> builtin_vcl.c echo ' */' >> builtin_vcl.c echo '#include "config.h"' >> builtin_vcl.c CCLD vhp_decode_test echo '#include "mgt/mgt.h"' >> builtin_vcl.c CCLD esi_parse_fuzzer echo '' >> builtin_vcl.c echo 'const char * const builtin_vcl =' >> builtin_vcl.c sed -e 's/"/\\"/g' \ - -e 's/$/\\n"/' \ - -e 's/^/ "/' ./builtin.vcl >> builtin_vcl.c echo ';' >> builtin_vcl.c CC varnishd-builtin_vcl.o Undefined first referenced symbol in file VSLb_ts esi_parse_fuzzer-cache_esi_parse.o ld: fatal: symbol referencing errors. No output written to esi_parse_fuzzer Makefile:978: recipe for target 'esi_parse_fuzzer' failed make[5]: *** [esi_parse_fuzzer] Error 2 make[5]: *** Waiting for unfinished jobs.... make[5]: Leaving directory '/tmp/vtest.64_sun12.4/varnish-cache/bin/varnishd' Makefile:884: recipe for target 'all' failed make[4]: *** [all] Error 2 make[4]: Leaving directory '/tmp/vtest.64_sun12.4/varnish-cache/bin/varnishd' Makefile:407: recipe for target 'all-recursive' failed make[3]: *** [all-recursive] Error 1 make[3]: Leaving directory '/tmp/vtest.64_sun12.4/varnish-cache/bin' Makefile:548: recipe for target 'all-recursive' failed -----BEGIN PGP SIGNATURE----- iQEzBAEBCAAdFiEEicdlqw+9Qa90PBmNHc2PV6OGi9cFAlrcYyEACgkQHc2PV6OG i9crHAf+MUrP/oTk9p1afxGlF+sVKFiGgiE+lsmdOGwWbhm7HvBjioexFazN/Bfq Za6b7TQJPsEFo4ZoRoRG0fPvrSN6BwB5wHDad86fX9t6db6WaTADRopiLuIZErES jEuSO6DGlRyli7TZvc548c9eOojuxowRrss2GblUmvlb8Eahm+GU/PeH8sYhTzR/ 0vQB9V0W9giI/ErQ9MGG5axk6Qi2YTRBjibWMkx1XP/qOqYX1TL/biQZ3EYS/biF wq/4U0oxGcd4IZ2eLhLzKNIxrxxIJJ3RmPDbDhN79iTPvbt3NnBZhy+b5Pqb1JU8 xvU3LU6vGv5pTB+JX/meTvfztYdk6g== =YziU -----END PGP SIGNATURE----- From fgsch at lodoss.net Sun Apr 22 21:16:16 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 22 Apr 2018 21:16:16 +0000 (UTC) Subject: [master] 88b2eb8 Plug more minor leaks Message-ID: <20180422211616.DC940A4969@lists.varnish-cache.org> commit 88b2eb8ebca45ecadc3370d55b73847ce1dba887 Author: Federico G. Schwindt Date: Sun Apr 22 17:07:42 2018 +0100 Plug more minor leaks diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 8da7c56..d814944 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -244,7 +244,10 @@ tst_cb(const struct vev *ve, int what) VEV_Stop(vb, jp->evt); free(jp->evt); } - + if (!jp->tst->ntodo) { + free(jp->tst->script); + FREE_OBJ(jp->tst); + } FREE_OBJ(jp); return (1); } diff --git a/tools/lsan.suppr b/tools/lsan.suppr index 89bc6c7..d08ee69 100644 --- a/tools/lsan.suppr +++ b/tools/lsan.suppr @@ -1,4 +1,6 @@ -leak:varnishtest +# varnishtest +leak:parse_string +leak:receive_frame # pp->{def,min,max} leak:MCF_ParamConf # pp->{def,min,max} From nils.goroll at uplex.de Sun Apr 22 22:57:14 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 22 Apr 2018 22:57:14 +0000 (UTC) Subject: [master] 39031d9 add a noop VSLb_ts to the fuzzer Message-ID: <20180422225714.6D444A64FD@lists.varnish-cache.org> commit 39031d9ba2395d4d27e72d3a3e204fbe99e2e1aa Author: Nils Goroll Date: Mon Apr 23 00:55:38 2018 +0200 add a noop VSLb_ts to the fuzzer suncc implements the inline functions from cache.h using it diff --git a/bin/varnishd/fuzzers/esi_parse_fuzzer.c b/bin/varnishd/fuzzers/esi_parse_fuzzer.c index 0a372b7..049d98a 100644 --- a/bin/varnishd/fuzzers/esi_parse_fuzzer.c +++ b/bin/varnishd/fuzzers/esi_parse_fuzzer.c @@ -55,6 +55,7 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) (void)fmt; } + void * WS_Alloc(struct ws *ws, unsigned bytes) { @@ -62,6 +63,17 @@ WS_Alloc(struct ws *ws, unsigned bytes) return (calloc(1, bytes)); } +void +VSLb_ts(struct vsl_log *l, const char *event, double first, double *pprev, + double now) +{ + (void)l; + (void)event; + (void)first; + (void)pprev; + (void)now; +} + int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { From fgsch at lodoss.net Mon Apr 23 17:48:17 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 23 Apr 2018 17:48:17 +0000 (UTC) Subject: [master] f6261d8 duh, travis-ci needs to join for this to work Message-ID: <20180423174817.69B37644CB@lists.varnish-cache.org> commit f6261d814030c0f1c86b444edabd4aa59960cf4f Author: Federico G. Schwindt Date: Mon Apr 23 13:06:51 2018 +0100 duh, travis-ci needs to join for this to work diff --git a/.travis.yml b/.travis.yml index 8a741f6..76a6f70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,6 @@ notifications: - "irc.linpro.no#varnish-hacking" on_success: change use_notice: true - skip_join: true before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; From fgsch at lodoss.net Mon Apr 23 17:48:17 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 23 Apr 2018 17:48:17 +0000 (UTC) Subject: [master] 282df44 Polish Message-ID: <20180423174817.5E9A46448A@lists.varnish-cache.org> commit 282df4408258dc30c4d85b14ee380fbc16f029ef Author: Federico G. Schwindt Date: Mon Apr 23 11:10:45 2018 +0100 Polish diff --git a/bin/varnishd/fuzzers/esi_parse_fuzzer.c b/bin/varnishd/fuzzers/esi_parse_fuzzer.c index 049d98a..c09c888 100644 --- a/bin/varnishd/fuzzers/esi_parse_fuzzer.c +++ b/bin/varnishd/fuzzers/esi_parse_fuzzer.c @@ -55,14 +55,6 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) (void)fmt; } - -void * -WS_Alloc(struct ws *ws, unsigned bytes) -{ - (void)ws; - return (calloc(1, bytes)); -} - void VSLb_ts(struct vsl_log *l, const char *event, double first, double *pprev, double now) @@ -74,6 +66,13 @@ VSLb_ts(struct vsl_log *l, const char *event, double first, double *pprev, (void)now; } +void * +WS_Alloc(struct ws *ws, unsigned bytes) +{ + (void)ws; + return (calloc(1, bytes)); +} + int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -90,6 +89,8 @@ LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) if (size < 1) return (0); + AN(data); + VSC_C_main = &__VSC_C_main; cache_param = &__cache_param; @@ -124,8 +125,8 @@ main(int argc, char **argv) int i; for (i = 1; i < argc; i++) { + len = 0; buf = VFIL_readfile(NULL, argv[i], &len); - AN(buf); LLVMFuzzerTestOneInput((uint8_t *)buf, len); free(buf); } From phk at FreeBSD.org Mon Apr 23 22:06:22 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Apr 2018 22:06:22 +0000 (UTC) Subject: [master] fd9664e Compress the admin health code a bit Message-ID: <20180423220622.A2BC093F27@lists.varnish-cache.org> commit fd9664eafd666e289275b807f7eb49dd563d73bd Author: Poul-Henning Kamp Date: Mon Apr 23 20:47:44 2018 +0000 Compress the admin health code a bit diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 3b6e4b7..bfc931a 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -47,10 +47,11 @@ struct vdi_ahealth { const char *name; + int health; }; -#define VBE_AHEALTH(l,u) \ - static const struct vdi_ahealth vdi_ah_##l[1] = {{#l}}; \ +#define VBE_AHEALTH(l,u,h) \ + static const struct vdi_ahealth vdi_ah_##l[1] = {{#l,h}}; \ const struct vdi_ahealth * const VDI_AH_##u = vdi_ah_##l; VBE_AHEALTH_LIST #undef VBE_AHEALTH @@ -58,7 +59,7 @@ VBE_AHEALTH_LIST static const struct vdi_ahealth * vdi_str2ahealth(const char *t) { -#define VBE_AHEALTH(l,u) if (!strcasecmp(t, #l)) return (VDI_AH_##u); +#define VBE_AHEALTH(l,u,h) if (!strcasecmp(t, #l)) return (VDI_AH_##u); VBE_AHEALTH_LIST #undef VBE_AHEALTH if (!strcasecmp(t, "auto")) return (VDI_AH_PROBE); @@ -265,23 +266,15 @@ unsigned VDI_Healthy(const struct director *d, double *changed) { CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + AN(d->admin_health); if (changed != NULL) *changed = d->health_changed; - if (d->admin_health == VDI_AH_PROBE) + if (d->admin_health->health < 0) return (d->health); - if (d->admin_health == VDI_AH_SICK) - return (0); - - if (d->admin_health == VDI_AH_DELETED) - return (0); - - if (d->admin_health == VDI_AH_HEALTHY) - return (1); - - WRONG("Wrong admin health"); + return (d->admin_health->health); } /*---------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 7db5a4b..9e98da7 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -97,13 +97,13 @@ void VCL_DelDirector(struct director *); /* cache_director.c */ -#define VBE_AHEALTH_LIST \ - VBE_AHEALTH(healthy, HEALTHY) \ - VBE_AHEALTH(sick, SICK) \ - VBE_AHEALTH(probe, PROBE) \ - VBE_AHEALTH(deleted, DELETED) +#define VBE_AHEALTH_LIST \ + VBE_AHEALTH(healthy, HEALTHY, 1) \ + VBE_AHEALTH(sick, SICK, 0) \ + VBE_AHEALTH(probe, PROBE, -1) \ + VBE_AHEALTH(deleted, DELETED, 0) -#define VBE_AHEALTH(l,u) extern const struct vdi_ahealth * const VDI_AH_##u; +#define VBE_AHEALTH(l,u,h) extern const struct vdi_ahealth * const VDI_AH_##u; VBE_AHEALTH_LIST #undef VBE_AHEALTH From phk at FreeBSD.org Mon Apr 23 22:06:22 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Apr 2018 22:06:22 +0000 (UTC) Subject: [master] 12a8790 Panic reporting of backend health is a director function Message-ID: <20180423220622.9CE6493F25@lists.varnish-cache.org> commit 12a87903f0761d4de4f013b671d05d8fc4b18911 Author: Poul-Henning Kamp Date: Mon Apr 23 20:32:37 2018 +0000 Panic reporting of backend health is a director function diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index acef648..610132a 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -413,11 +413,6 @@ vbe_panic(const struct director *d, struct vsb *vsb) VSB_printf(vsb, "ipv6 = %s,\n", bp->ipv6_addr); VSB_printf(vsb, "port = %s,\n", bp->port); VSB_printf(vsb, "hosthdr = %s,\n", bp->hosthdr); - VSB_printf(vsb, "health = %s,\n", - bp->director->health ? "healthy" : "sick"); - VSB_printf(vsb, "admin_health = %s, changed = %f,\n", - VDI_Ahealth(bp->director), - bp->director->health_changed); VSB_printf(vsb, "n_conn = %u,\n", bp->n_conn); } diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 30ae727..3b6e4b7 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -244,6 +244,9 @@ VDI_Panic(const struct director *d, struct vsb *vsb, const char *nm) VSB_printf(vsb, "%s = %p {\n", nm, d); VSB_indent(vsb, 2); VSB_printf(vsb, "vcl_name = %s,\n", d->vcl_name); + VSB_printf(vsb, "health = %s,\n", d->health ? "healthy" : "sick"); + VSB_printf(vsb, "admin_health = %s, changed = %f,\n", + VDI_Ahealth(d), d->health_changed); VSB_printf(vsb, "type = %s {\n", d->name); VSB_indent(vsb, 2); if (d->panic != NULL) From phk at FreeBSD.org Mon Apr 23 22:06:22 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Apr 2018 22:06:22 +0000 (UTC) Subject: [master] ce581db Call it "cli_name" instead of "display_name" Message-ID: <20180423220622.BEC5993F2A@lists.varnish-cache.org> commit ce581db32acf2f381ec4596c6a1d873779194b1c Author: Poul-Henning Kamp Date: Mon Apr 23 20:52:00 2018 +0000 Call it "cli_name" instead of "display_name" diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 610132a..90e9a4b 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -90,7 +90,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, if (!VDI_Healthy(bp->director, NULL)) { VSLb(bo->vsl, SLT_FetchError, - "backend %s: unhealthy", bp->director->display_name); + "backend %s: unhealthy", bp->director->cli_name); // XXX: per backend stats ? VSC_C_main->backend_unhealthy++; return (NULL); @@ -98,7 +98,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, if (bp->max_connections > 0 && bp->n_conn >= bp->max_connections) { VSLb(bo->vsl, SLT_FetchError, - "backend %s: busy", bp->director->display_name); + "backend %s: busy", bp->director->cli_name); // XXX: per backend stats ? VSC_C_main->backend_busy++; return (NULL); @@ -118,7 +118,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, if (pfd == NULL) { VSLb(bo->vsl, SLT_FetchError, "backend %s: fail errno %d (%s)", - bp->director->display_name, errno, strerror(errno)); + bp->director->cli_name, errno, strerror(errno)); // XXX: Per backend stats ? VSC_C_main->backend_fail++; bo->htc = NULL; @@ -141,7 +141,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, PFD_LocalName(pfd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); PFD_RemoteName(pfd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s", - *fdp, bp->director->display_name, abuf2, pbuf2, abuf1, pbuf1); + *fdp, bp->director->cli_name, abuf2, pbuf2, abuf1, pbuf1); INIT_OBJ(bo->htc, HTTP_CONN_MAGIC); bo->htc->priv = pfd; @@ -185,14 +185,14 @@ vbe_dir_finish(const struct director *d, struct worker *wrk, bo->htc->doclose == SC_RX_TIMEOUT); if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) { VSLb(bo->vsl, SLT_BackendClose, "%d %s", *PFD_Fd(pfd), - bp->director->display_name); + bp->director->cli_name); VTP_Close(&pfd); AZ(pfd); Lck_Lock(&bp->mtx); } else { assert (PFD_State(pfd) == PFD_STATE_USED); VSLb(bo->vsl, SLT_BackendReuse, "%d %s", *PFD_Fd(pfd), - bp->director->display_name); + bp->director->cli_name); Lck_Lock(&bp->mtx); VSC_C_main->backend_recycle++; VTP_Recycle(wrk, &pfd); @@ -406,7 +406,7 @@ vbe_panic(const struct director *d, struct vsb *vsb) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); - VSB_printf(vsb, "display_name = %s,\n", bp->director->display_name); + VSB_printf(vsb, "cli_name = %s,\n", bp->director->cli_name); if (bp->ipv4_addr != NULL) VSB_printf(vsb, "ipv4 = %s,\n", bp->ipv4_addr); if (bp->ipv6_addr != NULL) diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 478ad18..d180101 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -186,7 +186,7 @@ vbp_update_backend(struct vbp_target *vt) vt->backend->director->health = 0; } VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", - vt->backend->director->display_name, logmsg, bits, + vt->backend->director->cli_name, logmsg, bits, vt->good, vt->threshold, vt->window, vt->last, vt->avg, vt->resp_buf); VBE_SetHappy(vt->backend, vt->happy); diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index bfc931a..3ea2710 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -293,7 +293,7 @@ do_list(struct cli *cli, struct director *d, void *priv) if (d->admin_health == VDI_AH_DELETED) return (0); - VCLI_Out(cli, "\n%-30s", d->display_name); + VCLI_Out(cli, "\n%-30s", d->cli_name); VCLI_Out(cli, " %-10s", VDI_Ahealth(d)); diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 9e98da7..ba72d8b 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -80,7 +80,7 @@ struct director { /* Internal Housekeeping fields */ - char *display_name; + char *cli_name; VTAILQ_ENTRY(director) vcl_list; struct vcl *vcl; diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index c4d9ae1..2e17896 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -227,7 +227,7 @@ vcl_iterdir(struct cli *cli, const char *pat, const struct vcl *vcl, struct director *d; VTAILQ_FOREACH(d, &vcl->director_list, vcl_list) { - if (fnmatch(pat, d->display_name, 0)) + if (fnmatch(pat, d->cli_name, 0)) continue; found++; i = func(cli, d, priv); @@ -309,7 +309,7 @@ vcl_KillBackends(struct vcl *vcl) break; VTAILQ_REMOVE(&vcl->director_list, d, vcl_list); AN(d->destroy); - REPLACE(d->display_name, NULL); + REPLACE(d->cli_name, NULL); d->destroy(d); } } diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index 36bdf3f..ce94c75 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -138,7 +138,7 @@ VCL_AddDirector(struct vcl *vcl, struct director *d, const char *vcl_name) AN(vsb); VSB_printf(vsb, "%s.%s", VCL_Name(vcl), vcl_name); AZ(VSB_finish(vsb)); - REPLACE((d->display_name), VSB_data(vsb)); + REPLACE((d->cli_name), VSB_data(vsb)); VSB_destroy(&vsb); AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); @@ -179,7 +179,7 @@ VCL_DelDirector(struct director *d) VDI_Event(d, VCL_EVENT_COLD); AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); AN(d->destroy); - REPLACE(d->display_name, NULL); + REPLACE(d->cli_name, NULL); d->destroy(d); } From phk at FreeBSD.org Mon Apr 23 22:06:22 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Apr 2018 22:06:22 +0000 (UTC) Subject: [master] a99c21d whitespace ocd Message-ID: <20180423220622.DABEA93F2E@lists.varnish-cache.org> commit a99c21d76a4ef3854c8890ba008404c747b47fe7 Author: Poul-Henning Kamp Date: Mon Apr 23 21:10:07 2018 +0000 whitespace ocd diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index ba72d8b..91786ef 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -61,32 +61,32 @@ typedef void vdi_destroy_f(const struct director *); typedef void vdi_panic_f(const struct director *, struct vsb *); struct director { - unsigned magic; -#define DIRECTOR_MAGIC 0x3336351d - const char *name; - char *vcl_name; - vdi_http1pipe_f *http1pipe; - vdi_healthy_f *healthy; - vdi_resolve_f *resolve; - vdi_gethdrs_f *gethdrs; - vdi_getbody_f *getbody; - vdi_getip_f *getip; - vdi_finish_f *finish; - vdi_event_f *event; - vdi_destroy_f *destroy; - vdi_panic_f *panic; - void *priv; - const void *priv2; + unsigned magic; +#define DIRECTOR_MAGIC 0x3336351d + const char *name; + char *vcl_name; + vdi_http1pipe_f *http1pipe; + vdi_healthy_f *healthy; + vdi_resolve_f *resolve; + vdi_gethdrs_f *gethdrs; + vdi_getbody_f *getbody; + vdi_getip_f *getip; + vdi_finish_f *finish; + vdi_event_f *event; + vdi_destroy_f *destroy; + vdi_panic_f *panic; + void *priv; + const void *priv2; /* Internal Housekeeping fields */ - char *cli_name; - VTAILQ_ENTRY(director) vcl_list; - struct vcl *vcl; + char *cli_name; + VTAILQ_ENTRY(director) vcl_list; + struct vcl *vcl; - unsigned health; - const struct vdi_ahealth *admin_health; - double health_changed; + unsigned health; + const struct vdi_ahealth *admin_health; + double health_changed; }; unsigned VDI_Healthy(const struct director *, double *); From phk at FreeBSD.org Mon Apr 23 22:06:23 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Apr 2018 22:06:23 +0000 (UTC) Subject: [master] 4e701d6 Add a VRT_CTX arg to vdir_new(), we will need it Message-ID: <20180423220623.1AC2B93F31@lists.varnish-cache.org> commit 4e701d691e1b33922c9ce2cfb56bb2ff019746a6 Author: Poul-Henning Kamp Date: Mon Apr 23 21:27:57 2018 +0000 Add a VRT_CTX arg to vdir_new(), we will need it diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index 202431e..8ae3ddc 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -98,7 +98,7 @@ vmod_fallback__init(VRT_CTX, ALLOC_OBJ(fb, VMOD_DIRECTORS_FALLBACK_MAGIC); AN(fb); *fbp = fb; - vdir_new(&fb->vd, "fallback", vcl_name, vmod_fallback_healthy, + vdir_new(ctx, &fb->vd, "fallback", vcl_name, vmod_fallback_healthy, vmod_fallback_resolve, fb); fb->st = sticky; } diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index 80a792e..a053081 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -59,7 +59,7 @@ vmod_hash__init(VRT_CTX, struct vmod_directors_hash **rrp, ALLOC_OBJ(rr, VMOD_DIRECTORS_HASH_MAGIC); AN(rr); *rrp = rr; - vdir_new(&rr->vd, "hash", vcl_name, NULL, NULL, rr); + vdir_new(ctx, &rr->vd, "hash", vcl_name, NULL, NULL, rr); } VCL_VOID v_matchproto_() diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index 91fb3ce..4ce6328 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -86,7 +86,7 @@ vmod_random__init(VRT_CTX, struct vmod_directors_random **rrp, ALLOC_OBJ(rr, VMOD_DIRECTORS_RANDOM_MAGIC); AN(rr); *rrp = rr; - vdir_new(&rr->vd, "random", vcl_name, vmod_random_healthy, + vdir_new(ctx, &rr->vd, "random", vcl_name, vmod_random_healthy, vmod_random_resolve, rr); } diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index 3c896cb..7e888f3 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -95,7 +95,7 @@ vmod_round_robin__init(VRT_CTX, ALLOC_OBJ(rr, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); AN(rr); *rrp = rr; - vdir_new(&rr->vd, "round-robin", vcl_name, vmod_rr_healthy, + vdir_new(ctx, &rr->vd, "round-robin", vcl_name, vmod_rr_healthy, vmod_rr_resolve, rr); } diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 6a06412..95cc9df 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -50,11 +50,12 @@ vdir_expand(struct vdir *vd, unsigned n) } void -vdir_new(struct vdir **vdp, const char *name, const char *vcl_name, +vdir_new(VRT_CTX, struct vdir **vdp, const char *name, const char *vcl_name, vdi_healthy_f *healthy, vdi_resolve_f *resolve, void *priv) { struct vdir *vd; + AN(ctx); AN(name); AN(vcl_name); AN(vdp); diff --git a/lib/libvmod_directors/vdir.h b/lib/libvmod_directors/vdir.h index c4862e5..afdc421 100644 --- a/lib/libvmod_directors/vdir.h +++ b/lib/libvmod_directors/vdir.h @@ -41,7 +41,8 @@ struct vdir { struct vbitmap *vbm; }; -void vdir_new(struct vdir **vdp, const char *name, const char *vcl_name, +void vdir_new(VRT_CTX, struct vdir **vdp, const char *name, + const char *vcl_name, vdi_healthy_f *healthy, vdi_resolve_f *resolve, void *priv); void vdir_delete(struct vdir **vdp); void vdir_rdlock(struct vdir *vd); From phk at FreeBSD.org Mon Apr 23 22:06:23 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Apr 2018 22:06:23 +0000 (UTC) Subject: [master] cdc9884 Move Add/DelDirector into VRT, vmods will need to call them. Message-ID: <20180423220623.4CDE993F37@lists.varnish-cache.org> commit cdc9884aee1958f2eb37e5dacc6dac51ecaa1873 Author: Poul-Henning Kamp Date: Mon Apr 23 21:33:58 2018 +0000 Move Add/DelDirector into VRT, vmods will need to call them. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 90e9a4b..40b36e7 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -499,7 +499,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name); AN(be->vsc); - retval = VCL_AddDirector(ctx->vcl, d, vrt->vcl_name); + retval = VRT_AddDirector(ctx, d, vrt->vcl_name); if (retval == 0) return (d); @@ -571,7 +571,7 @@ VBE_Poll(void) if (be->n_conn > 0) continue; Lck_Unlock(&backends_mtx); - VCL_DelDirector(be->director); + VRT_DelDirector(NULL, be->director); Lck_Lock(&backends_mtx); } Lck_Unlock(&backends_mtx); diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 91786ef..c65118b 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -92,8 +92,8 @@ struct director { unsigned VDI_Healthy(const struct director *, double *); /* cache_vcl.c */ -int VCL_AddDirector(struct vcl *, struct director *, const char *); -void VCL_DelDirector(struct director *); +int VRT_AddDirector(VRT_CTX, struct director *, const char *); +void VRT_DelDirector(VRT_CTX, struct director *); /* cache_director.c */ diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index ce94c75..ca45c76 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -126,13 +126,16 @@ VCL_Rel(struct vcl **vcc) /*--------------------------------------------------------------------*/ int -VCL_AddDirector(struct vcl *vcl, struct director *d, const char *vcl_name) +VRT_AddDirector(VRT_CTX, struct director *d, const char *vcl_name) { struct vsb *vsb; + struct vcl *vcl; + vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); AN(d->destroy); + d->admin_health = VDI_AH_PROBE; vsb = VSB_new_auto(); AN(vsb); @@ -163,10 +166,11 @@ VCL_AddDirector(struct vcl *vcl, struct director *d, const char *vcl_name) } void -VCL_DelDirector(struct director *d) +VRT_DelDirector(VRT_CTX, struct director *d) { struct vcl *vcl; + (void)ctx; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); vcl = d->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); From nils.goroll at uplex.de Tue Apr 24 06:22:21 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 24 Apr 2018 06:22:21 +0000 (UTC) Subject: [master] 1e82f50 Complement VDP error handling Message-ID: <20180424062221.33574A3344@lists.varnish-cache.org> commit 1e82f502a027b89caa94be88496b20aaf3970c8d Author: Nils Goroll Date: Mon Apr 23 11:25:57 2018 +0200 Complement VDP error handling * remember push errors in the context * also fail additional pushes once we saw an error Try to fail requests as early as possible. Otherwise just calling VDP functions after an error is a noop. Fixes #2618 diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 20439b1..e3cdcee 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -73,7 +73,7 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) return (vdc->retval); } -void +int VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) { struct vdp_entry *vdpe; @@ -85,12 +85,18 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) AN(vdp->name); AN(vdp->func); + if (vdc->retval) + return (vdc->retval); + if (DO_DEBUG(DBG_PROCESSORS)) VSLb(req->vsl, SLT_Debug, "VDP_push(%s)", vdp->name); vdpe = WS_Alloc(req->ws, sizeof *vdpe); - if (vdpe == NULL) - return; + if (vdpe == NULL) { + AZ(vdc->retval); + vdc->retval = -1; + return (vdc->retval); + } INIT_OBJ(vdpe, VDP_ENTRY_MAGIC); vdpe->vdp = vdp; vdpe->priv = priv; @@ -100,7 +106,9 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) VTAILQ_INSERT_HEAD(&vdc->vdp, vdpe, list); vdc->nxt = VTAILQ_FIRST(&vdc->vdp); - AZ(vdpe->vdp->func(req, VDP_INIT, &vdpe->priv, NULL, 0)); + AZ(vdc->retval); + vdc->retval = vdpe->vdp->func(req, VDP_INIT, &vdpe->priv, NULL, 0); + return (vdc->retval); } void @@ -113,10 +121,15 @@ VDP_close(struct req *req) vdc = req->vdc; while (!VTAILQ_EMPTY(&vdc->vdp)) { vdpe = VTAILQ_FIRST(&vdc->vdp); - CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC); - VTAILQ_REMOVE(&vdc->vdp, vdpe, list); - AZ(vdpe->vdp->func(req, VDP_FINI, &vdpe->priv, NULL, 0)); - AZ(vdpe->priv); + if (vdc->retval >= 0) + AN(vdpe); + if (vdpe != NULL) { + CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC); + VTAILQ_REMOVE(&vdc->vdp, vdpe, list); + AZ(vdpe->vdp->func(req, VDP_FINI, &vdpe->priv, + NULL, 0)); + AZ(vdpe->priv); + } vdc->nxt = VTAILQ_FIRST(&vdc->vdp); } } diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 3af6985..1829804 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -815,9 +815,9 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) ved_stripgzip(req, boc); } else { if (ecx->isgzip && !i) - VDP_push(req, &ved_vdp_pgz, ecx, 1); + (void)VDP_push(req, &ved_vdp_pgz, ecx, 1); else - VDP_push(req, &ved_ved, ecx->preq, 1); + (void)VDP_push(req, &ved_ved, ecx->preq, 1); (void)VDP_DeliverObj(req); (void)VDP_bytes(req, VDP_FLUSH, NULL, 0); } diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 3192238..b66afb0 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -123,8 +123,8 @@ struct vdp_ctx { #define VDP_CTX_MAGIC 0xee501df7 struct vdp_entry_s vdp; struct vdp_entry *nxt; - unsigned retval; + int retval; }; int VDP_bytes(struct req *, enum vdp_action act, const void *ptr, ssize_t len); -void VDP_push(struct req *, const struct vdp *, void *priv, int bottom); +int VDP_push(struct req *, const struct vdp *, void *priv, int bottom); diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index acd7034..373d1f6 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -175,7 +175,8 @@ vrg_dorange(struct req *req, const char *r) vrg_priv->range_off = 0; vrg_priv->range_low = low; vrg_priv->range_high = high + 1; - VDP_push(req, &vrg_vdp, vrg_priv, 1); + if (VDP_push(req, &vrg_vdp, vrg_priv, 1)) + return ("WS too small"); http_PutResponse(req->resp, "HTTP/1.1", 206, NULL); return (NULL); } diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index e327c0d..00a0249 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -342,7 +342,7 @@ cnt_transmit(struct worker *wrk, struct req *req) struct boc *boc; const char *r; uint16_t status; - int sendbody; + int err, sendbody; intmax_t clval; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -375,15 +375,18 @@ cnt_transmit(struct worker *wrk, struct req *req) } else sendbody = 1; + err = 0; if (sendbody >= 0) { if (!req->disable_esi && req->resp_len != 0 && - ObjHasAttr(wrk, req->objcore, OA_ESIDATA)) - VDP_push(req, &VDP_esi, NULL, 0); + ObjHasAttr(wrk, req->objcore, OA_ESIDATA) && + VDP_push(req, &VDP_esi, NULL, 0) < 0) + err++; if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && - !RFC2616_Req_Gzip(req->http)) - VDP_push(req, &VDP_gunzip, NULL, 1); + !RFC2616_Req_Gzip(req->http) && + VDP_push(req, &VDP_gunzip, NULL, 1) < 0) + err++; if (cache_param->http_range_support && http_IsStatus(req->resp, 200)) { @@ -405,7 +408,12 @@ cnt_transmit(struct worker *wrk, struct req *req) "Content-Length: %jd", req->resp_len); } - req->transport->deliver(req, boc, sendbody); + if (err == 0) + req->transport->deliver(req, boc, sendbody); + else { + VSLb(req->vsl, SLT_Error, "Failure to push processors"); + req->doclose = SC_OVERLOAD; + } VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index c7a6226..fda38dd 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -128,8 +128,11 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) if (req->resp_len == 0) sendbody = 0; - if (sendbody) - VDP_push(req, &v1d_vdp, NULL, 1); + if (sendbody && VDP_push(req, &v1d_vdp, NULL, 1)) { + v1d_error(req, "workspace_thread overflow"); + AZ(req->wrk->v1l); + return; + } AZ(req->wrk->v1l); V1L_Open(req->wrk, req->wrk->aws, diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index a9f43e9..a0ff969 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -265,10 +265,13 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) WS_Release(req->ws, 0); - if (sendbody) { - VDP_push(req, &h2_vdp, NULL, 1); + /* XXX someone into H2 please add appropriate error handling */ + while (sendbody) { + err = VDP_push(req, &h2_vdp, NULL, 1); + if (err) + break; err = VDP_DeliverObj(req); - /*XXX*/(void)err; + break; } AZ(req->wrk->v1l); diff --git a/bin/varnishtest/tests/r02618.vtc b/bin/varnishtest/tests/r02618.vtc new file mode 100644 index 0000000..efe1a23 --- /dev/null +++ b/bin/varnishtest/tests/r02618.vtc @@ -0,0 +1,25 @@ +varnishtest "sweep through tight client workspace conditions in deliver" + +server s1 { + rxreq + txresp -hdr "Cache-Control: mag-age=3600" -bodylen 1024 +} -start + +varnish v1 -vcl+backend { + import vtc; + import std; + sub vcl_recv { + return (hash); + } + sub vcl_deliver { + vtc.workspace_alloc(client, -4 * + (std.integer(req.xid, 1001) - 1001) / 2); + } +} -start + +client c1 -repeat 100 { + txreq -url "/" + # some responses will fail (503), some won't. All we care + # about here is the fact that we don't panic + rxresp +} -run From hermunn at varnish-software.com Tue Apr 24 09:58:17 2018 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Tue, 24 Apr 2018 09:58:17 +0000 (UTC) Subject: [4.1] 26d5261 Fix memory leak of vary string on stevedore alloc fail Message-ID: <20180424095817.C9FDFA7201@lists.varnish-cache.org> commit 26d52615004730eb28ce603930600d44cafd8202 Author: Martin Blix Grydeland Date: Wed Mar 14 13:52:36 2018 +0100 Fix memory leak of vary string on stevedore alloc fail If the stevedore failed the object creation, we would leak the temporary VSB holding the computed vary string. This patch frees it. Problem exists in 4.1 and later. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index b356828..902ad17 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -125,8 +125,12 @@ vbf_beresp2obj(struct busyobj *bo) if (bo->uncacheable) bo->fetch_objcore->flags |= OC_F_PASS; - if (!vbf_allocobj(bo, l)) + if (!vbf_allocobj(bo, l)) { + if (vary != NULL) + VSB_destroy(&vary); + AZ(vary); return (-1); + } if (vary != NULL) { b = ObjSetattr(bo->wrk, bo->fetch_objcore, OA_VARY, varyl, From nils.goroll at uplex.de Tue Apr 24 10:07:12 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 24 Apr 2018 10:07:12 +0000 (UTC) Subject: [master] 47feebc add git Message-ID: <20180424100712.87767A7501@lists.varnish-cache.org> commit 47feebce4506342d5da7f7716ebec3fca37b47fc Author: Nils Goroll Date: Tue Apr 24 12:06:04 2018 +0200 add git diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index 101db8e..896d29c 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -100,6 +100,10 @@ Recommended, in particular if you plan on building custom vmods:: sudo apt-get install autoconf-archive +Optionally, to pull from a repository:: + + sudo apt-get install git + Build dependencies on Red Hat / CentOS -------------------------------------- @@ -125,6 +129,10 @@ Optionally, to rebuild the svg files:: yum install graphviz +Optionally, to pull from a repository:: + + yum install git + .. XXX autoconf-archive ? is this any helpful on the notoriously outdated Redhats? From hermunn at varnish-software.com Tue Apr 24 13:28:16 2018 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Tue, 24 Apr 2018 13:28:16 +0000 (UTC) Subject: [4.1] 618c3bc Update changelog: #2609 Message-ID: <20180424132816.15A4EAD05F@lists.varnish-cache.org> commit 618c3bc5e9c02544ab7df92d31197dd197585621 Author: P?l Hermunn Johansen Date: Tue Apr 24 14:53:47 2018 +0200 Update changelog: #2609 diff --git a/doc/changes.rst b/doc/changes.rst index cee7a79..e276414 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -25,6 +25,7 @@ Bugs fixed * 2502_ - objcore reference count leak * 2530_ - Varnish shouldn't test gunzip for range responses * 2582_ - Assert error in http1_minimal_response() +* 2609_ - Fix memory leak of vary string on stevedore alloc fail .. _1772: https://github.com/varnishcache/varnish-cache/issues/1772 .. _2135: https://github.com/varnishcache/varnish-cache/pull/2135 @@ -32,6 +33,7 @@ Bugs fixed .. _2502: https://github.com/varnishcache/varnish-cache/issues/2502 .. _2530: https://github.com/varnishcache/varnish-cache/issues/2530 .. _2582: https://github.com/varnishcache/varnish-cache/issues/2582 +.. _2609: https://github.com/varnishcache/varnish-cache/pull/2609 ================================ Varnish Cache 4.1.9 (2017-11-14) From phk at FreeBSD.org Tue Apr 24 20:53:17 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 24 Apr 2018 20:53:17 +0000 (UTC) Subject: [master] 579a1df Add a "list" method to directors, to participate in CLI::backend.list Message-ID: <20180424205317.77542B50C6@lists.varnish-cache.org> commit 579a1df977693effa2fe545a667f60eea960185b Author: Poul-Henning Kamp Date: Tue Apr 24 20:51:18 2018 +0000 Add a "list" method to directors, to participate in CLI::backend.list Add a new -v flag, just because we can. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 40b36e7..0235194 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -417,6 +417,23 @@ vbe_panic(const struct director *d, struct vsb *vsb) } /*-------------------------------------------------------------------- + */ + +static void +vbe_list(const struct director *d, struct vsb *vsb, int vflag, int pflag) +{ + struct backend *bp; + + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); + + if (bp->probe != NULL) + VBP_Status(vsb, bp, vflag | pflag); + else + VSB_printf(vsb, "%-10s", d->health ? "healthy" : "sick"); +} + +/*-------------------------------------------------------------------- * Create a new static or dynamic director::backend instance. */ @@ -473,6 +490,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, d->finish = vbe_dir_finish; d->event = vbe_dir_event; d->panic = vbe_panic; + d->list = vbe_list; d->destroy = vbe_destroy; d->health = 1; diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index b0a0442..5f8eca1 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -81,4 +81,4 @@ void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, struct tcp_pool *); void VBP_Remove(struct backend *b); void VBP_Control(const struct backend *b, int stop); -void VBP_Status(struct cli *cli, const struct backend *, int details); +void VBP_Status(struct vsb *, const struct backend *, int details); diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index d180101..caf3673 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -168,9 +168,9 @@ vbp_update_backend(struct vbp_target *vt) assert(i < sizeof bits); if (vt->good >= vt->threshold) { - if (vt->backend->director->health) + if (vt->backend->director->health) { logmsg = "Still healthy"; - else { + } else { logmsg = "Back healthy"; vt->backend->director->health_changed = VTIM_real(); @@ -181,8 +181,9 @@ vbp_update_backend(struct vbp_target *vt) logmsg = "Went sick"; vt->backend->director->health_changed = VTIM_real(); - } else + } else { logmsg = "Still sick"; + } vt->backend->director->health = 0; } VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", @@ -458,58 +459,56 @@ vbp_thread(struct worker *wrk, void *priv) */ static void -vbp_bitmap(struct cli *cli, char c, uint64_t map, const char *lbl) +vbp_bitmap(struct vsb *vsb, char c, uint64_t map, const char *lbl) { int i; uint64_t u = (1ULL << 63); - VCLI_Out(cli, " "); + VSB_printf(vsb, " "); for (i = 0; i < 64; i++) { if (map & u) - VCLI_Out(cli, "%c", c); + VSB_putc(vsb, c); else - VCLI_Out(cli, "-"); + VSB_putc(vsb, '-'); map <<= 1; } - VCLI_Out(cli, " %s\n", lbl); + VSB_printf(vsb, " %s\n", lbl); } /*lint -e{506} constant value boolean */ /*lint -e{774} constant value boolean */ -static void -vbp_health_one(struct cli *cli, const struct vbp_target *vt) +void +VBP_Status(struct vsb *vsb, const struct backend *be, int details) { + struct vbp_target *vt; + char buf[12]; + + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); + vt = be->probe; + CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); + + if (!details) { + bprintf(buf, "%d/%d %s", vt->good, vt->window, + vt->backend->director->health ? "good" : "bad"); + VSB_printf(vsb, "%-10s", buf); + return; + } - VCLI_Out(cli, - " Current states good: %2u threshold: %2u window: %2u\n", + VSB_printf(vsb, + "\nCurrent states good: %2u threshold: %2u window: %2u\n", vt->good, vt->threshold, vt->window); - VCLI_Out(cli, + VSB_printf(vsb, " Average response time of good probes: %.6f\n", vt->avg); - VCLI_Out(cli, + VSB_printf(vsb, " Oldest ======================" "============================ Newest\n"); #define BITMAP(n, c, t, b) \ if ((vt->n != 0) || (b)) \ - vbp_bitmap(cli, (c), vt->n, (t)); + vbp_bitmap(vsb, (c), vt->n, (t)); #include "tbl/backend_poll.h" } -void -VBP_Status(struct cli *cli, const struct backend *be, int details) -{ - struct vbp_target *vt; - - CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - vt = be->probe; - CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); - VCLI_Out(cli, "%d/%d", vt->good, vt->window); - if (details) { - VCLI_Out(cli, "\n"); - vbp_health_one(cli, vt); - } -} - /*-------------------------------------------------------------------- * Build request from probe spec */ diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 3ea2710..cd99f1a 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -38,7 +38,6 @@ #include "cache_varnishd.h" #include "cache_director.h" -#include "cache_backend.h" #include "vcli_serve.h" #include "vtim.h" @@ -279,62 +278,69 @@ VDI_Healthy(const struct director *d, double *changed) /*---------------------------------------------------------------------*/ +struct list_args { + unsigned magic; +#define LIST_ARGS_MAGIC 0x7e7cefeb + int p; + int v; +}; + static int v_matchproto_(vcl_be_func) do_list(struct cli *cli, struct director *d, void *priv) { - int *probes; char time_str[VTIM_FORMAT_SIZE]; - struct backend *be; + struct list_args *la; - AN(priv); - probes = priv; + CAST_OBJ_NOTNULL(la, priv, LIST_ARGS_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC); + if (d->admin_health == VDI_AH_DELETED) return (0); - VCLI_Out(cli, "\n%-30s", d->cli_name); - - VCLI_Out(cli, " %-10s", VDI_Ahealth(d)); + VCLI_Out(cli, "\n%-30s %-7s ", d->cli_name, VDI_Ahealth(d)); - if (be->probe == NULL) - VCLI_Out(cli, " %-20s", "Healthy (no probe)"); - else { - if (d->health) - VCLI_Out(cli, " %-20s", "Healthy "); - else - VCLI_Out(cli, " %-20s", "Sick "); - VBP_Status(cli, be, *probes); - } + if (d->list != NULL) + d->list(d, cli->sb, 0, 0); + else + VCLI_Out(cli, "%-10s", d->health ? "healthy" : "sick"); VTIM_format(d->health_changed, time_str); VCLI_Out(cli, " %s", time_str); - + if (la->p || la->v) + d->list(d, cli->sb, la->p, la->v); return (0); } static void v_matchproto_(cli_func_t) cli_backend_list(struct cli *cli, const char * const *av, void *priv) { - int probes = 0; + const char *p; + struct list_args la[1]; (void)priv; ASSERT_CLI(); - if (av[2] != NULL && !strcmp(av[2], "-p")) { + INIT_OBJ(la, LIST_ARGS_MAGIC); + while (av[2] != NULL && av[2][0] == '-') { + for(p = av[2] + 1; *p; p++) { + switch(*p) { + case 'p': la->p = !la->p; break; + case 'v': la->p = !la->p; break; + default: + VCLI_Out(cli, "Invalid flag %c", *p); + VCLI_SetResult(cli, CLIS_PARAM); + return; + } + } av++; - probes = 1; - } else if (av[2] != NULL && av[2][0] == '-') { - VCLI_Out(cli, "Invalid flags %s", av[2]); - VCLI_SetResult(cli, CLIS_PARAM); - return; - } else if (av[3] != NULL) { + } + if (av[3] != NULL) { VCLI_Out(cli, "Too many arguments"); VCLI_SetResult(cli, CLIS_PARAM); return; } - VCLI_Out(cli, "%-30s %-10s %-20s %s", "Backend name", "Admin", - "Probe", "Last updated"); - (void)VCL_IterDirector(cli, av[2], do_list, &probes); + VCLI_Out(cli, "%-30s %-7s %-10s %s", + "Backend name", "Admin", "Probe", "Last change"); + (void)VCL_IterDirector(cli, av[2], do_list, la); } /*---------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index c65118b..5a9cb60 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -60,6 +60,8 @@ typedef void vdi_destroy_f(const struct director *); typedef void vdi_panic_f(const struct director *, struct vsb *); +typedef void vdi_list_f(const struct director *, struct vsb *, int, int); + struct director { unsigned magic; #define DIRECTOR_MAGIC 0x3336351d @@ -75,6 +77,8 @@ struct director { vdi_event_f *event; vdi_destroy_f *destroy; vdi_panic_f *panic; + vdi_list_f *list; + void *priv; const void *priv2; diff --git a/bin/varnishtest/tests/o00004.vtc b/bin/varnishtest/tests/o00004.vtc index f51bcc6..6aa345c 100644 --- a/bin/varnishtest/tests/o00004.vtc +++ b/bin/varnishtest/tests/o00004.vtc @@ -49,5 +49,5 @@ varnish v2 -vcl { server s1 -wait delay 1 -varnish v2 -cliexpect "vcl1.bp1[ ]+probe[ ]+Healthy[ ]+1/1" backend.list -varnish v2 -cliexpect "vcl1.bp2[ ]+probe[ ]+Healthy[ ]+1/1" backend.list +varnish v2 -cliexpect "vcl1.bp1[ ]+probe[ ]+1/1[ ]+good" backend.list +varnish v2 -cliexpect "vcl1.bp2[ ]+probe[ ]+1/1[ ]+good" backend.list From nils.goroll at uplex.de Wed Apr 25 07:36:29 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 25 Apr 2018 07:36:29 +0000 (UTC) Subject: [master] 9e8e267 track previous commit in the changelog Message-ID: <20180425073629.EF6599356A@lists.varnish-cache.org> commit 9e8e267d279bafaa51c891742021219b6b31e85d Author: Nils Goroll Date: Wed Apr 25 09:35:16 2018 +0200 track previous commit in the changelog diff --git a/doc/changes.rst b/doc/changes.rst index a8dd755..50d5c82 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -2,6 +2,13 @@ Varnish Cache Trunk (ongoing) ============================= +varnishadm +---------- + +* The output format of the ``backend.list`` CLI command has been + changed. + + VCL and bundled VMODs --------------------- From nils.goroll at uplex.de Wed Apr 25 07:47:09 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 25 Apr 2018 07:47:09 +0000 (UTC) Subject: [master] e272b23 changelog Message-ID: <20180425074709.B80A2938B6@lists.varnish-cache.org> commit e272b236577cd083942cc09b97eda9dc9bbb0456 Author: Nils Goroll Date: Wed Apr 25 09:46:23 2018 +0200 changelog diff --git a/doc/changes.rst b/doc/changes.rst index 50d5c82..4c4db9b 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -14,6 +14,9 @@ VCL and bundled VMODs * VMOD unix now supports the ``getpeerucred(3)`` case. +* Fetch Processors (VFPs) can now be configured from VCL via + ``beresp.filters`` + bundled tools ------------- @@ -21,6 +24,13 @@ bundled tools for custom profile definitions to also contain a prefix to match the tag against. +C APIs (for vmod and utility authors) +------------------------------------- + +* XXX VFP + +* XXX backend add / del + ================================ Varnish Cache 6.0.0 (2018-03-15) ================================ From phk at FreeBSD.org Wed Apr 25 08:27:16 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 25 Apr 2018 08:27:16 +0000 (UTC) Subject: [master] 4f3ed52 Add an intersticial struct between vcl and directors Message-ID: <20180425082716.DF2AC9454E@lists.varnish-cache.org> commit 4f3ed523166cf41ac84fb52d27fc059d5229d3de Author: Poul-Henning Kamp Date: Wed Apr 25 06:50:41 2018 +0000 Add an intersticial struct between vcl and directors diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 5a9cb60..43e7e4b 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -37,6 +37,8 @@ * */ +struct vcldir; + typedef unsigned vdi_healthy_f(const struct director *, const struct busyobj *, double *changed); /* XXX need a VRT_CTX argument */ @@ -84,9 +86,9 @@ struct director { /* Internal Housekeeping fields */ + struct vcldir *vdir; + char *cli_name; - VTAILQ_ENTRY(director) vcl_list; - struct vcl *vcl; unsigned health; const struct vdi_ahealth *admin_health; diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 2e17896..7ca458c 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -224,13 +224,13 @@ vcl_iterdir(struct cli *cli, const char *pat, const struct vcl *vcl, vcl_be_func *func, void *priv) { int i, found = 0; - struct director *d; + struct vcldir *vdir; - VTAILQ_FOREACH(d, &vcl->director_list, vcl_list) { - if (fnmatch(pat, d->cli_name, 0)) + VTAILQ_FOREACH(vdir, &vcl->director_list, list) { + if (fnmatch(pat, vdir->dir->cli_name, 0)) continue; found++; - i = func(cli, d, priv); + i = func(cli, vdir->dir, priv); if (i < 0) return (i); found += i; @@ -285,32 +285,33 @@ VCL_IterDirector(struct cli *cli, const char *pat, static void vcl_BackendEvent(const struct vcl *vcl, enum vcl_event_e e) { - struct director *d; + struct vcldir *vdir; ASSERT_CLI(); CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); AZ(vcl->busy); - VTAILQ_FOREACH(d, &vcl->director_list, vcl_list) - VDI_Event(d, e); + VTAILQ_FOREACH(vdir, &vcl->director_list, list) + VDI_Event(vdir->dir, e); } static void vcl_KillBackends(struct vcl *vcl) { - struct director *d; + struct vcldir *vdir; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); AZ(vcl->busy); assert(VTAILQ_EMPTY(&vcl->ref_list)); while (1) { - d = VTAILQ_FIRST(&vcl->director_list); - if (d == NULL) + vdir = VTAILQ_FIRST(&vcl->director_list); + if (vdir == NULL) break; - VTAILQ_REMOVE(&vcl->director_list, d, vcl_list); - AN(d->destroy); - REPLACE(d->cli_name, NULL); - d->destroy(d); + VTAILQ_REMOVE(&vcl->director_list, vdir, list); + REPLACE(vdir->dir->cli_name, NULL); + AN(vdir->dir->destroy); + vdir->dir->destroy(vdir->dir); + FREE_OBJ(vdir); } } diff --git a/bin/varnishd/cache/cache_vcl.h b/bin/varnishd/cache/cache_vcl.h index f0789a8..e5a5453 100644 --- a/bin/varnishd/cache/cache_vcl.h +++ b/bin/varnishd/cache/cache_vcl.h @@ -35,6 +35,14 @@ struct vfp_filter; VTAILQ_HEAD(vfp_filter_head, vfp_filter); +struct vcldir { + unsigned magic; +#define VCLDIR_MAGIC 0xbf726c7d + struct director *dir; + struct vcl *vcl; + VTAILQ_ENTRY(vcldir) list; +}; + struct vcl { unsigned magic; #define VCL_MAGIC 0x214188f2 @@ -47,7 +55,7 @@ struct vcl { unsigned discard; const char *temp; pthread_rwlock_t temp_rwl; - VTAILQ_HEAD(,director) director_list; + VTAILQ_HEAD(,vcldir) director_list; VTAILQ_HEAD(,vclref) ref_list; int nrefs; struct vcl *label; diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index ca45c76..0848a56 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -130,13 +130,20 @@ VRT_AddDirector(VRT_CTX, struct director *d, const char *vcl_name) { struct vsb *vsb; struct vcl *vcl; + struct vcldir *vdir; + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); AN(d->destroy); - d->admin_health = VDI_AH_PROBE; + AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); + if (vcl->temp == VCL_TEMP_COOLING) { + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); + return (1); + } + + d->admin_health = VDI_AH_PROBE; vsb = VSB_new_auto(); AN(vsb); VSB_printf(vsb, "%s.%s", VCL_Name(vcl), vcl_name); @@ -144,15 +151,14 @@ VRT_AddDirector(VRT_CTX, struct director *d, const char *vcl_name) REPLACE((d->cli_name), VSB_data(vsb)); VSB_destroy(&vsb); - AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); - if (vcl->temp == VCL_TEMP_COOLING) { - AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - return (1); - } + ALLOC_OBJ(vdir, VCLDIR_MAGIC); + AN(vdir); + vdir->dir = d; + vdir->vcl = vcl; + d->vdir = vdir; Lck_Lock(&vcl_mtx); - VTAILQ_INSERT_TAIL(&vcl->director_list, d, vcl_list); - d->vcl = vcl; + VTAILQ_INSERT_TAIL(&vcl->director_list, vdir, list); Lck_Unlock(&vcl_mtx); if (VCL_WARM(vcl)) @@ -169,13 +175,16 @@ void VRT_DelDirector(VRT_CTX, struct director *d) { struct vcl *vcl; + struct vcldir *vdir; (void)ctx; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - vcl = d->vcl; + vdir = d->vdir; + CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC); + vcl = vdir->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); Lck_Lock(&vcl_mtx); - VTAILQ_REMOVE(&vcl->director_list, d, vcl_list); + VTAILQ_REMOVE(&vcl->director_list, vdir, list); Lck_Unlock(&vcl_mtx); AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); @@ -185,6 +194,7 @@ VRT_DelDirector(VRT_CTX, struct director *d) AN(d->destroy); REPLACE(d->cli_name, NULL); d->destroy(d); + FREE_OBJ(vdir); } /*--------------------------------------------------------------------*/ From phk at FreeBSD.org Wed Apr 25 08:27:16 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 25 Apr 2018 08:27:16 +0000 (UTC) Subject: [master] a11939e Sequester the director methods into their own structure. Message-ID: <20180425082716.F0C2994550@lists.varnish-cache.org> commit a11939eb64427847ec11571d328696ba162f8c6a Author: Poul-Henning Kamp Date: Wed Apr 25 08:25:48 2018 +0000 Sequester the director methods into their own structure. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 0235194..7ca57d5 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -434,6 +434,23 @@ vbe_list(const struct director *d, struct vsb *vsb, int vflag, int pflag) } /*-------------------------------------------------------------------- + */ + +static const struct director_methods vbe_methods[1] = {{ + .magic = DIRECTOR_METHODS_MAGIC, + .type = "backend", + .http1pipe = vbe_dir_http1pipe, + .healthy = vbe_dir_healthy, + .gethdrs = vbe_dir_gethdrs, + .getip = vbe_dir_getip, + .finish = vbe_dir_finish, + .event = vbe_dir_event, + .destroy = vbe_destroy, + .panic = vbe_panic, + .list = vbe_list, +}}; + +/*-------------------------------------------------------------------- * Create a new static or dynamic director::backend instance. */ @@ -481,17 +498,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, d = be->director; INIT_OBJ(d, DIRECTOR_MAGIC); d->priv = be; - d->name = "backend"; d->vcl_name = be->vcl_name; - d->http1pipe = vbe_dir_http1pipe; - d->healthy = vbe_dir_healthy; - d->gethdrs = vbe_dir_gethdrs; - d->getip = vbe_dir_getip; - d->finish = vbe_dir_finish; - d->event = vbe_dir_event; - d->panic = vbe_panic; - d->list = vbe_list; - d->destroy = vbe_destroy; + d->methods = vbe_methods; d->health = 1; d->health_changed = VTIM_real(); diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index cd99f1a..952506f 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -85,9 +85,10 @@ vdi_resolve(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - for (d = bo->director_req; d != NULL && d->resolve != NULL; d = d2) { + for (d = bo->director_req; d != NULL && + d->methods->resolve != NULL; d = d2) { CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - d2 = d->resolve(d, wrk, bo); + d2 = d->methods->resolve(d, wrk, bo); if (d2 == NULL) VSLb(bo->vsl, SLT_FetchError, "Director %s returned no backend", d->vcl_name); @@ -112,9 +113,9 @@ VDI_GetHdr(struct worker *wrk, struct busyobj *bo) d = vdi_resolve(wrk, bo); if (d != NULL) { - AN(d->gethdrs); + AN(d->methods->gethdrs); bo->director_state = DIR_S_HDRS; - i = d->gethdrs(d, wrk, bo); + i = d->methods->gethdrs(d, wrk, bo); } if (i) bo->director_state = DIR_S_NULL; @@ -133,13 +134,13 @@ VDI_GetBody(struct worker *wrk, struct busyobj *bo) d = bo->director_resp; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - AZ(d->resolve); + AZ(d->methods->resolve); assert(bo->director_state == DIR_S_HDRS); bo->director_state = DIR_S_BODY; - if (d->getbody == NULL) + if (d->methods->getbody == NULL) return (0); - return (d->getbody(d, wrk, bo)); + return (d->methods->getbody(d, wrk, bo)); } /* Get IP number (if any ) -------------------------------------------*/ @@ -156,10 +157,10 @@ VDI_GetIP(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); assert(bo->director_state == DIR_S_HDRS || bo->director_state == DIR_S_BODY); - AZ(d->resolve); - if (d->getip == NULL) + AZ(d->methods->resolve); + if (d->methods->getip == NULL) return (NULL); - return (d->getip(d, wrk, bo)); + return (d->methods->getip(d, wrk, bo)); } /* Finish fetch ------------------------------------------------------*/ @@ -175,11 +176,11 @@ VDI_Finish(struct worker *wrk, struct busyobj *bo) d = bo->director_resp; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - AZ(d->resolve); - AN(d->finish); + AZ(d->methods->resolve); + AN(d->methods->finish); assert(bo->director_state != DIR_S_NULL); - d->finish(d, wrk, bo); + d->methods->finish(d, wrk, bo); bo->director_state = DIR_S_NULL; } @@ -194,11 +195,11 @@ VDI_Http1Pipe(struct req *req, struct busyobj *bo) CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); d = vdi_resolve(req->wrk, bo); - if (d == NULL || d->http1pipe == NULL) { + if (d == NULL || d->methods->http1pipe == NULL) { VSLb(bo->vsl, SLT_VCL_Error, "Backend does not support pipe"); return (SC_TX_ERROR); } - return (d->http1pipe(d, req, bo)); + return (d->methods->http1pipe(d, req, bo)); } /* Check health -------------------------------------------------------- @@ -216,9 +217,9 @@ VRT_Healthy(VRT_CTX, VCL_BACKEND d) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); if (!VDI_Healthy(d, NULL)) return (0); - if (d->healthy == NULL) + if (d->methods->healthy == NULL) return (1); - return (d->healthy(d, ctx->bo, NULL)); + return (d->methods->healthy(d, ctx->bo, NULL)); } /* Send Event ---------------------------------------------------------- @@ -229,8 +230,8 @@ VDI_Event(const struct director *d, enum vcl_event_e ev) { CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - if (d->event != NULL) - d->event(d, ev); + if (d->methods->event != NULL) + d->methods->event(d, ev); } /* Dump panic info ----------------------------------------------------- @@ -247,10 +248,10 @@ VDI_Panic(const struct director *d, struct vsb *vsb, const char *nm) VSB_printf(vsb, "health = %s,\n", d->health ? "healthy" : "sick"); VSB_printf(vsb, "admin_health = %s, changed = %f,\n", VDI_Ahealth(d), d->health_changed); - VSB_printf(vsb, "type = %s {\n", d->name); + VSB_printf(vsb, "type = %s {\n", d->methods->type); VSB_indent(vsb, 2); - if (d->panic != NULL) - d->panic(d, vsb); + if (d->methods->panic != NULL) + d->methods->panic(d, vsb); VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); VSB_indent(vsb, -2); @@ -299,15 +300,15 @@ do_list(struct cli *cli, struct director *d, void *priv) VCLI_Out(cli, "\n%-30s %-7s ", d->cli_name, VDI_Ahealth(d)); - if (d->list != NULL) - d->list(d, cli->sb, 0, 0); + if (d->methods->list != NULL) + d->methods->list(d, cli->sb, 0, 0); else VCLI_Out(cli, "%-10s", d->health ? "healthy" : "sick"); VTIM_format(d->health_changed, time_str); VCLI_Out(cli, " %s", time_str); - if (la->p || la->v) - d->list(d, cli->sb, la->p, la->v); + if ((la->p || la->v) && d->methods->list != NULL) + d->methods->list(d, cli->sb, la->p, la->v); return (0); } diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 43e7e4b..6004a09 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -64,11 +64,10 @@ typedef void vdi_panic_f(const struct director *, struct vsb *); typedef void vdi_list_f(const struct director *, struct vsb *, int, int); -struct director { +struct director_methods { unsigned magic; -#define DIRECTOR_MAGIC 0x3336351d - const char *name; - char *vcl_name; +#define DIRECTOR_METHODS_MAGIC 0x4ec0c4bb + const char *type; vdi_http1pipe_f *http1pipe; vdi_healthy_f *healthy; vdi_resolve_f *resolve; @@ -80,6 +79,13 @@ struct director { vdi_destroy_f *destroy; vdi_panic_f *panic; vdi_list_f *list; +}; + +struct director { + unsigned magic; +#define DIRECTOR_MAGIC 0x3336351d + const struct director_methods *methods; + char *vcl_name; void *priv; const void *priv2; diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 7ca458c..178ae2c 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -309,8 +309,8 @@ vcl_KillBackends(struct vcl *vcl) break; VTAILQ_REMOVE(&vcl->director_list, vdir, list); REPLACE(vdir->dir->cli_name, NULL); - AN(vdir->dir->destroy); - vdir->dir->destroy(vdir->dir); + AN(vdir->dir->methods->destroy); + vdir->dir->methods->destroy(vdir->dir); FREE_OBJ(vdir); } } diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index 0848a56..0d32de6 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -135,7 +135,8 @@ VRT_AddDirector(VRT_CTX, struct director *d, const char *vcl_name) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - AN(d->destroy); + CHECK_OBJ_NOTNULL(d->methods, DIRECTOR_METHODS_MAGIC); + AN(d->methods->destroy); AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); if (vcl->temp == VCL_TEMP_COOLING) { @@ -191,9 +192,9 @@ VRT_DelDirector(VRT_CTX, struct director *d) if (VCL_WARM(vcl)) VDI_Event(d, VCL_EVENT_COLD); AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - AN(d->destroy); + AN(d->methods->destroy); REPLACE(d->cli_name, NULL); - d->destroy(d); + d->methods->destroy(d); FREE_OBJ(vdir); } diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index 8ae3ddc..2b7367d 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -75,7 +75,7 @@ vmod_fallback_resolve(const struct director *dir, struct worker *wrk, for (u = 0; u < fb->vd->n_backend; u++) { be = fb->vd->backend[fb->cur]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - if (be->healthy(be, bo, NULL)) + if (be->methods->healthy(be, bo, NULL)) break; if (++fb->cur == fb->vd->n_backend) fb->cur = 0; @@ -86,6 +86,14 @@ vmod_fallback_resolve(const struct director *dir, struct worker *wrk, return (be); } +static const struct director_methods vmod_fallback_methods[1] = {{ + .magic = DIRECTOR_METHODS_MAGIC, + .type = "fallback", + .healthy = vmod_fallback_healthy, + .resolve = vmod_fallback_resolve, +}}; + + VCL_VOID v_matchproto_() vmod_fallback__init(VRT_CTX, struct vmod_directors_fallback **fbp, const char *vcl_name, VCL_BOOL sticky) @@ -98,8 +106,7 @@ vmod_fallback__init(VRT_CTX, ALLOC_OBJ(fb, VMOD_DIRECTORS_FALLBACK_MAGIC); AN(fb); *fbp = fb; - vdir_new(ctx, &fb->vd, "fallback", vcl_name, vmod_fallback_healthy, - vmod_fallback_resolve, fb); + vdir_new(ctx, &fb->vd, vcl_name, vmod_fallback_methods, fb); fb->st = sticky; } diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index a053081..50d4ce9 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -47,6 +47,12 @@ struct vmod_directors_hash { struct vdir *vd; }; +static const struct director_methods vmod_hash_methods[1] = {{ + .magic = DIRECTOR_METHODS_MAGIC, + .type = "hash", +}}; + + VCL_VOID v_matchproto_() vmod_hash__init(VRT_CTX, struct vmod_directors_hash **rrp, const char *vcl_name) @@ -59,7 +65,7 @@ vmod_hash__init(VRT_CTX, struct vmod_directors_hash **rrp, ALLOC_OBJ(rr, VMOD_DIRECTORS_HASH_MAGIC); AN(rr); *rrp = rr; - vdir_new(ctx, &rr->vd, "hash", vcl_name, NULL, NULL, rr); + vdir_new(ctx, &rr->vd, vcl_name, vmod_hash_methods, rr); } VCL_VOID v_matchproto_() diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index 4ce6328..2fd6a35 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -74,6 +74,14 @@ vmod_random_resolve(const struct director *dir, struct worker *wrk, return (be); } +static const struct director_methods vmod_random_methods[1] = {{ + .magic = DIRECTOR_METHODS_MAGIC, + .type = "random", + .healthy = vmod_random_healthy, + .resolve = vmod_random_resolve, +}}; + + VCL_VOID v_matchproto_() vmod_random__init(VRT_CTX, struct vmod_directors_random **rrp, const char *vcl_name) @@ -86,8 +94,7 @@ vmod_random__init(VRT_CTX, struct vmod_directors_random **rrp, ALLOC_OBJ(rr, VMOD_DIRECTORS_RANDOM_MAGIC); AN(rr); *rrp = rr; - vdir_new(ctx, &rr->vd, "random", vcl_name, vmod_random_healthy, - vmod_random_resolve, rr); + vdir_new(ctx, &rr->vd, vcl_name, vmod_random_methods, rr); } VCL_VOID v_matchproto_() diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index 7e888f3..1199abb 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -74,7 +74,7 @@ vmod_rr_resolve(const struct director *dir, struct worker *wrk, rr->nxt = nxt + 1; be = rr->vd->backend[nxt]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - if (be->healthy(be, bo, NULL)) + if (be->methods->healthy(be, bo, NULL)) break; } vdir_unlock(rr->vd); @@ -83,6 +83,13 @@ vmod_rr_resolve(const struct director *dir, struct worker *wrk, return (be); } +static const struct director_methods vmod_rr_methods[1] = {{ + .magic = DIRECTOR_METHODS_MAGIC, + .type = "round-robin", + .healthy = vmod_rr_healthy, + .resolve = vmod_rr_resolve, +}}; + VCL_VOID v_matchproto_() vmod_round_robin__init(VRT_CTX, struct vmod_directors_round_robin **rrp, const char *vcl_name) @@ -95,8 +102,7 @@ vmod_round_robin__init(VRT_CTX, ALLOC_OBJ(rr, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); AN(rr); *rrp = rr; - vdir_new(ctx, &rr->vd, "round-robin", vcl_name, vmod_rr_healthy, - vmod_rr_resolve, rr); + vdir_new(ctx, &rr->vd, vcl_name, vmod_rr_methods, rr); } VCL_VOID v_matchproto_() diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 7a667c3..2594962 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -189,7 +189,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy) sbe = NULL; be = state->shardd->backend[c].backend; AN(be); - if (be->healthy(be, state->ctx->bo, &changed)) { + if (be->methods->healthy(be, state->ctx->bo, &changed)) { if (skip-- == 0) { chosen = c; sbe = &state->last; @@ -326,7 +326,7 @@ sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo, for (u = 0; u < shardd->n_backend; u++) { be = shardd->backend[u].backend; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - retval = be->healthy(be, bo, &c); + retval = be->methods->healthy(be, bo, &c); if (changed != NULL && c > *changed) *changed = c; if (retval) diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 95cc9df..f57e303 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -50,13 +50,13 @@ vdir_expand(struct vdir *vd, unsigned n) } void -vdir_new(VRT_CTX, struct vdir **vdp, const char *name, const char *vcl_name, - vdi_healthy_f *healthy, vdi_resolve_f *resolve, void *priv) +vdir_new(VRT_CTX, struct vdir **vdp, const char *vcl_name, + const struct director_methods *m, void *priv) { struct vdir *vd; AN(ctx); - AN(name); + CHECK_OBJ_NOTNULL(m, DIRECTOR_METHODS_MAGIC); AN(vcl_name); AN(vdp); AZ(*vdp); @@ -67,11 +67,9 @@ vdir_new(VRT_CTX, struct vdir **vdp, const char *name, const char *vcl_name, ALLOC_OBJ(vd->dir, DIRECTOR_MAGIC); AN(vd->dir); - vd->dir->name = name; + vd->dir->methods = m; REPLACE(vd->dir->vcl_name, vcl_name); vd->dir->priv = priv; - vd->dir->healthy = healthy; - vd->dir->resolve = resolve; vd->dir->admin_health = VDI_AH_HEALTHY; vd->vbm = vbit_new(8); AN(vd->vbm); @@ -188,7 +186,7 @@ vdir_any_healthy(struct vdir *vd, const struct busyobj *bo, double *changed) for (u = 0; u < vd->n_backend; u++) { be = vd->backend[u]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - retval = be->healthy(be, bo, &c); + retval = be->methods->healthy(be, bo, &c); if (changed != NULL && c > *changed) *changed = c; if (retval) @@ -228,7 +226,7 @@ vdir_pick_be(struct vdir *vd, double w, const struct busyobj *bo) vdir_wrlock(vd); for (u = 0; u < vd->n_backend; u++) { - if (vd->backend[u]->healthy(vd->backend[u], bo, NULL)) { + if (vd->backend[u]->methods->healthy(vd->backend[u], bo, NULL)) { vbit_clr(vd->vbm, u); tw += vd->weight[u]; } else diff --git a/lib/libvmod_directors/vdir.h b/lib/libvmod_directors/vdir.h index afdc421..4e7d9e6 100644 --- a/lib/libvmod_directors/vdir.h +++ b/lib/libvmod_directors/vdir.h @@ -41,9 +41,8 @@ struct vdir { struct vbitmap *vbm; }; -void vdir_new(VRT_CTX, struct vdir **vdp, const char *name, - const char *vcl_name, - vdi_healthy_f *healthy, vdi_resolve_f *resolve, void *priv); +void vdir_new(VRT_CTX, struct vdir **vdp, const char *vcl_name, + const struct director_methods *, void *priv); void vdir_delete(struct vdir **vdp); void vdir_rdlock(struct vdir *vd); void vdir_wrlock(struct vdir *vd); diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 1d6877e..58830b2 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -196,6 +196,14 @@ shard__assert(void) assert(t2a == t2b); } +static const struct director_methods vmod_shard_methods[1] = {{ + .magic = DIRECTOR_METHODS_MAGIC, + .type = "shard", + .resolve = vmod_shard_resolve, + .healthy = vmod_shard_healthy, +}}; + + VCL_VOID v_matchproto_(td_directors_shard__init) vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, const char *vcl_name) @@ -217,8 +225,7 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, AN(vshard->dir); REPLACE(vshard->dir->vcl_name, vcl_name); vshard->dir->priv = vshard; - vshard->dir->resolve = vmod_shard_resolve; - vshard->dir->healthy = vmod_shard_healthy; + vshard->dir->methods = vmod_shard_methods; vshard->dir->admin_health = VDI_AH_HEALTHY; } From hermunn at varnish-software.com Wed Apr 25 10:02:16 2018 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 25 Apr 2018 10:02:16 +0000 (UTC) Subject: [4.1] f609870 Pass delivery abandoned does not qualify as an error Message-ID: <20180425100216.E7C62966A1@lists.varnish-cache.org> commit f609870a12d7104da250c78236bfdcd0dda306af Author: Nils Goroll Date: Fri Oct 6 12:58:24 2017 +0200 Pass delivery abandoned does not qualify as an error ... so log it under the Debug tag. FetchErrors should be actual errors which can be addressed. In this case, nothing is wrong in any way, the fact that we abort a fetch if we don't need the body is a varnish internal optimization (which makes sense, but comes at the cost of closing a connection). Merges #2450 Conflicts: bin/varnishd/cache/cache_fetch.c diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 902ad17..eb3d58e 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -495,8 +495,8 @@ vbf_fetch_body_helper(struct busyobj *bo) * objects to be created. */ AN(vfc->oc->flags & OC_F_PASS); - VSLb(vfc->wrk->vsl, SLT_FetchError, - "Pass delivery abandoned"); + VSLb(vfc->wrk->vsl, SLT_Debug, + "Fetch: Pass delivery abandoned"); vfps = VFP_END; bo->htc->doclose = SC_RX_BODY; break; diff --git a/doc/changes.rst b/doc/changes.rst index e276414..663f91f 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -21,6 +21,7 @@ Bugs fixed * 1772_ - Honor first_byte_timeout for recycled backend connections * 2135_ - Limit Backend Connection retries to a single retry +* 2450_ - Pass delivery abandoned does not qualify as an error * 2495_ - Avoid leaking an OH ref on reembark failure * 2502_ - objcore reference count leak * 2530_ - Varnish shouldn't test gunzip for range responses @@ -29,6 +30,7 @@ Bugs fixed .. _1772: https://github.com/varnishcache/varnish-cache/issues/1772 .. _2135: https://github.com/varnishcache/varnish-cache/pull/2135 +.. _2450: https://github.com/varnishcache/varnish-cache/pull/2450 .. _2495: https://github.com/varnishcache/varnish-cache/issues/2495 .. _2502: https://github.com/varnishcache/varnish-cache/issues/2502 .. _2530: https://github.com/varnishcache/varnish-cache/issues/2530 From fgsch at lodoss.net Wed Apr 25 10:53:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 25 Apr 2018 10:53:12 +0000 (UTC) Subject: [master] 72ab7cd Upgrade brew's Python to avoid linking errors Message-ID: <20180425105312.D9375976C6@lists.varnish-cache.org> commit 72ab7cd7d80fa1410fc0ee9f07ff27b32a610737 Author: Carla Date: Wed Apr 25 12:35:55 2018 +0200 Upgrade brew's Python to avoid linking errors See: https://github.com/Homebrew/homebrew-core/issues/26358 diff --git a/.travis.yml b/.travis.yml index 76a6f70..2a84088 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ notifications: before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; + brew upgrade python; brew install docutils sphinx-doc nghttp2; fi - if [[ -n "$CLANG" ]]; then From hermunn at varnish-software.com Wed Apr 25 11:29:12 2018 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 25 Apr 2018 11:29:12 +0000 (UTC) Subject: [4.1] 1d090c5 Prepare for 4.1.10 release Message-ID: <20180425112912.3D6FB981C2@lists.varnish-cache.org> commit 1d090c5a08f41c36562644bafcce9d3cb85d824f Author: P?l Hermunn Johansen Date: Wed Apr 25 10:56:44 2018 +0200 Prepare for 4.1.10 release diff --git a/configure.ac b/configure.ac index 6e4e859..e4c2462 100644 --- a/configure.ac +++ b/configure.ac @@ -1,8 +1,8 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS -Copyright (c) 2006-2017 Varnish Software]) +Copyright (c) 2006-2018 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [4.1.9], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [4.1.10], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 663f91f..5a4e7e2 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,5 +1,5 @@ ================================= -Varnish Cache 4.1.10 (unreleased) +Varnish Cache 4.1.10 (2018-04-25) ================================= Changes since 4.1.9: From nils.goroll at uplex.de Thu Apr 26 06:43:28 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 26 Apr 2018 06:43:28 +0000 (UTC) Subject: [master] 16e5338 fix regression from 1bb0688c9a744e3f89fe0624f58c1ba0d4347867 Message-ID: <20180426064328.4364EB49C4@lists.varnish-cache.org> commit 16e5338ec62ec4206a2de6aa35bb95e52040eb5e Author: Nils Goroll Date: Thu Apr 26 08:41:39 2018 +0200 fix regression from 1bb0688c9a744e3f89fe0624f58c1ba0d4347867 thanks to coverity diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 58830b2..0a0fd90 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -815,7 +815,7 @@ shard_param_task(VRT_CTX, const void *id, if (task->priv) { p = task->priv; CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); - assert(p->scope = SCOPE_TASK); + assert(p->scope == SCOPE_TASK); /* XXX VSL(SLT_Debug, 0, "shard_param_task(id %p, pa %p) = %p (found, ws=%p)", From phk at FreeBSD.org Thu Apr 26 08:31:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Apr 2018 08:31:10 +0000 (UTC) Subject: [master] 5536f10 Pass VCL_BACKEND->resolve() VRT_CTX Message-ID: <20180426083111.0DC4EB6813@lists.varnish-cache.org> commit 5536f102b677e6b469f191fedab1c4cf51144e2c Author: Poul-Henning Kamp Date: Thu Apr 26 07:03:20 2018 +0000 Pass VCL_BACKEND->resolve() VRT_CTX diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 952506f..dca50c3 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -40,6 +40,7 @@ #include "cache_director.h" #include "vcli_serve.h" +#include "vcl.h" #include "vtim.h" /* -------------------------------------------------------------------*/ @@ -77,18 +78,29 @@ VDI_Ahealth(const struct director *d) /* Resolve director --------------------------------------------------*/ static const struct director * -vdi_resolve(struct worker *wrk, struct busyobj *bo) +vdi_resolve(struct busyobj *bo) { const struct director *d; const struct director *d2; + struct vrt_ctx ctx; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + INIT_OBJ(&ctx, VRT_CTX_MAGIC); + ctx.vcl = bo->vcl; + ctx.vsl = bo->vsl; + ctx.http_bereq = bo->bereq; + ctx.http_beresp = bo->beresp; + ctx.bo = bo; + ctx.sp = bo->sp; + ctx.now = bo->t_prev; + ctx.ws = bo->ws; + ctx.method = VCL_MET_BACKEND_FETCH; // XXX: Not quite true + for (d = bo->director_req; d != NULL && d->methods->resolve != NULL; d = d2) { CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - d2 = d->methods->resolve(d, wrk, bo); + d2 = d->methods->resolve(&ctx, d); if (d2 == NULL) VSLb(bo->vsl, SLT_FetchError, "Director %s returned no backend", d->vcl_name); @@ -111,7 +123,7 @@ VDI_GetHdr(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - d = vdi_resolve(wrk, bo); + d = vdi_resolve(bo); if (d != NULL) { AN(d->methods->gethdrs); bo->director_state = DIR_S_HDRS; @@ -194,7 +206,7 @@ VDI_Http1Pipe(struct req *req, struct busyobj *bo) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - d = vdi_resolve(req->wrk, bo); + d = vdi_resolve(bo); if (d == NULL || d->methods->http1pipe == NULL) { VSLb(bo->vsl, SLT_VCL_Error, "Backend does not support pipe"); return (SC_TX_ERROR); diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 6004a09..b6629bd 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -41,9 +41,7 @@ struct vcldir; typedef unsigned vdi_healthy_f(const struct director *, const struct busyobj *, double *changed); -/* XXX need a VRT_CTX argument */ -typedef const struct director *vdi_resolve_f(const struct director *, - struct worker *, struct busyobj *); +typedef VCL_BACKEND vdi_resolve_f(VRT_CTX, VCL_BACKEND); typedef int vdi_gethdrs_f(const struct director *, struct worker *, struct busyobj *); typedef int vdi_getbody_f(const struct director *, struct worker *, diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index 2b7367d..46d1cf0 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -56,17 +56,15 @@ vmod_fallback_healthy(const struct director *dir, const struct busyobj *bo, return (vdir_any_healthy(fb->vd, bo, changed)); } -static const struct director * v_matchproto_(vdi_resolve_f) -vmod_fallback_resolve(const struct director *dir, struct worker *wrk, - struct busyobj *bo) +static VCL_BACKEND v_matchproto_(vdi_resolve_f) +vmod_fallback_resolve(VRT_CTX, VCL_BACKEND dir) { struct vmod_directors_fallback *fb; unsigned u; VCL_BACKEND be = NULL; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(fb, dir->priv, VMOD_DIRECTORS_FALLBACK_MAGIC); vdir_wrlock(fb->vd); @@ -75,7 +73,7 @@ vmod_fallback_resolve(const struct director *dir, struct worker *wrk, for (u = 0; u < fb->vd->n_backend; u++) { be = fb->vd->backend[fb->cur]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - if (be->methods->healthy(be, bo, NULL)) + if (be->methods->healthy(be, ctx->bo, NULL)) break; if (++fb->cur == fb->vd->n_backend) fb->cur = 0; diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index 2fd6a35..cab3f7c 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -56,21 +56,19 @@ vmod_random_healthy(const struct director *dir, const struct busyobj *bo, return (vdir_any_healthy(rr->vd, bo, changed)); } -static const struct director * v_matchproto_(vdi_resolve_f) -vmod_random_resolve(const struct director *dir, struct worker *wrk, - struct busyobj *bo) +static VCL_BACKEND v_matchproto_(vdi_resolve_f) +vmod_random_resolve(VRT_CTX, VCL_BACKEND dir) { struct vmod_directors_random *rr; VCL_BACKEND be; double r; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC); r = scalbn(VRND_RandomTestable(), -31); assert(r >= 0 && r < 1.0); - be = vdir_pick_be(rr->vd, r, bo); + be = vdir_pick_be(rr->vd, r, ctx->bo); return (be); } diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index 1199abb..adaf0f5 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -55,18 +55,16 @@ vmod_rr_healthy(const struct director *dir, const struct busyobj *bo, return (vdir_any_healthy(rr->vd, bo, changed)); } -static const struct director * v_matchproto_(vdi_resolve_f) -vmod_rr_resolve(const struct director *dir, struct worker *wrk, - struct busyobj *bo) +static VCL_BACKEND v_matchproto_(vdi_resolve_f) +vmod_rr_resolve(VRT_CTX, VCL_BACKEND dir) { struct vmod_directors_round_robin *rr; unsigned u; VCL_BACKEND be = NULL; unsigned nxt; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); vdir_rdlock(rr->vd); for (u = 0; u < rr->vd->n_backend; u++) { @@ -74,7 +72,7 @@ vmod_rr_resolve(const struct director *dir, struct worker *wrk, rr->nxt = nxt + 1; be = rr->vd->backend[nxt]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - if (be->methods->healthy(be, bo, NULL)) + if (be->methods->healthy(be, ctx->bo, NULL)) break; } vdir_unlock(rr->vd); diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 0a0fd90..563f0a8 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -134,9 +134,7 @@ static unsigned v_matchproto_(vdi_healthy) vmod_shard_healthy(const struct director *dir, const struct busyobj *bo, double *changed); -static const struct director * v_matchproto_(vdi_resolve_f) -vmod_shard_resolve(const struct director *dir, struct worker *wrk, - struct busyobj *bo); +static vdi_resolve_f vmod_shard_resolve; struct vmod_directors_shard { unsigned magic; @@ -692,32 +690,17 @@ vmod_shard_healthy(const struct director *dir, const struct busyobj *bo, return (sharddir_any_healthy(vshard->shardd, bo, changed)); } -static const struct director * v_matchproto_(vdi_resolve_f) -vmod_shard_resolve(const struct director *dir, struct worker *wrk, - struct busyobj *bo) +static VCL_BACKEND v_matchproto_(vdi_resolve_f) +vmod_shard_resolve(VRT_CTX, VCL_BACKEND dir) { struct vmod_directors_shard *vshard; struct vmod_directors_shard_param pstk[1]; const struct vmod_directors_shard_param *pp; - struct vrt_ctx ctx[1]; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(vshard, dir->priv, VMOD_SHARD_SHARD_MAGIC); - // Ref: vcl_call_method() - INIT_OBJ(ctx, VRT_CTX_MAGIC); - ctx->vsl = bo->vsl; - ctx->vcl = bo->vcl; - ctx->http_bereq = bo->bereq; - ctx->http_beresp = bo->beresp; - ctx->bo = bo; - ctx->sp = bo->sp; - ctx->now = bo->t_prev; - ctx->ws = bo->ws; - ctx->method = VCL_MET_BACKEND_FETCH; - pp = vmod_shard_param_read(ctx, vshard, vshard->param, pstk, "shard_resolve"); if (pp == NULL) From phk at FreeBSD.org Thu Apr 26 08:31:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Apr 2018 08:31:11 +0000 (UTC) Subject: [master] c25ed7e Give VCL_BACKEND->healthy() a VRT_CTX and VCL_typed args Message-ID: <20180426083111.23CA2B6815@lists.varnish-cache.org> commit c25ed7e1e71c6743b95461909e1bc40c341cb71c Author: Poul-Henning Kamp Date: Thu Apr 26 07:31:18 2018 +0000 Give VCL_BACKEND->healthy() a VRT_CTX and VCL_typed args diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 7ca57d5..2fb9ba8 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -153,14 +153,13 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, return (pfd); } -static unsigned v_matchproto_(vdi_healthy_f) -vbe_dir_healthy(const struct director *d, const struct busyobj *bo, - double *changed) +static VCL_BOOL v_matchproto_(vdi_healthy_f) +vbe_dir_healthy(VRT_CTX, VCL_BACKEND d, VCL_TIME *changed) { struct backend *be; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC); return (VDI_Healthy(be->director, changed)); } diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index dca50c3..145d86e 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -231,7 +231,7 @@ VRT_Healthy(VRT_CTX, VCL_BACKEND d) return (0); if (d->methods->healthy == NULL) return (1); - return (d->methods->healthy(d, ctx->bo, NULL)); + return (d->methods->healthy(ctx, d, NULL)); } /* Send Event ---------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index b6629bd..05f282a 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -39,9 +39,9 @@ struct vcldir; -typedef unsigned vdi_healthy_f(const struct director *, const struct busyobj *, - double *changed); +typedef VCL_BOOL vdi_healthy_f(VRT_CTX, VCL_BACKEND, VCL_TIME *); typedef VCL_BACKEND vdi_resolve_f(VRT_CTX, VCL_BACKEND); + typedef int vdi_gethdrs_f(const struct director *, struct worker *, struct busyobj *); typedef int vdi_getbody_f(const struct director *, struct worker *, diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index 46d1cf0..01f32fa 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -46,14 +46,15 @@ struct vmod_directors_fallback { unsigned cur; }; -static unsigned v_matchproto_(vdi_healthy) -vmod_fallback_healthy(const struct director *dir, const struct busyobj *bo, - double *changed) +static VCL_BOOL v_matchproto_(vdi_healthy) +vmod_fallback_healthy(VRT_CTX, VCL_BACKEND dir, VCL_TIME *changed) { struct vmod_directors_fallback *fb; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(fb, dir->priv, VMOD_DIRECTORS_FALLBACK_MAGIC); - return (vdir_any_healthy(fb->vd, bo, changed)); + return (vdir_any_healthy(ctx, fb->vd, changed)); } static VCL_BACKEND v_matchproto_(vdi_resolve_f) @@ -73,7 +74,7 @@ vmod_fallback_resolve(VRT_CTX, VCL_BACKEND dir) for (u = 0; u < fb->vd->n_backend; u++) { be = fb->vd->backend[fb->cur]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - if (be->methods->healthy(be, ctx->bo, NULL)) + if (be->methods->healthy(ctx, be, NULL)) break; if (++fb->cur == fb->vd->n_backend) fb->cur = 0; diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index 50d4ce9..1e4d9de 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -129,6 +129,6 @@ vmod_hash_backend(VRT_CTX, struct vmod_directors_hash *rr, r = vbe32dec(sha256); r = scalbn(r, -32); assert(r >= 0 && r <= 1.0); - be = vdir_pick_be(rr->vd, r, ctx->bo); + be = vdir_pick_be(ctx, rr->vd, r); return (be); } diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index cab3f7c..c89fb8e 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -46,14 +46,15 @@ struct vmod_directors_random { struct vdir *vd; }; -static unsigned v_matchproto_(vdi_healthy) -vmod_random_healthy(const struct director *dir, const struct busyobj *bo, - double *changed) +static VCL_BOOL v_matchproto_(vdi_healthy) +vmod_random_healthy(VRT_CTX, VCL_BACKEND dir, VCL_TIME *changed) { struct vmod_directors_random *rr; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC); - return (vdir_any_healthy(rr->vd, bo, changed)); + return (vdir_any_healthy(ctx, rr->vd, changed)); } static VCL_BACKEND v_matchproto_(vdi_resolve_f) @@ -68,7 +69,7 @@ vmod_random_resolve(VRT_CTX, VCL_BACKEND dir) CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC); r = scalbn(VRND_RandomTestable(), -31); assert(r >= 0 && r < 1.0); - be = vdir_pick_be(rr->vd, r, ctx->bo); + be = vdir_pick_be(ctx, rr->vd, r); return (be); } diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index adaf0f5..3a983a7 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -45,14 +45,15 @@ struct vmod_directors_round_robin { unsigned nxt; }; -static unsigned v_matchproto_(vdi_healthy) -vmod_rr_healthy(const struct director *dir, const struct busyobj *bo, - double *changed) +static VCL_BOOL v_matchproto_(vdi_healthy) +vmod_rr_healthy(VRT_CTX, VCL_BACKEND dir, VCL_TIME *changed) { struct vmod_directors_round_robin *rr; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); - return (vdir_any_healthy(rr->vd, bo, changed)); + return (vdir_any_healthy(ctx, rr->vd, changed)); } static VCL_BACKEND v_matchproto_(vdi_resolve_f) @@ -72,7 +73,7 @@ vmod_rr_resolve(VRT_CTX, VCL_BACKEND dir) rr->nxt = nxt + 1; be = rr->vd->backend[nxt]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - if (be->methods->healthy(be, ctx->bo, NULL)) + if (be->methods->healthy(ctx, be, NULL)) break; } vdir_unlock(rr->vd); diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 2594962..20cfc42 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -189,7 +189,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy) sbe = NULL; be = state->shardd->backend[c].backend; AN(be); - if (be->methods->healthy(be, state->ctx->bo, &changed)) { + if (be->methods->healthy(state->ctx, be, &changed)) { if (skip-- == 0) { chosen = c; sbe = &state->last; @@ -309,9 +309,8 @@ init_state(struct shard_state *state, * - XXX should we return the health state of the actual backend * for healthy=IGNORE ? */ -unsigned -sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo, - double *changed) +VCL_BOOL +sharddir_any_healthy(VRT_CTX, struct sharddir *shardd, VCL_TIME *changed) { unsigned retval = 0; VCL_BACKEND be; @@ -319,14 +318,13 @@ sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo, double c; CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); - CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); sharddir_rdlock(shardd); if (changed != NULL) *changed = 0; for (u = 0; u < shardd->n_backend; u++) { be = shardd->backend[u].backend; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - retval = be->methods->healthy(be, bo, &c); + retval = be->methods->healthy(ctx, be, &c); if (changed != NULL && c > *changed) *changed = c; if (retval) diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index f1b1b00..350437f 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -128,8 +128,7 @@ void sharddir_new(struct sharddir **sharddp, const char *vcl_name); void sharddir_delete(struct sharddir **sharddp); void sharddir_wrlock(struct sharddir *shardd); void sharddir_unlock(struct sharddir *shardd); -unsigned sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo, - double *changed); +VCL_BOOL sharddir_any_healthy(VRT_CTX, struct sharddir *, VCL_TIME *); VCL_BACKEND sharddir_pick_be(VRT_CTX, struct sharddir *, uint32_t, VCL_INT, VCL_REAL, VCL_BOOL, enum healthy_e); diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index f57e303..81cb8ee 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -55,7 +55,7 @@ vdir_new(VRT_CTX, struct vdir **vdp, const char *vcl_name, { struct vdir *vd; - AN(ctx); + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(m, DIRECTOR_METHODS_MAGIC); AN(vcl_name); AN(vdp); @@ -118,6 +118,7 @@ vdir_add_backend(VRT_CTX, struct vdir *vd, VCL_BACKEND be, double weight) { unsigned u; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); if (be == NULL) { VRT_fail(ctx, "NULL backend cannot be added"); @@ -140,6 +141,7 @@ vdir_remove_backend(VRT_CTX, struct vdir *vd, VCL_BACKEND be, unsigned *cur) { unsigned u, n; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); if (be == NULL) { VRT_fail(ctx, "NULL backend cannot be removed"); @@ -170,23 +172,23 @@ vdir_remove_backend(VRT_CTX, struct vdir *vd, VCL_BACKEND be, unsigned *cur) vdir_unlock(vd); } -unsigned -vdir_any_healthy(struct vdir *vd, const struct busyobj *bo, double *changed) +VCL_BOOL +vdir_any_healthy(VRT_CTX, struct vdir *vd, VCL_TIME *changed) { unsigned retval = 0; VCL_BACKEND be; unsigned u; double c; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); - CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); vdir_rdlock(vd); if (changed != NULL) *changed = 0; for (u = 0; u < vd->n_backend; u++) { be = vd->backend[u]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - retval = be->methods->healthy(be, bo, &c); + retval = be->methods->healthy(ctx, be, &c); if (changed != NULL && c > *changed) *changed = c; if (retval) @@ -218,15 +220,17 @@ vdir_pick_by_weight(const struct vdir *vd, double w, } VCL_BACKEND -vdir_pick_be(struct vdir *vd, double w, const struct busyobj *bo) +vdir_pick_be(VRT_CTX, struct vdir *vd, double w) { unsigned u; double tw = 0.0; VCL_BACKEND be = NULL; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); vdir_wrlock(vd); for (u = 0; u < vd->n_backend; u++) { - if (vd->backend[u]->methods->healthy(vd->backend[u], bo, NULL)) { + if (vd->backend[u]->methods->healthy(ctx, vd->backend[u], NULL)) { vbit_clr(vd->vbm, u); tw += vd->weight[u]; } else diff --git a/lib/libvmod_directors/vdir.h b/lib/libvmod_directors/vdir.h index 4e7d9e6..6c8331d 100644 --- a/lib/libvmod_directors/vdir.h +++ b/lib/libvmod_directors/vdir.h @@ -49,6 +49,5 @@ void vdir_wrlock(struct vdir *vd); void vdir_unlock(struct vdir *vd); void vdir_add_backend(VRT_CTX, struct vdir *, VCL_BACKEND, double weight); void vdir_remove_backend(VRT_CTX, struct vdir *, VCL_BACKEND, unsigned *cur); -unsigned vdir_any_healthy(struct vdir *, const struct busyobj *, - double *changed); -VCL_BACKEND vdir_pick_be(struct vdir *, double w, const struct busyobj *); +VCL_BOOL vdir_any_healthy(VRT_CTX, struct vdir *, VCL_TIME *); +VCL_BACKEND vdir_pick_be(VRT_CTX, struct vdir *, double w); diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 563f0a8..b546495 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -130,10 +130,7 @@ vmod_shard_param_read(VRT_CTX, const void *id, /* ------------------------------------------------------------------------- * shard vmod interface */ -static unsigned v_matchproto_(vdi_healthy) -vmod_shard_healthy(const struct director *dir, const struct busyobj *bo, - double *changed); - +static vdi_healthy_f vmod_shard_healthy; static vdi_resolve_f vmod_shard_resolve; struct vmod_directors_shard { @@ -680,14 +677,15 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, pp->rampup, pp->healthy)); } -static unsigned v_matchproto_(vdi_healthy) -vmod_shard_healthy(const struct director *dir, const struct busyobj *bo, - double *changed) +static VCL_BOOL v_matchproto_(vdi_healthy) +vmod_shard_healthy(VRT_CTX, VCL_BACKEND dir, VCL_TIME *changed) { struct vmod_directors_shard *vshard; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vshard, dir->priv, VMOD_SHARD_SHARD_MAGIC); - return (sharddir_any_healthy(vshard->shardd, bo, changed)); + return (sharddir_any_healthy(ctx, vshard->shardd, changed)); } static VCL_BACKEND v_matchproto_(vdi_resolve_f) From phk at FreeBSD.org Thu Apr 26 08:31:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Apr 2018 08:31:11 +0000 (UTC) Subject: [master] 401d1dd Make everybody use VRT_Healthy() and simplify backends as result. Message-ID: <20180426083111.4BFB4B6818@lists.varnish-cache.org> commit 401d1dd92aec97e9a41733549442737100375266 Author: Poul-Henning Kamp Date: Thu Apr 26 08:18:30 2018 +0000 Make everybody use VRT_Healthy() and simplify backends as result. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 2fb9ba8..21258a8 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -88,7 +88,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC); AN(bp->vsc); - if (!VDI_Healthy(bp->director, NULL)) { + if (!bp->director->health) { VSLb(bo->vsl, SLT_FetchError, "backend %s: unhealthy", bp->director->cli_name); // XXX: per backend stats ? @@ -153,17 +153,6 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, return (pfd); } -static VCL_BOOL v_matchproto_(vdi_healthy_f) -vbe_dir_healthy(VRT_CTX, VCL_BACKEND d, VCL_TIME *changed) -{ - struct backend *be; - - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC); - return (VDI_Healthy(be->director, changed)); -} - static void v_matchproto_(vdi_finish_f) vbe_dir_finish(const struct director *d, struct worker *wrk, struct busyobj *bo) @@ -439,7 +428,6 @@ static const struct director_methods vbe_methods[1] = {{ .magic = DIRECTOR_METHODS_MAGIC, .type = "backend", .http1pipe = vbe_dir_http1pipe, - .healthy = vbe_dir_healthy, .gethdrs = vbe_dir_gethdrs, .getip = vbe_dir_getip, .finish = vbe_dir_finish, diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 145d86e..7e8b4bc 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -219,19 +219,32 @@ VDI_Http1Pipe(struct req *req, struct busyobj *bo) * If director has no healthy method, we just assume it is healthy. */ -int -VRT_Healthy(VRT_CTX, VCL_BACKEND d) +/*-------------------------------------------------------------------- + * Test if backend is healthy and report when that last changed + */ + +VCL_BOOL +VRT_Healthy(VRT_CTX, VCL_BACKEND d, VCL_TIME *changed) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (d == NULL) return (0); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - if (!VDI_Healthy(d, NULL)) - return (0); - if (d->methods->healthy == NULL) - return (1); - return (d->methods->healthy(ctx, d, NULL)); + + if (d->admin_health->health >= 0) { + if (changed != NULL) + *changed = d->health_changed; + return (d->admin_health->health); + } + + if (d->methods->healthy == NULL) { + if (changed != NULL) + *changed = d->health_changed; + return (d->health); + } + + return (d->methods->healthy(ctx, d, changed)); } /* Send Event ---------------------------------------------------------- @@ -270,24 +283,6 @@ VDI_Panic(const struct director *d, struct vsb *vsb, const char *nm) VSB_printf(vsb, "},\n"); } -/*-------------------------------------------------------------------- - * Test if backend is healthy and report when it last changed - */ - -unsigned -VDI_Healthy(const struct director *d, double *changed) -{ - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - AN(d->admin_health); - - if (changed != NULL) - *changed = d->health_changed; - - if (d->admin_health->health < 0) - return (d->health); - - return (d->admin_health->health); -} /*---------------------------------------------------------------------*/ @@ -358,43 +353,49 @@ cli_backend_list(struct cli *cli, const char * const *av, void *priv) /*---------------------------------------------------------------------*/ +struct set_health { + unsigned magic; +#define SET_HEALTH_MAGIC 0x0c46b9fb + const struct vdi_ahealth *ah; +}; + static int v_matchproto_(vcl_be_func) do_set_health(struct cli *cli, struct director *d, void *priv) { - unsigned prev; + struct set_health *sh; (void)cli; - AN(priv); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CAST_OBJ_NOTNULL(sh, priv, SET_HEALTH_MAGIC); if (d->admin_health == VDI_AH_DELETED) return (0); - prev = VDI_Healthy(d, NULL); - d->admin_health = *(const struct vdi_ahealth **)priv; - (void)VDI_Ahealth(d); // Acts like type-check - if (prev != VDI_Healthy(d, NULL)) + if (d->admin_health != sh->ah) { d->health_changed = VTIM_real(); - + d->admin_health = sh->ah; + d->health = sh->ah->health ? 1 : 0; + } return (0); } static void v_matchproto_() cli_backend_set_health(struct cli *cli, const char * const *av, void *priv) { - const struct vdi_ahealth *ah; int n; + struct set_health sh[1]; (void)av; (void)priv; ASSERT_CLI(); AN(av[2]); AN(av[3]); - ah = vdi_str2ahealth(av[3]); - if (ah == NULL || ah == VDI_AH_DELETED) { + INIT_OBJ(sh, SET_HEALTH_MAGIC); + sh->ah = vdi_str2ahealth(av[3]); + if (sh->ah == NULL || sh->ah == VDI_AH_DELETED) { VCLI_Out(cli, "Invalid state %s", av[3]); VCLI_SetResult(cli, CLIS_PARAM); return; } - n = VCL_IterDirector(cli, av[2], do_set_health, &ah); + n = VCL_IterDirector(cli, av[2], do_set_health, sh); if (n == 0) { VCLI_Out(cli, "No Backends matches"); VCLI_SetResult(cli, CLIS_PARAM); diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 05f282a..3e502ec 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -99,7 +99,6 @@ struct director { double health_changed; }; -unsigned VDI_Healthy(const struct director *, double *); /* cache_vcl.c */ int VRT_AddDirector(VRT_CTX, struct director *, const char *); diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc index 4d96e29..161c0d5 100644 --- a/bin/varnishtest/tests/c00048.vtc +++ b/bin/varnishtest/tests/c00048.vtc @@ -23,7 +23,7 @@ varnish v1 -vcl { } -start -delay 1 +varnish v1 -vsl_catchup varnish v1 -cliok "vcl.list" varnish v1 -cliok "backend.list -p" @@ -36,6 +36,8 @@ client c1 { expect resp.status == 200 } -run +varnish v1 -vsl_catchup + varnish v1 -cliok "backend.list" varnish v1 -cliok "backend.set_health s1 sick" varnish v1 -cliok "backend.list" @@ -46,6 +48,8 @@ client c1 { expect resp.status == 503 } -run +varnish v1 -vsl_catchup + varnish v1 -cliok "backend.list" varnish v1 -cliok "backend.set_health s1 healthy" varnish v1 -cliok "backend.list" @@ -56,6 +60,8 @@ client c1 { expect resp.status == 200 } -run +varnish v1 -vsl_catchup + varnish v1 -clierr 106 "backend.set_health s1 foo" varnish v1 -clierr 106 "backend.set_health s2 foo" varnish v1 -clierr 106 "backend.set_health s2 auto" diff --git a/bin/varnishtest/tests/d00005.vtc b/bin/varnishtest/tests/d00005.vtc index 4d3b857..1706901 100644 --- a/bin/varnishtest/tests/d00005.vtc +++ b/bin/varnishtest/tests/d00005.vtc @@ -61,7 +61,10 @@ client c1 { expect resp.body == "22" } -run +varnish v1 -vsl_catchup + varnish v1 -cliok "backend.set_health s1 sick" +varnish v1 -cliok "backend.list" client c1 { txreq @@ -72,7 +75,10 @@ client c1 { expect resp.body == "22" } -run +varnish v1 -vsl_catchup + varnish v1 -cliok "backend.set_health s2 sick" +varnish v1 -cliok "backend.list" client c1 { txreq diff --git a/include/vrt.h b/include/vrt.h index 7939459..2e776c3 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,6 +52,8 @@ * binary/load-time compatible, increment MAJOR version * * + * TRUNK (2018-09-15) + * VRT_Healthy() changed prototype * 7.0 (2018-03-15) * lots of stuff moved from cache.h to cache_varnishd.h * (ie: from "$Abi vrt" to "$Abi strict") @@ -416,7 +418,7 @@ struct vsmw_cluster *VRT_VSM_Cluster_New(VRT_CTX, size_t); void VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsmw_cluster **); /* cache_director.c */ -int VRT_Healthy(VRT_CTX, VCL_BACKEND); +VCL_BOOL VRT_Healthy(VRT_CTX, VCL_BACKEND, VCL_TIME *); /* Suckaddr related */ int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst); diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index 01f32fa..07d8c97 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -74,7 +74,7 @@ vmod_fallback_resolve(VRT_CTX, VCL_BACKEND dir) for (u = 0; u < fb->vd->n_backend; u++) { be = fb->vd->backend[fb->cur]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - if (be->methods->healthy(ctx, be, NULL)) + if (VRT_Healthy(ctx, be, NULL)) break; if (++fb->cur == fb->vd->n_backend) fb->cur = 0; diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index 3a983a7..c69fbf1 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -73,7 +73,7 @@ vmod_rr_resolve(VRT_CTX, VCL_BACKEND dir) rr->nxt = nxt + 1; be = rr->vd->backend[nxt]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - if (be->methods->healthy(ctx, be, NULL)) + if (VRT_Healthy(ctx, be, NULL)) break; } vdir_unlock(rr->vd); diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 20cfc42..9677546 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -189,7 +189,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy) sbe = NULL; be = state->shardd->backend[c].backend; AN(be); - if (be->methods->healthy(state->ctx, be, &changed)) { + if (VRT_Healthy(state->ctx, be, &changed)) { if (skip-- == 0) { chosen = c; sbe = &state->last; @@ -324,7 +324,7 @@ sharddir_any_healthy(VRT_CTX, struct sharddir *shardd, VCL_TIME *changed) for (u = 0; u < shardd->n_backend; u++) { be = shardd->backend[u].backend; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - retval = be->methods->healthy(ctx, be, &c); + retval = VRT_Healthy(ctx, be, &c); if (changed != NULL && c > *changed) *changed = c; if (retval) diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 81cb8ee..94bb195 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -70,7 +70,7 @@ vdir_new(VRT_CTX, struct vdir **vdp, const char *vcl_name, vd->dir->methods = m; REPLACE(vd->dir->vcl_name, vcl_name); vd->dir->priv = priv; - vd->dir->admin_health = VDI_AH_HEALTHY; + vd->dir->admin_health = VDI_AH_PROBE; vd->vbm = vbit_new(8); AN(vd->vbm); } @@ -188,7 +188,7 @@ vdir_any_healthy(VRT_CTX, struct vdir *vd, VCL_TIME *changed) for (u = 0; u < vd->n_backend; u++) { be = vd->backend[u]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); - retval = be->methods->healthy(ctx, be, &c); + retval = VRT_Healthy(ctx, be, &c); if (changed != NULL && c > *changed) *changed = c; if (retval) @@ -230,7 +230,7 @@ vdir_pick_be(VRT_CTX, struct vdir *vd, double w) CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); vdir_wrlock(vd); for (u = 0; u < vd->n_backend; u++) { - if (vd->backend[u]->methods->healthy(ctx, vd->backend[u], NULL)) { + if (VRT_Healthy(ctx, vd->backend[u], NULL)) { vbit_clr(vd->vbm, u); tw += vd->weight[u]; } else diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 8651993..a4a2e0f 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -209,7 +209,7 @@ vmod_healthy(VRT_CTX, VCL_BACKEND be) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_ORNULL(be, DIRECTOR_MAGIC); - return (VRT_Healthy(ctx, be)); + return (VRT_Healthy(ctx, be, NULL)); } VCL_INT v_matchproto_(td_std_port) From phk at FreeBSD.org Thu Apr 26 08:31:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Apr 2018 08:31:11 +0000 (UTC) Subject: [master] b2155ae Make sure daemonized haproxy processes gets killed Message-ID: <20180426083111.68CB6B681D@lists.varnish-cache.org> commit b2155ae46d0887526a51cefd81ac33eddd39f141 Author: Poul-Henning Kamp Date: Thu Apr 26 08:29:21 2018 +0000 Make sure daemonized haproxy processes gets killed diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 8d4118f..35dd2e8 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -327,15 +327,16 @@ haproxy_wait(struct haproxy *h) sig = SIGINT; n = 0; vtc_log(h->vl, 2, "Stop HAproxy pid=%ld", (long)h->pid); - while (!h->opt_check_mode && !h->its_dead_jim) { + while (h->opt_daemon || (!h->opt_check_mode && !h->its_dead_jim)) { assert(h->pid > 0); if (n == 0) { - i= kill(h->pid, sig); - vtc_log(h->vl, 4, - "Kill(%d)=%d: %s", sig, i, strerror(errno)); - h->expect_signal = -sig; + i = kill(h->pid, sig); + if (i == 0) + h->expect_signal = -sig; if (i && errno == ESRCH) break; + vtc_log(h->vl, 4, + "Kill(%d)=%d: %s", sig, i, strerror(errno)); } usleep(100000); if (++n == 20) { From fgsch at lodoss.net Thu Apr 26 09:38:13 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 26 Apr 2018 09:38:13 +0000 (UTC) Subject: [master] 70b5a72 On startup, tell what varnish version this is Message-ID: <20180426093813.EFF11B7CB4@lists.varnish-cache.org> commit 70b5a727e18c4ea5c8a9d7cab6b7c68ff2325537 Author: Federico G. Schwindt Date: Thu Apr 26 10:25:26 2018 +0100 On startup, tell what varnish version this is Fixes #2661. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 237a45b..cad50a4 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -842,6 +842,7 @@ main(int argc, char * const *argv) assert(!VPF_Write(pfh1)); assert(pfh2 == NULL || !VPF_Write(pfh2)); + MGT_Complain(C_DEBUG, "Version: %s", VCS_version); MGT_Complain(C_DEBUG, "Platform: %s", VSB_data(vident) + 1); if (d_flag) From fgsch at lodoss.net Thu Apr 26 09:38:13 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 26 Apr 2018 09:38:13 +0000 (UTC) Subject: [master] 42f7fc2 Ensure sphinx is in the PATH Message-ID: <20180426093813.E9406B7C7E@lists.varnish-cache.org> commit 42f7fc230d61005f2d967a32d6c01252e6a0023c Author: Federico G. Schwindt Date: Thu Apr 26 10:25:09 2018 +0100 Ensure sphinx is in the PATH diff --git a/.travis.yml b/.travis.yml index 2a84088..2cae8db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,7 @@ before_install: - ./configure ${CONFIGURE_ARGS} script: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - export PYTHONPATH=`brew --prefix`/lib/python2.7/site-packages; fi + export PYTHONPATH=`brew --prefix`/lib/python2.7/site-packages; + export PATH="/usr/local/opt/sphinx-doc/bin:$PATH"; fi - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then make -j3 distcheck; fi - if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then make -j3 check VERBOSE=1; fi From guillaume at varnish-software.com Thu Apr 26 19:55:19 2018 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 26 Apr 2018 19:55:19 +0000 (UTC) Subject: [master] 9efef69 Include header to support musl Message-ID: <20180426195519.87C0F9570D@lists.varnish-cache.org> commit 9efef6933a61901078aa1304be0ad189231293f5 Author: Guillaume Quintard Date: Thu Apr 26 21:50:38 2018 +0200 Include header to support musl diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 35dd2e8..88162a9 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -33,6 +33,7 @@ #include #include #include +#include /* for MUSL (mode_t) */ #include #include "vtc.h" From phk at FreeBSD.org Thu Apr 26 21:18:12 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Apr 2018 21:18:12 +0000 (UTC) Subject: [master] eb50758 Use VCL_BACKEND type Message-ID: <20180426211812.C29F297035@lists.varnish-cache.org> commit eb507582e77a83e995a727961ade2673be3efac2 Author: Poul-Henning Kamp Date: Thu Apr 26 18:48:23 2018 +0000 Use VCL_BACKEND type diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 3e502ec..00de48e 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -42,25 +42,22 @@ struct vcldir; typedef VCL_BOOL vdi_healthy_f(VRT_CTX, VCL_BACKEND, VCL_TIME *); typedef VCL_BACKEND vdi_resolve_f(VRT_CTX, VCL_BACKEND); -typedef int vdi_gethdrs_f(const struct director *, struct worker *, - struct busyobj *); -typedef int vdi_getbody_f(const struct director *, struct worker *, - struct busyobj *); -typedef const struct suckaddr *vdi_getip_f(const struct director *, +typedef int vdi_gethdrs_f(VCL_BACKEND, struct worker *, struct busyobj *); +typedef int vdi_getbody_f(VCL_BACKEND, struct worker *, struct busyobj *); +typedef const struct suckaddr *vdi_getip_f(VCL_BACKEND, struct worker *, struct busyobj *); -typedef void vdi_finish_f(const struct director *, struct worker *, - struct busyobj *); +typedef void vdi_finish_f(VCL_BACKEND, struct worker *, struct busyobj *); -typedef enum sess_close vdi_http1pipe_f(const struct director *, struct req *, +typedef enum sess_close vdi_http1pipe_f(VCL_BACKEND, struct req *, struct busyobj *); -typedef void vdi_event_f(const struct director *, enum vcl_event_e); +typedef void vdi_event_f(VCL_BACKEND, enum vcl_event_e); -typedef void vdi_destroy_f(const struct director *); +typedef void vdi_destroy_f(VCL_BACKEND); -typedef void vdi_panic_f(const struct director *, struct vsb *); +typedef void vdi_panic_f(VCL_BACKEND, struct vsb *); -typedef void vdi_list_f(const struct director *, struct vsb *, int, int); +typedef void vdi_list_f(VCL_BACKEND, struct vsb *, int, int); struct director_methods { unsigned magic; From phk at FreeBSD.org Thu Apr 26 21:18:12 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Apr 2018 21:18:12 +0000 (UTC) Subject: [master] c857656 Tell which director the NULL backend was attempted on Message-ID: <20180426211812.D81CB97038@lists.varnish-cache.org> commit c8576564c1e8287f899b5b7874fd88b4e1928cc3 Author: Poul-Henning Kamp Date: Thu Apr 26 21:15:30 2018 +0000 Tell which director the NULL backend was attempted on diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 94bb195..ffecf08 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -121,7 +121,8 @@ vdir_add_backend(VRT_CTX, struct vdir *vd, VCL_BACKEND be, double weight) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); if (be == NULL) { - VRT_fail(ctx, "NULL backend cannot be added"); + VRT_fail(ctx, "%s: NULL backend cannot be added", + vd->dir->vcl_name); return; } AN(be); @@ -144,7 +145,8 @@ vdir_remove_backend(VRT_CTX, struct vdir *vd, VCL_BACKEND be, unsigned *cur) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); if (be == NULL) { - VRT_fail(ctx, "NULL backend cannot be removed"); + VRT_fail(ctx, "%s: NULL backend cannot be removed", + vd->dir->vcl_name); return; } CHECK_OBJ(be, DIRECTOR_MAGIC); From phk at FreeBSD.org Thu Apr 26 21:18:12 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Apr 2018 21:18:12 +0000 (UTC) Subject: [master] 1113031 Shuffle code from backend to director level Message-ID: <20180426211812.F33759703C@lists.varnish-cache.org> commit 1113031f9cc5a6c65af8924dbd7ab72f23bdd872 Author: Poul-Henning Kamp Date: Thu Apr 26 21:16:46 2018 +0000 Shuffle code from backend to director level diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 21258a8..1171c27 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -485,12 +485,17 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, d = be->director; INIT_OBJ(d, DIRECTOR_MAGIC); d->priv = be; - d->vcl_name = be->vcl_name; d->methods = vbe_methods; - d->health = 1; - d->health_changed = VTIM_real(); - d->admin_health = VDI_AH_PROBE; + be->vsc = VSC_vbe_New(vc, &be->vsc_seg, + "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name); + AN(be->vsc); + + retval = VRT_AddDirector(ctx, d, "%s", vrt->vcl_name); + if (retval) { + VRT_delete_backend(ctx, &d); + return (NULL); + } vbp = vrt->probe; if (vbp == NULL) @@ -508,18 +513,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, VBP_Insert(be, vbp, be->tcp_pool); } - be->vsc = VSC_vbe_New(vc, &be->vsc_seg, - "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name); - AN(be->vsc); - - retval = VRT_AddDirector(ctx, d, vrt->vcl_name); - - if (retval == 0) - return (d); - - VRT_delete_backend(ctx, &d); - AZ(d); - return (NULL); + return (d); } struct director * v_matchproto_() diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 00de48e..2918bc1 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -98,7 +98,9 @@ struct director { /* cache_vcl.c */ -int VRT_AddDirector(VRT_CTX, struct director *, const char *); +int VRT_AddDirector(VRT_CTX, struct director *, const char *, ...) + v_printflike_(3, 4); + void VRT_DelDirector(VRT_CTX, struct director *); /* cache_director.c */ diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index 0d32de6..6ef4758 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -38,6 +38,7 @@ #include "vcl.h" #include "vct.h" +#include "vtim.h" #include "cache_director.h" #include "cache_vcl.h" @@ -126,11 +127,13 @@ VCL_Rel(struct vcl **vcc) /*--------------------------------------------------------------------*/ int -VRT_AddDirector(VRT_CTX, struct director *d, const char *vcl_name) +VRT_AddDirector(VRT_CTX, struct director *d, const char *fmt, ...) { struct vsb *vsb; struct vcl *vcl; struct vcldir *vdir; + va_list ap; + int i; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); vcl = ctx->vcl; @@ -147,16 +150,24 @@ VRT_AddDirector(VRT_CTX, struct director *d, const char *vcl_name) d->admin_health = VDI_AH_PROBE; vsb = VSB_new_auto(); AN(vsb); - VSB_printf(vsb, "%s.%s", VCL_Name(vcl), vcl_name); + VSB_printf(vsb, "%s.", VCL_Name(vcl)); + i = VSB_len(vsb); + va_start(ap, fmt); + VSB_vprintf(vsb, fmt, ap); + va_end(ap); AZ(VSB_finish(vsb)); REPLACE((d->cli_name), VSB_data(vsb)); VSB_destroy(&vsb); + d->vcl_name = d->cli_name + i; ALLOC_OBJ(vdir, VCLDIR_MAGIC); AN(vdir); vdir->dir = d; vdir->vcl = vcl; d->vdir = vdir; + d->health = 1; + d->admin_health = VDI_AH_PROBE; + d->health_changed = VTIM_real(); Lck_Lock(&vcl_mtx); VTAILQ_INSERT_TAIL(&vcl->director_list, vdir, list); From fgsch at lodoss.net Thu Apr 26 23:20:15 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 26 Apr 2018 23:20:15 +0000 (UTC) Subject: [master] 5220c39 Fix compiling under epel6 Message-ID: <20180426232015.2105599523@lists.varnish-cache.org> commit 5220c394232c25bb7a807a35e7394059ecefa821 Author: Federico G. Schwindt Date: Fri Apr 27 00:07:14 2018 +0100 Fix compiling under epel6 Reported by ingvar diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 7ab093b..5e3816b 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -485,7 +485,7 @@ process_stdout(const struct vev *ev, int what) vtc_dump(p->vl, 4, "stdout", buf, i); else if (p->log == 3) vtc_hexdump(p->vl, 4, "stdout", buf, i); - (void)write(p->f_stdout, buf, i); + assert(write(p->f_stdout, buf, i) == i); AZ(pthread_mutex_lock(&p->mtx)); teken_input(p->tek, buf, i); AZ(pthread_mutex_unlock(&p->mtx)); @@ -510,7 +510,7 @@ process_stderr(const struct vev *ev, int what) p->stderr_bytes += i; AZ(pthread_mutex_unlock(&p->mtx)); vtc_dump(p->vl, 4, "stderr", buf, i); - (void)write(p->f_stderr, buf, i); + assert(write(p->f_stderr, buf, i) == i); return (0); } From phk at FreeBSD.org Fri Apr 27 14:04:25 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 27 Apr 2018 14:04:25 +0000 (UTC) Subject: [master] 4d56826 Add functions to manipulate VCL_BACKEND state Message-ID: <20180427140426.061BFB1631@lists.varnish-cache.org> commit 4d568263bf3f25d2b591befaca8df2249cf39f8a Author: Poul-Henning Kamp Date: Fri Apr 27 07:47:42 2018 +0000 Add functions to manipulate VCL_BACKEND state diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 1fc37aa..18cd2ff 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -539,8 +539,7 @@ VRT_delete_backend(VRT_CTX, VCL_BACKEND *dp) TAKE_OBJ_NOTNULL(d, dp, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC); Lck_Lock(&be->mtx); - be->director->admin_health = VDI_AH_DELETED; - be->director->health_changed = VTIM_real(); + VRT_DisableDirector(be->director); be->cooled = VTIM_real() + 60.; Lck_Unlock(&be->mtx); Lck_Lock(&backends_mtx); diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index caf3673..7e17793 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -175,7 +175,7 @@ vbp_update_backend(struct vbp_target *vt) vt->backend->director->health_changed = VTIM_real(); } - vt->backend->director->health = 1; + VRT_SetHealth(vt->backend->director, 1); } else { if (vt->backend->director->health) { logmsg = "Went sick"; @@ -184,7 +184,7 @@ vbp_update_backend(struct vbp_target *vt) } else { logmsg = "Still sick"; } - vt->backend->director->health = 0; + VRT_SetHealth(vt->backend->director, 0); } VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", vt->backend->director->cli_name, logmsg, bits, @@ -637,7 +637,7 @@ VBP_Remove(struct backend *be) CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); Lck_Lock(&vbp_mtx); - be->director->health = 1; + VRT_SetHealth(be->director, 1); be->probe = NULL; vt->backend = NULL; if (vt->running) { diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 668a3a7..6a84263 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -101,6 +101,8 @@ struct director { int VRT_AddDirector(VRT_CTX, struct director *, const char *, ...) v_printflike_(3, 4); +void VRT_SetHealth(VCL_BACKEND d, int health); +void VRT_DisableDirector(VCL_BACKEND); void VRT_DelDirector(struct director *); /* cache_director.c */ diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index d276aa7..815f048 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -208,6 +208,32 @@ VRT_DelDirector(struct director *d) FREE_OBJ(vdir); } +void +VRT_SetHealth(VCL_BACKEND d, int health) +{ + struct vcldir *vdir; + + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + vdir = d->vdir; + CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC); + + vdir->dir->health = health; +} + +void +VRT_DisableDirector(VCL_BACKEND d) +{ + struct vcldir *vdir; + + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + vdir = d->vdir; + CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC); + + vdir->dir->admin_health = VDI_AH_DELETED; + vdir->dir->health = 0; + vdir->dir->health_changed = VTIM_real(); +} + /*--------------------------------------------------------------------*/ VCL_BACKEND From phk at FreeBSD.org Fri Apr 27 14:04:25 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 27 Apr 2018 14:04:25 +0000 (UTC) Subject: [master] a59d33e Register all directors&backends, and require them to be so. Message-ID: <20180427140425.E645DB162E@lists.varnish-cache.org> commit a59d33e4296246b593d7e3ead97ba817ec53bb21 Author: Poul-Henning Kamp Date: Fri Apr 27 06:58:43 2018 +0000 Register all directors&backends, and require them to be so. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 1171c27..1e17934 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -578,7 +578,7 @@ VBE_Poll(void) if (be->n_conn > 0) continue; Lck_Unlock(&backends_mtx); - VRT_DelDirector(NULL, be->director); + VRT_DelDirector(be->director); Lck_Lock(&backends_mtx); } Lck_Unlock(&backends_mtx); diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 7e8b4bc..d93482e 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -85,6 +85,7 @@ vdi_resolve(struct busyobj *bo) struct vrt_ctx ctx; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_ORNULL(bo->director_req, DIRECTOR_MAGIC); INIT_OBJ(&ctx, VRT_CTX_MAGIC); ctx.vcl = bo->vcl; @@ -100,6 +101,7 @@ vdi_resolve(struct busyobj *bo) for (d = bo->director_req; d != NULL && d->methods->resolve != NULL; d = d2) { CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + AN(d->vdir); d2 = d->methods->resolve(&ctx, d); if (d2 == NULL) VSLb(bo->vsl, SLT_FetchError, @@ -108,6 +110,8 @@ vdi_resolve(struct busyobj *bo) CHECK_OBJ_ORNULL(d, DIRECTOR_MAGIC); if (d == NULL) VSLb(bo->vsl, SLT_FetchError, "No backend"); + else + AN(d->vdir); bo->director_resp = d; return (d); } diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 2918bc1..668a3a7 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -101,7 +101,7 @@ struct director { int VRT_AddDirector(VRT_CTX, struct director *, const char *, ...) v_printflike_(3, 4); -void VRT_DelDirector(VRT_CTX, struct director *); +void VRT_DelDirector(struct director *); /* cache_director.c */ diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index 6ef4758..77cf521 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -139,7 +139,7 @@ VRT_AddDirector(VRT_CTX, struct director *d, const char *fmt, ...) vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(d->methods, DIRECTOR_METHODS_MAGIC); - AN(d->methods->destroy); + // AN(d->methods->destroy); AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); if (vcl->temp == VCL_TEMP_COOLING) { @@ -184,12 +184,11 @@ VRT_AddDirector(VRT_CTX, struct director *d, const char *fmt, ...) } void -VRT_DelDirector(VRT_CTX, struct director *d) +VRT_DelDirector(struct director *d) { struct vcl *vcl; struct vcldir *vdir; - (void)ctx; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); vdir = d->vdir; CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC); @@ -203,9 +202,9 @@ VRT_DelDirector(VRT_CTX, struct director *d) if (VCL_WARM(vcl)) VDI_Event(d, VCL_EVENT_COLD); AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - AN(d->methods->destroy); REPLACE(d->cli_name, NULL); - d->methods->destroy(d); + if(d->methods->destroy != NULL) + d->methods->destroy(d); FREE_OBJ(vdir); } diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index ffecf08..7763486 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -68,9 +68,8 @@ vdir_new(VRT_CTX, struct vdir **vdp, const char *vcl_name, ALLOC_OBJ(vd->dir, DIRECTOR_MAGIC); AN(vd->dir); vd->dir->methods = m; - REPLACE(vd->dir->vcl_name, vcl_name); vd->dir->priv = priv; - vd->dir->admin_health = VDI_AH_PROBE; + AZ(VRT_AddDirector(ctx, vd->dir, "%s", vcl_name)); vd->vbm = vbit_new(8); AN(vd->vbm); } @@ -85,7 +84,7 @@ vdir_delete(struct vdir **vdp) free(vd->backend); free(vd->weight); AZ(pthread_rwlock_destroy(&vd->mtx)); - free(vd->dir->vcl_name); + VRT_DelDirector(vd->dir); FREE_OBJ(vd->dir); vbit_destroy(vd->vbm); FREE_OBJ(vd); diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index b546495..8e39aeb 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -218,10 +218,9 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, vshard->param = &shard_param_default; ALLOC_OBJ(vshard->dir, DIRECTOR_MAGIC); AN(vshard->dir); - REPLACE(vshard->dir->vcl_name, vcl_name); vshard->dir->priv = vshard; vshard->dir->methods = vmod_shard_methods; - vshard->dir->admin_health = VDI_AH_HEALTHY; + AZ(VRT_AddDirector(ctx, vshard->dir, "%s", vcl_name)); } VCL_VOID v_matchproto_(td_directors_shard__fini) @@ -232,7 +231,7 @@ vmod_shard__fini(struct vmod_directors_shard **vshardp) *vshardp = NULL; CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); sharddir_delete(&vshard->shardd); - free(vshard->dir->vcl_name); + VRT_DelDirector(vshard->dir); FREE_OBJ(vshard->dir); FREE_OBJ(vshard); } From phk at FreeBSD.org Fri Apr 27 14:04:25 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 27 Apr 2018 14:04:25 +0000 (UTC) Subject: [master] c9d3b3f Change from "struct director *" to VCL_BACKEND (=constify) Message-ID: <20180427140426.0B789B1632@lists.varnish-cache.org> commit c9d3b3ff5ff3fc400fe72f1e7d66693f88e1b04f Author: Poul-Henning Kamp Date: Fri Apr 27 07:25:12 2018 +0000 Change from "struct director *" to VCL_BACKEND (=constify) diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 1e17934..1fc37aa 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -448,12 +448,13 @@ VRT_backend_vsm_need(VRT_CTX) return (VRT_VSC_Overhead(VSC_vbe_size)); } -struct director * v_matchproto_() +VCL_BACKEND v_matchproto_() VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, const struct vrt_backend *vrt) { struct backend *be; struct director *d; + VCL_BACKEND bb; struct vcl *vcl; const struct vrt_backend_probe *vbp; int retval; @@ -493,7 +494,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, retval = VRT_AddDirector(ctx, d, "%s", vrt->vcl_name); if (retval) { - VRT_delete_backend(ctx, &d); + bb = d; + VRT_delete_backend(ctx, &bb); return (NULL); } @@ -516,7 +518,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, return (d); } -struct director * v_matchproto_() +VCL_BACKEND v_matchproto_() VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt) { return (VRT_new_backend_clustered(ctx, NULL, vrt)); @@ -528,9 +530,9 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt) */ void -VRT_delete_backend(VRT_CTX, struct director **dp) +VRT_delete_backend(VRT_CTX, VCL_BACKEND *dp) { - struct director *d; + VCL_BACKEND d; struct backend *be; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index bec82c1..e43a782 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -385,7 +385,7 @@ enum vry_finish_flag { KEEP, DISCARD }; void VRY_Finish(struct req *req, enum vry_finish_flag); /* cache_vcl.c */ -struct director *VCL_DefaultDirector(const struct vcl *); +VCL_BACKEND VCL_DefaultDirector(const struct vcl *); const struct vrt_backend_probe *VCL_DefaultProbe(const struct vcl *); void VCL_Init(void); void VCL_Panic(struct vsb *, const struct vcl *); diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index 77cf521..d276aa7 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -210,7 +210,7 @@ VRT_DelDirector(struct director *d) /*--------------------------------------------------------------------*/ -struct director * +VCL_BACKEND VCL_DefaultDirector(const struct vcl *vcl) { diff --git a/include/vrt.h b/include/vrt.h index 2e776c3..37a0937 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -406,12 +406,12 @@ VCL_VOID VRT_Rollback(VRT_CTX, VCL_HTTP); VCL_VOID VRT_synth_page(VRT_CTX, const char *, ...); /* Backend related */ -struct director *VRT_new_backend(VRT_CTX, const struct vrt_backend *); -struct director *VRT_new_backend_clustered(VRT_CTX, +VCL_BACKEND VRT_new_backend(VRT_CTX, const struct vrt_backend *); +VCL_BACKEND VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *, const struct vrt_backend *); size_t VRT_backend_vsm_need(VRT_CTX); -void VRT_delete_backend(VRT_CTX, struct director **); -int VRT_backend_healthy(VRT_CTX, struct director *); +void VRT_delete_backend(VRT_CTX, VCL_BACKEND *); +int VRT_backend_healthy(VRT_CTX, VCL_BACKEND); /* VSM related */ struct vsmw_cluster *VRT_VSM_Cluster_New(VRT_CTX, size_t); diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index b7d81d4..1d45837 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -656,7 +656,7 @@ struct VCL_conf { #define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */ unsigned syntax; - struct director **default_director; + VCL_BACKEND *default_director; const struct vrt_backend_probe *default_probe; unsigned nref; const struct vrt_ref *ref; diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 3f21cdb..4243b9d 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -547,7 +547,7 @@ vcc_ParseBackend(struct vcc *tl) sym->noref = 1; } } - Fh(tl, 0, "\nstatic struct director *%s;\n", dn); + Fh(tl, 0, "\nstatic VCL_BACKEND %s;\n", dn); vcc_ParseHostDef(tl, t_be, dn); if (tl->err) { VSB_printf(tl->sb, diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index 5fa8833..aa78bfb 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -48,7 +48,7 @@ struct xyzzy_debug_dyn { #define VMOD_DEBUG_DYN_MAGIC 0x9b77ccbd pthread_mutex_t mtx; char *vcl_name; - struct director *dir; + VCL_BACKEND dir; }; struct xyzzy_debug_dyn_uds { @@ -56,7 +56,7 @@ struct xyzzy_debug_dyn_uds { #define VMOD_DEBUG_UDS_MAGIC 0x6c7370e6 pthread_mutex_t mtx; char *vcl_name; - struct director *dir; + VCL_BACKEND dir; }; static void @@ -65,7 +65,7 @@ dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, { struct addrinfo hints, *res = NULL; struct suckaddr *sa; - struct director *dir, *dir2; + VCL_BACKEND dir, dir2; struct vrt_backend vrt; CHECK_OBJ_NOTNULL(dyn, VMOD_DEBUG_DYN_MAGIC); @@ -183,7 +183,7 @@ xyzzy_dyn_refresh(VRT_CTX, struct xyzzy_debug_dyn *dyn, static int dyn_uds_init(VRT_CTX, struct xyzzy_debug_dyn_uds *uds, VCL_STRING path) { - struct director *dir, *dir2; + VCL_BACKEND dir, dir2; struct vrt_backend vrt; struct stat st; From phk at FreeBSD.org Fri Apr 27 14:04:26 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 27 Apr 2018 14:04:26 +0000 (UTC) Subject: [master] 2f869e0 Allocate the director structure centrally Message-ID: <20180427140426.1977BB1634@lists.varnish-cache.org> commit 2f869e04f619bfdce3249812271b082c1d2ec9ff Author: Poul-Henning Kamp Date: Fri Apr 27 09:10:12 2018 +0000 Allocate the director structure centrally diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 18cd2ff..fcd48cb 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -453,11 +453,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, const struct vrt_backend *vrt) { struct backend *be; - struct director *d; - VCL_BACKEND bb; struct vcl *vcl; const struct vrt_backend_probe *vbp; - int retval; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC); @@ -483,19 +480,15 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, #undef DA #undef DN - d = be->director; - INIT_OBJ(d, DIRECTOR_MAGIC); - d->priv = be; - d->methods = vbe_methods; - be->vsc = VSC_vbe_New(vc, &be->vsc_seg, "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name); AN(be->vsc); - retval = VRT_AddDirector(ctx, d, "%s", vrt->vcl_name); - if (retval) { - bb = d; - VRT_delete_backend(ctx, &bb); + be->director = VRT_AddDirector(ctx, vbe_methods, be, + "%s", vrt->vcl_name); + if (be->director == NULL) { + // XXX + VRT_delete_backend(ctx, &be->director); return (NULL); } @@ -515,7 +508,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, VBP_Insert(be, vbp, be->tcp_pool); } - return (d); + return (be->director); } VCL_BACKEND v_matchproto_() @@ -579,7 +572,7 @@ VBE_Poll(void) if (be->n_conn > 0) continue; Lck_Unlock(&backends_mtx); - VRT_DelDirector(be->director); + VRT_DelDirector(&be->director); Lck_Lock(&backends_mtx); } Lck_Unlock(&backends_mtx); diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 5f8eca1..1872c70 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -64,7 +64,7 @@ struct backend { struct tcp_pool *tcp_pool; - struct director director[1]; + VCL_BACKEND director; double cooled; }; diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 7e17793..dbf0520 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -172,19 +172,15 @@ vbp_update_backend(struct vbp_target *vt) logmsg = "Still healthy"; } else { logmsg = "Back healthy"; - vt->backend->director->health_changed = - VTIM_real(); + VRT_SetHealth(vt->backend->director, 1); } - VRT_SetHealth(vt->backend->director, 1); } else { if (vt->backend->director->health) { logmsg = "Went sick"; - vt->backend->director->health_changed = - VTIM_real(); + VRT_SetHealth(vt->backend->director, 0); } else { logmsg = "Still sick"; } - VRT_SetHealth(vt->backend->director, 0); } VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", vt->backend->director->cli_name, logmsg, bits, diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 6a84263..7cce8cf 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -83,7 +83,6 @@ struct director { char *vcl_name; void *priv; - const void *priv2; /* Internal Housekeeping fields */ @@ -98,12 +97,12 @@ struct director { /* cache_vcl.c */ -int VRT_AddDirector(VRT_CTX, struct director *, const char *, ...) - v_printflike_(3, 4); +VCL_BACKEND VRT_AddDirector(VRT_CTX, const struct director_methods *, + void *, const char *, ...) v_printflike_(4, 5); void VRT_SetHealth(VCL_BACKEND d, int health); void VRT_DisableDirector(VCL_BACKEND); -void VRT_DelDirector(struct director *); +void VRT_DelDirector(VCL_BACKEND *); /* cache_director.c */ diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c index 815f048..6e45c9f 100644 --- a/bin/varnishd/cache/cache_vcl_vrt.c +++ b/bin/varnishd/cache/cache_vcl_vrt.c @@ -126,27 +126,32 @@ VCL_Rel(struct vcl **vcc) /*--------------------------------------------------------------------*/ -int -VRT_AddDirector(VRT_CTX, struct director *d, const char *fmt, ...) +VCL_BACKEND +VRT_AddDirector(VRT_CTX, const struct director_methods *m, void *priv, + const char *fmt, ...) { struct vsb *vsb; struct vcl *vcl; struct vcldir *vdir; + struct director *d; va_list ap; int i; - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(m, DIRECTOR_METHODS_MAGIC); + AN(fmt); vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(d->methods, DIRECTOR_METHODS_MAGIC); - // AN(d->methods->destroy); - AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); if (vcl->temp == VCL_TEMP_COOLING) { AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - return (1); + return (NULL); } + ALLOC_OBJ(d, DIRECTOR_MAGIC); + AN(d); + d->methods = m; + d->priv = priv; d->admin_health = VDI_AH_PROBE; vsb = VSB_new_auto(); AN(vsb); @@ -180,16 +185,17 @@ VRT_AddDirector(VRT_CTX, struct director *d, const char *fmt, ...) WRONG("Dynamic Backends can only be added to warm VCLs"); AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - return (0); + return (d); } void -VRT_DelDirector(struct director *d) +VRT_DelDirector(VCL_BACKEND *bp) { struct vcl *vcl; struct vcldir *vdir; + VCL_BACKEND d; - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + TAKE_OBJ_NOTNULL(d, bp, DIRECTOR_MAGIC); vdir = d->vdir; CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC); vcl = vdir->vcl; @@ -202,9 +208,10 @@ VRT_DelDirector(struct director *d) if (VCL_WARM(vcl)) VDI_Event(d, VCL_EVENT_COLD); AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); - REPLACE(d->cli_name, NULL); if(d->methods->destroy != NULL) d->methods->destroy(d); + free(d->cli_name); + FREE_OBJ(vdir->dir); FREE_OBJ(vdir); } @@ -218,6 +225,7 @@ VRT_SetHealth(VCL_BACKEND d, int health) CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC); vdir->dir->health = health; + vdir->dir->health_changed = VTIM_real(); } void diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 8ec868d..4664df7 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -94,6 +94,7 @@ +libh mgt_event.h +-sem(VRT_AddDirector, custodial(3)) -sem(VCP_New, custodial(3)) -sem(vsmw_addseg, custodial(2)) -sem(BAN_Free, custodial(1)) diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 7763486..226e060 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -64,12 +64,7 @@ vdir_new(VRT_CTX, struct vdir **vdp, const char *vcl_name, AN(vd); *vdp = vd; AZ(pthread_rwlock_init(&vd->mtx, NULL)); - - ALLOC_OBJ(vd->dir, DIRECTOR_MAGIC); - AN(vd->dir); - vd->dir->methods = m; - vd->dir->priv = priv; - AZ(VRT_AddDirector(ctx, vd->dir, "%s", vcl_name)); + vd->dir = VRT_AddDirector(ctx, m, priv, "%s", vcl_name); vd->vbm = vbit_new(8); AN(vd->vbm); } @@ -84,8 +79,7 @@ vdir_delete(struct vdir **vdp) free(vd->backend); free(vd->weight); AZ(pthread_rwlock_destroy(&vd->mtx)); - VRT_DelDirector(vd->dir); - FREE_OBJ(vd->dir); + VRT_DelDirector(&vd->dir); vbit_destroy(vd->vbm); FREE_OBJ(vd); } diff --git a/lib/libvmod_directors/vdir.h b/lib/libvmod_directors/vdir.h index 6c8331d..5531931 100644 --- a/lib/libvmod_directors/vdir.h +++ b/lib/libvmod_directors/vdir.h @@ -37,7 +37,7 @@ struct vdir { VCL_BACKEND *backend; double *weight; double total_weight; - struct director *dir; + VCL_BACKEND dir; struct vbitmap *vbm; }; diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 8e39aeb..a470c6e 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -91,7 +91,7 @@ struct vmod_directors_shard_param { enum healthy_e healthy; uint32_t mask; VCL_BOOL rampup; - VCL_INT alt; + VCL_INT alt; VCL_REAL warmup; }; @@ -137,7 +137,7 @@ struct vmod_directors_shard { unsigned magic; #define VMOD_SHARD_SHARD_MAGIC 0x6e63e1bf struct sharddir *shardd; - struct director *dir; + VCL_BACKEND dir; const struct vmod_directors_shard_param *param; }; @@ -216,11 +216,8 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, sharddir_new(&vshard->shardd, vcl_name); vshard->param = &shard_param_default; - ALLOC_OBJ(vshard->dir, DIRECTOR_MAGIC); - AN(vshard->dir); - vshard->dir->priv = vshard; - vshard->dir->methods = vmod_shard_methods; - AZ(VRT_AddDirector(ctx, vshard->dir, "%s", vcl_name)); + vshard->dir = + VRT_AddDirector(ctx, vmod_shard_methods, vshard, "%s", vcl_name); } VCL_VOID v_matchproto_(td_directors_shard__fini) @@ -231,8 +228,7 @@ vmod_shard__fini(struct vmod_directors_shard **vshardp) *vshardp = NULL; CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); sharddir_delete(&vshard->shardd); - VRT_DelDirector(vshard->dir); - FREE_OBJ(vshard->dir); + VRT_DelDirector(&vshard->dir); FREE_OBJ(vshard); } From nils.goroll at uplex.de Fri Apr 27 14:40:15 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 27 Apr 2018 14:40:15 +0000 (UTC) Subject: [master] 2694906 vmodtool: require method names to start with a dot Message-ID: <20180427144015.89AE8B2077@lists.varnish-cache.org> commit 2694906c91434340dc65a69742af9a1d7a9c1356 Author: Nils Goroll Date: Fri Apr 27 16:39:06 2018 +0200 vmodtool: require method names to start with a dot diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index b6495c5..b4de6a0 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -799,6 +799,9 @@ class s_method(stanza): assert type(p) == s_object self.pfx = p.proto.name self.proto = prototype(self, prefix=self.pfx) + if not self.proto.bname.startswith("."): + err("$Method %s: Method names need to start with . (dot)" + % self.proto.bname, warn=False) self.proto.obj = "x" + self.pfx self.rstlbl = "func_" + self.proto.name p.methods.append(self) From nils.goroll at uplex.de Fri Apr 27 15:00:30 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 27 Apr 2018 15:00:30 +0000 (UTC) Subject: [master] ceaff5f vmodtool: fix object destructor signature Message-ID: <20180427150030.C4F76B27B2@lists.varnish-cache.org> commit ceaff5f4673a543e1b6eecea2a05847796642eac Author: Nils Goroll Date: Fri Apr 27 16:58:01 2018 +0200 vmodtool: fix object destructor signature ... when the constructor has optional arguments diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index b4de6a0..8937aef 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -719,6 +719,7 @@ class s_object(stanza): self.fini = copy.copy(self.proto) self.fini.name += '__fini' + self.fini.argstruct = False self.fini.args = [] self.rstlbl = "obj_" + self.proto.name From dridi at varni.sh Mon Apr 30 06:43:51 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 30 Apr 2018 08:43:51 +0200 Subject: [master] ce581db Call it "cli_name" instead of "display_name" In-Reply-To: <20180423220622.BEC5993F2A@lists.varnish-cache.org> References: <20180423220622.BEC5993F2A@lists.varnish-cache.org> Message-ID: On Tue, Apr 24, 2018 at 12:06 AM, Poul-Henning Kamp wrote: > > commit ce581db32acf2f381ec4596c6a1d873779194b1c > Author: Poul-Henning Kamp > Date: Mon Apr 23 20:52:00 2018 +0000 > > Call it "cli_name" instead of "display_name" > > diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h > index 9e98da7..ba72d8b 100644 > --- a/bin/varnishd/cache/cache_director.h > +++ b/bin/varnishd/cache/cache_director.h > @@ -80,7 +80,7 @@ struct director { > > /* Internal Housekeeping fields */ > > - char *display_name; > + char *cli_name; > VTAILQ_ENTRY(director) vcl_list; > struct vcl *vcl; Is this struct part of the VRT ABI? In which case we'd need a major bump. Dridi From dridi at varni.sh Mon Apr 30 06:48:06 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 30 Apr 2018 08:48:06 +0200 Subject: [master] cdc9884 Move Add/DelDirector into VRT, vmods will need to call them. In-Reply-To: <20180423220623.4CDE993F37@lists.varnish-cache.org> References: <20180423220623.4CDE993F37@lists.varnish-cache.org> Message-ID: On Tue, Apr 24, 2018 at 12:06 AM, Poul-Henning Kamp wrote: > > commit cdc9884aee1958f2eb37e5dacc6dac51ecaa1873 > Author: Poul-Henning Kamp > Date: Mon Apr 23 21:33:58 2018 +0000 > > Move Add/DelDirector into VRT, vmods will need to call them. > > diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h > index 91786ef..c65118b 100644 > --- a/bin/varnishd/cache/cache_director.h > +++ b/bin/varnishd/cache/cache_director.h > @@ -92,8 +92,8 @@ struct director { > unsigned VDI_Healthy(const struct director *, double *); > > /* cache_vcl.c */ > -int VCL_AddDirector(struct vcl *, struct director *, const char *); > -void VCL_DelDirector(struct director *); > +int VRT_AddDirector(VRT_CTX, struct director *, const char *); > +void VRT_DelDirector(VRT_CTX, struct director *); > Shouldn't we move the prototypes to vrt.h? Dridi From phk at phk.freebsd.dk Mon Apr 30 06:56:18 2018 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 30 Apr 2018 06:56:18 +0000 Subject: [master] cdc9884 Move Add/DelDirector into VRT, vmods will need to call them. In-Reply-To: References: <20180423220623.4CDE993F37@lists.varnish-cache.org> Message-ID: <7734.1525071378@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >> /* cache_vcl.c */ >> -int VCL_AddDirector(struct vcl *, struct director *, const char *); >> -void VCL_DelDirector(struct director *); >> +int VRT_AddDirector(VRT_CTX, struct director *, const char *); >> +void VRT_DelDirector(VRT_CTX, struct director *); >> > > >Shouldn't we move the prototypes to vrt.h? That may or may not happen once I'm done and know how the API looks. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at phk.freebsd.dk Mon Apr 30 06:57:13 2018 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 30 Apr 2018 06:57:13 +0000 Subject: [master] ce581db Call it "cli_name" instead of "display_name" In-Reply-To: References: <20180423220622.BEC5993F2A@lists.varnish-cache.org> Message-ID: <7748.1525071433@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >> @@ -80,7 +80,7 @@ struct director { > >Is this struct part of the VRT ABI? In which case we'd need a major bump. That's probably pretty ill defined right now and a clear "NO" in the future. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Mon Apr 30 10:10:28 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 30 Apr 2018 10:10:28 +0000 (UTC) Subject: [master] e37b93a Stabilize this test Message-ID: <20180430101028.8EB36ADDE2@lists.varnish-cache.org> commit e37b93a2eb03c24410bb54d682f8fa88034086f7 Author: Poul-Henning Kamp Date: Mon Apr 30 09:29:24 2018 +0000 Stabilize this test diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc index fc555ff..5813db9 100644 --- a/bin/varnishtest/tests/u00010.vtc +++ b/bin/varnishtest/tests/u00010.vtc @@ -2,7 +2,7 @@ varnishtest "trivial run of varnishtop in curses mode" server s1 { rxreq - txresp + txresp -hdr "Connection: close" } -start varnish v1 -vcl+backend {} -start From phk at FreeBSD.org Mon Apr 30 10:10:28 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 30 Apr 2018 10:10:28 +0000 (UTC) Subject: [master] 96501ef Use VRT_BACKEND_string() to get name of backends. Message-ID: <20180430101028.74916ADDE0@lists.varnish-cache.org> commit 96501efa6b4ac05871c36f45daa7a7efcfa6ad44 Author: Poul-Henning Kamp Date: Mon Apr 30 08:59:37 2018 +0000 Use VRT_BACKEND_string() to get name of backends. diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index 4674a40..152d8df 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -255,8 +255,9 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas) ident = shardd->backend[i].ident ? shardd->backend[i].ident - : shardd->backend[i].backend->vcl_name; + : VRT_BACKEND_string(shardd->backend[i].backend); + AN(ident); assert(ident[0] != '\0'); for (j = 0; j < replicas; j++) { @@ -325,11 +326,13 @@ shardcfg_backend_cmp(const struct shard_backend *a, return a->backend != b->backend; if (ai == NULL) - ai = a->backend->vcl_name; + ai = VRT_BACKEND_string(a->backend); if (bi == NULL) - bi = b->backend->vcl_name; + bi = VRT_BACKEND_string(b->backend); + AN(ai); + AN(bi); return strcmp(ai, bi); } @@ -549,7 +552,7 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd, shard_err(ctx, shardd, "(notice) backend %s%s%s " "already exists - skipping", - b->backend->vcl_name, + VRT_BACKEND_string(b->backend), ident ? "/" : "", ident ? ident : ""); break; diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 226e060..714a655 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -115,7 +115,7 @@ vdir_add_backend(VRT_CTX, struct vdir *vd, VCL_BACKEND be, double weight) CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); if (be == NULL) { VRT_fail(ctx, "%s: NULL backend cannot be added", - vd->dir->vcl_name); + VRT_BACKEND_string(vd->dir)); return; } AN(be); @@ -139,7 +139,7 @@ vdir_remove_backend(VRT_CTX, struct vdir *vd, VCL_BACKEND be, unsigned *cur) CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); if (be == NULL) { VRT_fail(ctx, "%s: NULL backend cannot be removed", - vd->dir->vcl_name); + VRT_BACKEND_string(vd->dir)); return; } CHECK_OBJ(be, DIRECTOR_MAGIC); From phk at FreeBSD.org Mon Apr 30 10:10:28 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 30 Apr 2018 10:10:28 +0000 (UTC) Subject: [master] 45472bb SLT records should use backends vcl_name Message-ID: <20180430101028.92A53ADE06@lists.varnish-cache.org> commit 45472bbd40847940ff9747348ebc6f2cc56c4a82 Author: Poul-Henning Kamp Date: Mon Apr 30 09:57:57 2018 +0000 SLT records should use backends vcl_name diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index fcd48cb..c5bf86a 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -90,7 +90,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, if (!bp->director->health) { VSLb(bo->vsl, SLT_FetchError, - "backend %s: unhealthy", bp->director->cli_name); + "backend %s: unhealthy", VRT_BACKEND_string(bp->director)); // XXX: per backend stats ? VSC_C_main->backend_unhealthy++; return (NULL); @@ -98,7 +98,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, if (bp->max_connections > 0 && bp->n_conn >= bp->max_connections) { VSLb(bo->vsl, SLT_FetchError, - "backend %s: busy", bp->director->cli_name); + "backend %s: busy", VRT_BACKEND_string(bp->director)); // XXX: per backend stats ? VSC_C_main->backend_busy++; return (NULL); @@ -118,7 +118,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, if (pfd == NULL) { VSLb(bo->vsl, SLT_FetchError, "backend %s: fail errno %d (%s)", - bp->director->cli_name, errno, strerror(errno)); + VRT_BACKEND_string(bp->director), errno, strerror(errno)); // XXX: Per backend stats ? VSC_C_main->backend_fail++; bo->htc = NULL; @@ -141,7 +141,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, PFD_LocalName(pfd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); PFD_RemoteName(pfd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s", - *fdp, bp->director->cli_name, abuf2, pbuf2, abuf1, pbuf1); + *fdp, VRT_BACKEND_string(bp->director), abuf2, pbuf2, abuf1, pbuf1); INIT_OBJ(bo->htc, HTTP_CONN_MAGIC); bo->htc->priv = pfd; @@ -173,14 +173,14 @@ vbe_dir_finish(const struct director *d, struct worker *wrk, bo->htc->doclose == SC_RX_TIMEOUT); if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) { VSLb(bo->vsl, SLT_BackendClose, "%d %s", *PFD_Fd(pfd), - bp->director->cli_name); + VRT_BACKEND_string(bp->director)); VTP_Close(&pfd); AZ(pfd); Lck_Lock(&bp->mtx); } else { assert (PFD_State(pfd) == PFD_STATE_USED); VSLb(bo->vsl, SLT_BackendReuse, "%d %s", *PFD_Fd(pfd), - bp->director->cli_name); + VRT_BACKEND_string(bp->director)); Lck_Lock(&bp->mtx); VSC_C_main->backend_recycle++; VTP_Recycle(wrk, &pfd); @@ -394,7 +394,6 @@ vbe_panic(const struct director *d, struct vsb *vsb) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); - VSB_printf(vsb, "cli_name = %s,\n", bp->director->cli_name); if (bp->ipv4_addr != NULL) VSB_printf(vsb, "ipv4 = %s,\n", bp->ipv4_addr); if (bp->ipv6_addr != NULL) diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index d93482e..e97c174 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -273,7 +273,7 @@ VDI_Panic(const struct director *d, struct vsb *vsb, const char *nm) return; VSB_printf(vsb, "%s = %p {\n", nm, d); VSB_indent(vsb, 2); - VSB_printf(vsb, "vcl_name = %s,\n", d->vcl_name); + VSB_printf(vsb, "cli_name = %s,\n", d->cli_name); VSB_printf(vsb, "health = %s,\n", d->health ? "healthy" : "sick"); VSB_printf(vsb, "admin_health = %s, changed = %f,\n", VDI_Ahealth(d), d->health_changed); diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc index 5813db9..5eee23a 100644 --- a/bin/varnishtest/tests/u00010.vtc +++ b/bin/varnishtest/tests/u00010.vtc @@ -9,6 +9,8 @@ varnish v1 -vcl+backend {} -start process p1 -dump {varnishtop -n ${v1_name}} -start +process p1 -winsz 30 80 + process p1 -expect-text 1 1 {list length} delay 1 From nils.goroll at uplex.de Mon Apr 30 11:51:16 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Apr 2018 11:51:16 +0000 (UTC) Subject: [master] fb342dd document a potential race Message-ID: <20180430115117.126F2AFA1B@lists.varnish-cache.org> commit fb342dde523c3b87a62d2186422071311ae18fdd Author: Nils Goroll Date: Mon Apr 30 13:48:16 2018 +0200 document a potential race closes #2625 diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 86777ae..fd416e8 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -667,6 +667,10 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds, if (cp != NULL) return (cp->priv); + /* + * this is racy - we could end up with additional pools on the same id / + * destination address with just a single connection + */ ALLOC_OBJ(tp, TCP_POOL_MAGIC); AN(tp); if (uds != NULL) { From nils.goroll at uplex.de Mon Apr 30 12:00:16 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Apr 2018 12:00:16 +0000 (UTC) Subject: [master] 991e101 Symbolic status for HTC FetchError Message-ID: <20180430120017.026EFAFCFE@lists.varnish-cache.org> commit 991e101bbc3e41681b0bbe462f8daa6662735007 Author: Nils Goroll Date: Wed Feb 21 14:21:19 2018 +0100 Symbolic status for HTC FetchError Merges #2577 diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 3e96b78..a6c8f83 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -182,6 +182,21 @@ SES_Get_String_Attr(const struct sess *sp, enum sess_attr a) /*--------------------------------------------------------------------*/ +const char * +HTC_Status(enum htc_status_e e) +{ + switch (e) { +#define HTC_STATUS(e, n, s, l) \ + case HTC_S_ ## e: return (s); +#include "tbl/htc.h" + default: + WRONG("HTC_Status"); + } + NEEDLESS(return (NULL)); +} + +/*--------------------------------------------------------------------*/ + void HTC_RxInit(struct http_conn *htc, struct ws *ws) { diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index e43a782..d80b347 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -343,6 +343,7 @@ void SES_Ref(struct sess *sp); void SES_Rel(struct sess *sp); int SES_Reschedule_Req(struct req *, enum task_prio); +const char * HTC_Status(enum htc_status_e); void HTC_RxInit(struct http_conn *htc, struct ws *ws); void HTC_RxPipeline(struct http_conn *htc, void *); enum htc_status_e HTC_RxStuff(struct http_conn *, htc_complete_f *, @@ -356,15 +357,8 @@ void SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src); enum htc_status_e { - HTC_S_JUNK = -5, - HTC_S_CLOSE = -4, - HTC_S_TIMEOUT = -3, - HTC_S_OVERFLOW = -2, - HTC_S_EOF = -1, - HTC_S_EMPTY = 0, - HTC_S_MORE = 1, - HTC_S_COMPLETE = 2, - HTC_S_IDLE = 3, +#define HTC_STATUS(e, n, s, l) HTC_S_ ## e = n, +#include "tbl/htc.h" }; /* cache_shmlog.c */ diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index 27bf521..2ddc581 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -192,7 +192,8 @@ V1F_FetchRespHdr(struct busyobj *bo) htc->doclose = SC_RX_OVERFLOW; break; default: - VSLb(bo->vsl, SLT_FetchError, "HTC status %d", hs); + VSLb(bo->vsl, SLT_FetchError, "HTC %s (%d)", + HTC_Status(hs), hs); htc->doclose = SC_RX_BAD; break; } diff --git a/include/Makefile.am b/include/Makefile.am index d0d88c9..3a6bb82 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -98,7 +98,8 @@ nobase_noinst_HEADERS = \ vss.h \ vtcp.h \ vtree.h \ - vus.h + vus.h \ + tbl/htc.h ## keep in sync with lib/libvcc/Makefile.am vcl.h: \ diff --git a/include/tbl/htc.h b/include/tbl/htc.h new file mode 100644 index 0000000..a46b75e --- /dev/null +++ b/include/tbl/htc.h @@ -0,0 +1,45 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * HTC status values + */ + +/*lint -save -e525 -e539 */ + +// enum htc_status_e n short long +HTC_STATUS(JUNK, -5, "junk", "Received unexpected data") +HTC_STATUS(CLOSE, -4, "close", "Connection closed") // unused? +HTC_STATUS(TIMEOUT, -3, "timeout", "Timed out") +HTC_STATUS(OVERFLOW, -2, "overflow", "Buffer too small") +HTC_STATUS(EOF, -1, "eof", "EOF received") +HTC_STATUS(EMPTY, 0, "empty", "Empty response") +HTC_STATUS(MORE, 1, "more", "More data required") +HTC_STATUS(COMPLETE, 2, "complete", "Data complete") +HTC_STATUS(IDLE, 3, "idle", "Return to waiter") +#undef HTC_STATUS +/*lint -restore */ From gquintard at users.noreply.github.com Mon Apr 9 08:58:15 2018 From: gquintard at users.noreply.github.com (guillaume quintard) Date: Mon, 09 Apr 2018 08:58:15 -0000 Subject: [master] 77a5310 Include headers to compile on musl Message-ID: <20180409085815.4E5F992D23@lists.varnish-cache.org> commit 77a5310e0c3b5b971cb752321de82c26f8e207b1 Author: Guillaume Quintard Date: Sat Apr 7 14:36:29 2018 +0200 Include headers to compile on musl diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 035e6b2..25bdfda 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -35,6 +35,7 @@ #include #include +#include /* for MUSL */ #include "vtc.h" #include "vtcp.h" diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index e852a0d..e268326 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -40,6 +40,7 @@ #include #include #include +#include /* for MUSL */ #include "compat/daemon.h" #include "vdef.h" diff --git a/lib/libvarnishapi/vxp.c b/lib/libvarnishapi/vxp.c index 039fafe..5c50498 100644 --- a/lib/libvarnishapi/vxp.c +++ b/lib/libvarnishapi/vxp.c @@ -33,6 +33,7 @@ #include #include #include +#include /* for MUSL */ #include "vdef.h" #include "vas.h" diff --git a/lib/libvarnishapi/vxp_lexer.c b/lib/libvarnishapi/vxp_lexer.c index 527bbb7..584cf4f 100644 --- a/lib/libvarnishapi/vxp_lexer.c +++ b/lib/libvarnishapi/vxp_lexer.c @@ -33,6 +33,7 @@ #include #include #include +#include /* for MUSL */ #include "vdef.h" #include "vas.h" From gquintard at users.noreply.github.com Sun Apr 29 17:32:25 2018 From: gquintard at users.noreply.github.com (guillaume quintard) Date: Sun, 29 Apr 2018 17:32:25 -0000 Subject: [master] 9f50c4b Fix gensequences for BusyBox awk Message-ID: <20180429173222.B4DEA93063@lists.varnish-cache.org> commit 9f50c4bd12303674890ca742ce6b9592a29e3ba9 Author: Guillaume Quintard Date: Thu Apr 26 21:59:41 2018 +0200 Fix gensequences for BusyBox awk I never thought that I'd have to fix a string concatenation problem in a BusyBox awk program to generate VT100 code in a container, but here we are: echo | awk 'END {print "foo" "" ++a, "foo" ++a}' should output "foo0 foo1", and for all the ?awk I tested, it does, except for BusyBox awk who thought funny to output "0 foo1", breaking the teken_state.h file. diff --git a/bin/varnishtest/gensequences b/bin/varnishtest/gensequences index 83a3d10..8c2299a 100644 --- a/bin/varnishtest/gensequences +++ b/bin/varnishtest/gensequences @@ -61,7 +61,7 @@ while (getline > 0) { l_prefix_parent[n] = prefix; l_prefix_suffix[n] = sequence[i]; if (!l_prefix_name[n]) - l_prefix_name[n] = "teken_state_" ++npr; + l_prefix_name[n] = "teken_state_" "" ++npr; prefix = n; } From gquintard at users.noreply.github.com Sun Apr 29 17:32:25 2018 From: gquintard at users.noreply.github.com (guillaume quintard) Date: Sun, 29 Apr 2018 17:32:25 -0000 Subject: [master] b573729 Make a00009.vtc pass on alpine Message-ID: <20180429173222.D4F7693064@lists.varnish-cache.org> commit b5737297178e3b657abb14ca48cd28ce03163d1c Author: Guillaume Quintard Date: Thu Apr 26 23:43:59 2018 +0200 Make a00009.vtc pass on alpine alpine's getopt returns "progname: unrecognized option: A" diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc index 37565c7..4b82d4c 100644 --- a/bin/varnishtest/tests/a00009.vtc +++ b/bin/varnishtest/tests/a00009.vtc @@ -13,7 +13,7 @@ shell -err -expect {Too many arguments for -x} "varnishd -x foo bar" shell -err -expect {Invalid -x argument} "varnishd -x foo " # This one is tricky, the getopt message on stderr is not standardized. -shell -err -expect {option --} "varnishd -A " +shell -err -expect {option} "varnishd -A " shell -err -expect {Usage: varnishd [options]} "varnishd -? " shell -err -expect {Too many arguments} "varnishd foo "