From dridi.boukelmoune at gmail.com Mon Mar 2 14:09:08 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 14:09:08 +0000 (UTC) Subject: [master] a7e9da291 Remove unused PARAM() macro arguments Message-ID: <20200302140908.7F144ABBE6@lists.varnish-cache.org> commit a7e9da291b0c15b0e8c4a1d2f8e20d83a91db5bb Author: Dridi Boukelmoune Date: Mon Mar 2 11:23:40 2020 +0100 Remove unused PARAM() macro arguments This is the first of a series of mechanical patches made possible by vim macros. This is a first step towards continuing the work initiated in 39e0bc53b512 and the first observation is that we have some dead weight in the table. diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h index d9e2b8437..0fdeaaefd 100644 --- a/bin/varnishd/common/common_param.h +++ b/bin/varnishd/common/common_param.h @@ -82,7 +82,7 @@ struct params { #define ptyp_uint unsigned #define ptyp_vsl_buffer unsigned #define ptyp_vsl_reclen unsigned -#define PARAM(nm, ty, mi, ma, de, un, fl, st, lt, fn) ptyp_##ty nm; +#define PARAM(nm, ty, mi, ma, de, un, fl, st) ptyp_##ty nm; #include #undef ptyp_bool #undef ptyp_bytes diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index adb448a91..4f417cd76 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -45,7 +45,7 @@ "\tmax_age\tmax age of free element." struct parspec mgt_parspec[] = { -#define PARAM(nm, ty, mi, ma, de, un, fl, st, lt, fn) \ +#define PARAM(nm, ty, mi, ma, de, un, fl, st) \ { #nm, tweak_##ty, &mgt_param.nm, mi, ma, st, fl, de, un }, #include "tbl/params.h" diff --git a/include/tbl/params.h b/include/tbl/params.h index a37f60bcb..db734ffcb 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -27,7 +27,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * PARAM(nm, ty, mi, ma, de, un, fl, st, lt, fn) + * PARAM(nm, ty, mi, ma, de, un, fl, st) */ /*lint -save -e525 -e539 */ @@ -55,9 +55,7 @@ PARAM( "Enabling accept_filter may prevent some requests to reach Varnish " "in the first place. Malformed requests may go unnoticed and not " "increase the client_req_400 counter. GET or HEAD requests with a " - "body may be blocked altogether.", - /* l-text */ NULL, - /* func */ NULL + "body may be blocked altogether." ) #undef XYZZY @@ -73,9 +71,7 @@ PARAM( "If we run out of resources, such as file descriptors or worker " "threads, the acceptor will sleep between accepts.\n" "This parameter (multiplicatively) reduce the sleep duration for " - "each successful accept. (ie: 0.9 = reduce by 10%)", - /* l-text */ "", - /* func */ NULL + "each successful accept. (ie: 0.9 = reduce by 10%)" ) PARAM( @@ -90,9 +86,7 @@ PARAM( "If we run out of resources, such as file descriptors or worker " "threads, the acceptor will sleep between accepts.\n" "This parameter control how much longer we sleep, each time we " - "fail to accept a new connection.", - /* l-text */ "", - /* func */ NULL + "fail to accept a new connection." ) PARAM( @@ -107,9 +101,7 @@ PARAM( "If we run out of resources, such as file descriptors or worker " "threads, the acceptor will sleep between accepts.\n" "This parameter limits how long it can sleep between attempts to " - "accept new connections.", - /* l-text */ "", - /* func */ NULL + "accept new connections." ) PARAM( @@ -121,9 +113,7 @@ PARAM( /* units */ "bool", /* flags */ 0, /* s-text */ - "Automatically restart the child/worker process if it dies.", - /* l-text */ "", - /* func */ NULL + "Automatically restart the child/worker process if it dies." ) PARAM( @@ -138,9 +128,7 @@ PARAM( "Eliminate older identical bans when a new ban is added. This saves " "CPU cycles by not comparing objects to identical bans.\n" "This is a waste of time if you have many bans which are never " - "identical.", - /* l-text */ "", - /* func */ NULL + "identical." ) PARAM( @@ -170,9 +158,7 @@ PARAM( "additional latency due to request ban testing is in the order of " "ban_cutoff / rate(bans_lurker_tests_tested). For example, for " "rate(bans_lurker_tests_tested) = 2M/s and a tolerable latency of " - "100ms, a good value for ban_cutoff may be 200K.", - /* l-text */ "", - /* func */ NULL + "100ms, a good value for ban_cutoff may be 200K." ) PARAM( @@ -189,9 +175,7 @@ PARAM( "as part of object lookup. Because many applications issue bans in " "bursts, this parameter holds the ban-lurker off until the rush is " "over.\n" - "This should be set to the approximate time which a ban-burst takes.", - /* l-text */ "", - /* func */ NULL + "This should be set to the approximate time which a ban-burst takes." ) PARAM( @@ -205,9 +189,7 @@ PARAM( /* s-text */ "The ban lurker sleeps ${ban_lurker_sleep} after examining this " "many objects." - " Use this to pace the ban-lurker if it eats too many resources.", - /* l-text */ "", - /* func */ NULL + " Use this to pace the ban-lurker if it eats too many resources." ) PARAM( @@ -222,9 +204,7 @@ PARAM( "How long the ban lurker sleeps after examining ${ban_lurker_batch} " "objects." " Use this to pace the ban-lurker if it eats too many resources.\n" - "A value of zero will disable the ban lurker entirely.", - /* l-text */ "", - /* func */ NULL + "A value of zero will disable the ban lurker entirely." ) PARAM( @@ -237,9 +217,7 @@ PARAM( /* flags */ EXPERIMENTAL, /* s-text */ "How long the ban lurker sleeps when giving way to lookup" - " due to lock contention.", - /* l-text */ "", - /* func */ NULL + " due to lock contention." ) PARAM( @@ -255,9 +233,7 @@ PARAM( "wait for this many seconds for the first byte before giving up.\n" "VCL can override this default value for each backend and backend " "request.\n" - "This parameter does not apply to pipe'ed requests.", - /* l-text */ "", - /* func */ NULL + "This parameter does not apply to pipe'ed requests." ) PARAM( @@ -272,9 +248,7 @@ PARAM( "We only wait for this many seconds between bytes received from " "the backend before giving up the fetch.\n" "VCL values, per backend or per backend request take precedence.\n" - "This parameter does not apply to pipe'ed requests.", - /* l-text */ "", - /* func */ NULL + "This parameter does not apply to pipe'ed requests." ) PARAM( @@ -286,9 +260,7 @@ PARAM( /* units */ "seconds", /* flags */ 0, /* s-text */ - "Timeout before we close unused backend connections.", - /* l-text */ "", - /* func */ NULL + "Timeout before we close unused backend connections." ) PARAM( @@ -306,9 +278,7 @@ PARAM( "may worsen the situation due to the complexity of the operations " "involved in the kernel.\n" "This parameter prevents repeated connection attempts for the " - "configured duration.", - /* l-text */ "", - /* func */ NULL + "configured duration." ) PARAM( @@ -325,9 +295,7 @@ PARAM( "not accepting connections or routing problems for which repeated " "connection attempts are considered useless\n" "This parameter prevents repeated connection attempts for the " - "configured duration.", - /* l-text */ "", - /* func */ NULL + "configured duration." ) PARAM( @@ -341,9 +309,7 @@ PARAM( /* s-text */ "Maximum size of CLI response. If the response exceeds this " "limit, the response code will be 201 instead of 200 and the last " - "line will indicate the truncation.", - /* l-text */ "", - /* func */ NULL + "line will indicate the truncation." ) PARAM( @@ -356,9 +322,7 @@ PARAM( /* flags */ 0, /* s-text */ "Timeout for the childs replies to CLI requests from the " - "mgt_param.", - /* l-text */ "", - /* func */ NULL + "mgt_param." ) PARAM( @@ -371,9 +335,7 @@ PARAM( /* flags */ 0, /* s-text */ "How much clockskew we are willing to accept between the backend " - "and our own clock.", - /* l-text */ "", - /* func */ NULL + "and our own clock." ) PARAM( @@ -386,9 +348,7 @@ PARAM( /* flags */ 0, /* s-text */ "How much observed clock step we are willing to accept before " - "we panic.", - /* l-text */ "", - /* func */ NULL + "we panic." ) PARAM( @@ -403,9 +363,7 @@ PARAM( "Default connection timeout for backend connections. We only try " "to connect to the backend for this many seconds before giving up. " "VCL can override this default value for each backend and backend " - "request.", - /* l-text */ "", - /* func */ NULL + "request." ) PARAM( @@ -418,9 +376,7 @@ PARAM( /* flags */ WIZARD, /* s-text */ "How long the critbit hasher keeps deleted objheads on the cooloff " - "list.", - /* l-text */ "", - /* func */ NULL + "list." ) #if 0 @@ -449,9 +405,7 @@ PARAM( " lurker VSL Ban lurker\n" " esi_chop Chop ESI fetch to bits\n" " flush_head Flush after http1 head\n" - " vtc_mode Varnishtest Mode", - /* l-text */ "", - /* func */ NULL + " vtc_mode Varnishtest Mode" ) #endif @@ -466,9 +420,7 @@ PARAM( /* s-text */ "Default grace period. We will deliver an object this long after " "it has expired, provided another thread is attempting to get a " - "new copy.", - /* l-text */ "", - /* func */ NULL + "new copy." ) PARAM( @@ -483,9 +435,7 @@ PARAM( "Default keep period. We will keep a useless object around this " "long, making it available for conditional backend fetches. That " "means that the object will be removed from the cache at the end " - "of ttl+grace+keep.", - /* l-text */ "", - /* func */ NULL + "of ttl+grace+keep." ) PARAM( @@ -498,9 +448,7 @@ PARAM( /* flags */ OBJ_STICKY, /* s-text */ "The TTL assigned to objects if neither the backend nor the VCL " - "code assigns one.", - /* l-text */ "", - /* func */ NULL + "code assigns one." ) PARAM( @@ -514,9 +462,7 @@ PARAM( /* s-text */ "Number of io vectors to allocate for HTTP1 protocol transmission." " A HTTP1 header needs 7 + 2 per HTTP header field." - " Allocated from workspace_thread.", - /* l-text */ "", - /* func */ NULL + " Allocated from workspace_thread." ) #if 0 @@ -541,9 +487,7 @@ PARAM( " esi_ignore_https Treat HTTPS as HTTP in ESI:includes\n" " esi_disable_xml_check Don't check of body looks like XML\n" " esi_ignore_other_elements Ignore non-esi XML-elements\n" - " esi_remove_bom Remove UTF-8 BOM", - /* l-text */ "", - /* func */ NULL + " esi_remove_bom Remove UTF-8 BOM" ) #endif @@ -559,9 +503,7 @@ PARAM( "The default chunksize used by fetcher. This should be bigger than " "the majority of objects with short TTLs.\n" "Internal limits in the storage_file module makes increases above " - "128kb a dubious idea.", - /* l-text */ "", - /* func */ NULL + "128kb a dubious idea." ) PARAM( @@ -574,9 +516,7 @@ PARAM( /* flags */ EXPERIMENTAL, /* s-text */ "The maximum chunksize we attempt to allocate from storage. Making " - "this too large may cause delays and storage fragmentation.", - /* l-text */ "", - /* func */ NULL + "this too large may cause delays and storage fragmentation." ) PARAM( @@ -592,9 +532,7 @@ PARAM( "These buffers are used for in-transit data, for instance " "gunzip'ed data being sent to a client.Making this space to small " "results in more overhead, writes to sockets etc, making it too " - "big is probably just a waste of memory.", - /* l-text */ "", - /* func */ NULL + "big is probably just a waste of memory." ) PARAM( @@ -606,9 +544,7 @@ PARAM( /* units */ NULL, /* flags */ 0, /* s-text */ - "Gzip compression level: 0=debug, 1=fast, 9=best", - /* l-text */ "", - /* func */ NULL + "Gzip compression level: 0=debug, 1=fast, 9=best" ) PARAM( @@ -621,9 +557,7 @@ PARAM( /* flags */ 0, /* s-text */ "Gzip memory level 1=slow/least, 9=fast/most compression.\n" - "Memory impact is 1=1k, 2=2k, ... 9=256k.", - /* l-text */ "", - /* func */ NULL + "Memory impact is 1=1k, 2=2k, ... 9=256k." ) PARAM( @@ -647,10 +581,8 @@ PARAM( "please see the chapter on gzip in the Varnish reference.\n" "\n" "When gzip support is disabled the variables beresp.do_gzip and " - "beresp.do_gunzip have no effect in VCL.", + "beresp.do_gunzip have no effect in VCL." /* XXX: what about the effect on beresp.filters? */ - /* l-text */ "", - /* func */ NULL ) PARAM( @@ -666,9 +598,7 @@ PARAM( "{req|resp|bereq|beresp}.http (obj.http is autosized to the exact " "number of headers).\n" "Cheap, ~20 bytes, in terms of workspace memory.\n" - "Note that the first line occupies five header lines.", - /* l-text */ "", - /* func */ NULL + "Note that the first line occupies five header lines." ) PARAM( @@ -680,9 +610,7 @@ PARAM( /* units */ "bool", /* flags */ 0, /* s-text */ - "Enable support for HTTP Range headers.", - /* l-text */ "", - /* func */ NULL + "Enable support for HTTP Range headers." ) PARAM( @@ -695,9 +623,7 @@ PARAM( /* flags */ 0, /* s-text */ "Maximum length of any HTTP client request header we will allow. " - "The limit is inclusive its continuation lines.", - /* l-text */ "", - /* func */ NULL + "The limit is inclusive its continuation lines." ) PARAM( @@ -714,9 +640,7 @@ PARAM( "ends the HTTP request.\n" "The memory for the request is allocated from the client workspace " "(param: workspace_client) and this parameter limits how much of " - "that the request is allowed to take up.", - /* l-text */ "", - /* func */ NULL + "that the request is allowed to take up." ) PARAM( @@ -729,9 +653,7 @@ PARAM( /* flags */ 0, /* s-text */ "Maximum length of any HTTP backend response header we will allow. " - " The limit is inclusive its continuation lines.", - /* l-text */ "", - /* func */ NULL + " The limit is inclusive its continuation lines." ) PARAM( @@ -748,9 +670,7 @@ PARAM( "which ends the HTTP response.\n" "The memory for the response is allocated from the backend workspace " "(param: workspace_backend) and this parameter limits how much " - "of that the response is allowed to take up.", - /* l-text */ "", - /* func */ NULL + "of that the response is allowed to take up." ) #if defined(XYZZY) @@ -775,9 +695,7 @@ PARAM( " May get extended if 'send_timeout' applies.\n\n" "When this timeout is hit, the session is closed.\n\n" "See the man page for `setsockopt(2)` or `socket(7)` under" - " ``SO_SNDTIMEO`` for more information.", - /* l-text */ "", - /* func */ NULL + " ``SO_SNDTIMEO`` for more information." ) #undef XYZZY @@ -790,9 +708,7 @@ PARAM( /* units */ "connections", /* flags */ MUST_RESTART, /* s-text */ - "Listen queue depth.", - /* l-text */ "", - /* func */ NULL + "Listen queue depth." ) PARAM( @@ -808,9 +724,7 @@ PARAM( "Objects are only moved to the front of the LRU list if they have " "not been moved there already inside this timeout period. This " "reduces the amount of lock operations necessary for LRU list " - "access.", - /* l-text */ "", - /* func */ NULL + "access." ) PARAM( @@ -822,9 +736,7 @@ PARAM( /* units */ "levels", /* flags */ 0, /* s-text */ - "Maximum depth of esi:include processing.", - /* l-text */ "", - /* func */ NULL + "Maximum depth of esi:include processing." ) PARAM( @@ -836,9 +748,7 @@ PARAM( /* units */ "restarts", /* flags */ 0, /* s-text */ - "Upper limit on how many times a request can restart.", - /* l-text */ "", - /* func */ NULL + "Upper limit on how many times a request can restart." ) PARAM( @@ -850,9 +760,7 @@ PARAM( /* units */ "retries", /* flags */ 0, /* s-text */ - "Upper limit on how many times a backend fetch can retry.", - /* l-text */ "", - /* func */ NULL + "Upper limit on how many times a backend fetch can retry." ) PARAM( @@ -865,9 +773,7 @@ PARAM( /* flags */ EXPERIMENTAL, /* s-text */ "Maximum number of objects we attempt to nuke in order to make " - "space for a object body.", - /* l-text */ "", - /* func */ NULL + "space for a object body." ) #if 0 @@ -882,9 +788,7 @@ PARAM( /* flags */ 0, /* s-text */ "The limit for the number of internal matching function calls in " - "a pcre_exec() execution.", - /* l-text */ "", - /* func */ NULL + "a pcre_exec() execution." ) /* actual location mgt_param_tbl.c */ @@ -898,9 +802,7 @@ PARAM( /* flags */ 0, /* s-text */ "The limit for the number of internal matching function " - "recursions in a pcre_exec() execution.", - /* l-text */ "", - /* func */ NULL + "recursions in a pcre_exec() execution." ) #endif @@ -915,9 +817,7 @@ PARAM( /* s-text */ "Interval between pings from parent to child.\n" "Zero will disable pinging entirely, which makes it possible to " - "attach a debugger to the child.", - /* l-text */ "", - /* func */ NULL + "attach a debugger to the child." ) PARAM( @@ -929,9 +829,7 @@ PARAM( /* units */ "connections", /* flags */ 0, /* s-text */ - "Maximum number of sessions dedicated to pipe transactions.", - /* l-text */ "", - /* func */ NULL + "Maximum number of sessions dedicated to pipe transactions." ) PARAM( @@ -944,9 +842,7 @@ PARAM( /* flags */ 0, /* s-text */ "Idle timeout for PIPE sessions. If nothing have been received in " - "either direction for this many seconds, the session is closed.", - /* l-text */ "", - /* func */ NULL + "either direction for this many seconds, the session is closed." ) #if 0 @@ -961,9 +857,7 @@ PARAM( /* flags */ 0, /* s-text */ "Parameters for per worker pool request memory pool.\n" - MEMPOOL_TEXT, - /* l-text */ "", - /* func */ NULL + MEMPOOL_TEXT ) /* actual location mgt_param_tbl.c */ @@ -977,9 +871,7 @@ PARAM( /* flags */ 0, /* s-text */ "Parameters for per worker pool session memory pool.\n" - MEMPOOL_TEXT, - /* l-text */ "", - /* func */ NULL + MEMPOOL_TEXT ) /* actual location mgt_param_tbl.c */ @@ -993,9 +885,7 @@ PARAM( /* flags */ 0, /* s-text */ "Parameters for backend object fetch memory pool.\n" - MEMPOOL_TEXT, - /* l-text */ "", - /* func */ NULL + MEMPOOL_TEXT ) #endif @@ -1009,9 +899,7 @@ PARAM( /* flags */ 0, /* s-text */ "Prefer IPv6 address when connecting to backends which have both " - "IPv4 and IPv6 addresses.", - /* l-text */ "", - /* func */ NULL + "IPv4 and IPv6 addresses." ) PARAM( @@ -1026,9 +914,7 @@ PARAM( "How many parked request we start for each completed request on " "the object.\n" "NB: Even with the implict delay of delivery, this parameter " - "controls an exponential increase in number of worker threads.", - /* l-text */ "", - /* func */ NULL + "controls an exponential increase in number of worker threads." ) #if defined(XYZZY) @@ -1054,9 +940,7 @@ PARAM( "When 'idle_send_timeout' is hit while sending an HTTP1 response, the" " timeout is extended unless the total time already taken for sending" " the response in its entirety exceeds this many seconds.\n\n" - "When this timeout is hit, the session is closed", - /* l-text */ "", - /* func */ NULL + "When this timeout is hit, the session is closed" ) #undef XYZZY @@ -1070,9 +954,7 @@ PARAM( /* flags */ 0, /* s-text */ "Objects created with (ttl+grace+keep) shorter than this are " - "always put in transient storage.", - /* l-text */ "", - /* func */ NULL + "always put in transient storage." ) PARAM( @@ -1085,9 +967,7 @@ PARAM( /* flags */ MUST_RESTART, /* s-text */ "Install a signal handler which tries to dump debug information on " - "segmentation faults, bus errors and abort signals.", - /* l-text */ "", - /* func */ NULL + "segmentation faults, bus errors and abort signals." ) PARAM( @@ -1099,9 +979,7 @@ PARAM( /* units */ "bool", /* flags */ 0, /* s-text */ - "Log all CLI traffic to syslog(LOG_INFO).", - /* l-text */ "", - /* func */ NULL + "Log all CLI traffic to syslog(LOG_INFO)." ) #if defined(HAVE_TCP_FASTOPEN) @@ -1118,9 +996,7 @@ PARAM( /* units */ "bool", /* flags */ XYZZY, /* s-text */ - "Enable TCP Fast Open extension.", - /* l-text */ NULL, - /* func */ NULL + "Enable TCP Fast Open extension." ) #undef XYZZY @@ -1139,9 +1015,7 @@ PARAM( /* flags */ XYZZY, /* s-text */ "The number of seconds between TCP keep-alive probes. " - "Ignored for Unix domain sockets.", - /* l-text */ "", - /* func */ NULL + "Ignored for Unix domain sockets." ) PARAM( @@ -1155,9 +1029,7 @@ PARAM( /* s-text */ "The maximum number of TCP keep-alive probes to send before giving " "up and killing the connection if no response is obtained from the " - "other end. Ignored for Unix domain sockets.", - /* l-text */ "", - /* func */ NULL + "other end. Ignored for Unix domain sockets." ) PARAM( @@ -1171,9 +1043,7 @@ PARAM( /* s-text */ "The number of seconds a connection needs to be idle before TCP " "begins sending out keep-alive probes. " - "Ignored for Unix domain sockets.", - /* l-text */ "", - /* func */ NULL + "Ignored for Unix domain sockets." ) #undef XYZZY @@ -1194,9 +1064,7 @@ PARAM( "creating threads.\n" "Set this to a few milliseconds if you see the 'threads_failed' " "counter grow too much.\n" - "Setting this too high results in insufficient worker threads.", - /* l-text */ "", - /* func */ NULL + "Setting this too high results in insufficient worker threads." ) /* actual location mgt_pool.c */ @@ -1212,9 +1080,7 @@ PARAM( "Thread queue stuck watchdog.\n" "\n" "If no queued work have been released for this long," - " the worker process panics itself.", - /* l-text */ "", - /* func */ NULL + " the worker process panics itself." ) /* actual location mgt_pool.c */ @@ -1228,9 +1094,7 @@ PARAM( /* flags */ DELAYED_EFFECT| EXPERIMENTAL, /* s-text */ "Wait this long after destroying a thread.\n" - "This controls the decay of thread pools when idle(-ish).", - /* l-text */ "", - /* func */ NULL + "This controls the decay of thread pools when idle(-ish)." ) /* actual location mgt_pool.c */ @@ -1255,9 +1119,7 @@ PARAM( "\n" "It may also help to increase thread_pool_timeout and " "thread_pool_min, to reduce the rate at which treads are destroyed " - "and later recreated.", - /* l-text */ "", - /* func */ NULL + "and later recreated." ) /* actual location mgt_pool.c */ @@ -1274,9 +1136,7 @@ PARAM( "\n" "Do not set this higher than you have to, since excess worker " "threads soak up RAM and CPU and generally just get in the way of " - "getting work done.", - /* l-text */ "", - /* func */ NULL + "getting work done." ) /* actual location mgt_pool.c */ @@ -1293,9 +1153,7 @@ PARAM( "\n" "Increasing this may help ramp up faster from low load situations " "or when threads have expired." - "Minimum is 10 threads.", - /* l-text */ "", - /* func */ NULL + "Minimum is 10 threads." ) /* actual location mgt_pool.c */ @@ -1322,9 +1180,7 @@ PARAM( "unused.\n" "\n" "Default is 0 to auto-tune (currently 5% of thread_pool_min).\n" - "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", - /* l-text */ "", - /* func */ NULL + "Minimum is 1 otherwise, maximum is 95% of thread_pool_min." ) /* actual location mgt_pool.c */ @@ -1339,9 +1195,7 @@ PARAM( /* s-text */ "Worker thread stack size.\n" "This will likely be rounded up to a multiple of 4k (or whatever " - "the page_size might be) by the kernel.", - /* l-text */ "", - /* func */ NULL + "the page_size might be) by the kernel." ) /* actual location mgt_pool.c */ @@ -1357,9 +1211,7 @@ PARAM( "Thread idle threshold.\n" "\n" "Threads in excess of thread_pool_min, which have been idle for at " - "least this long, will be destroyed.", - /* l-text */ "", - /* func */ NULL + "least this long, will be destroyed." ) /* actual location mgt_pool.c */ @@ -1384,9 +1236,7 @@ PARAM( "pool for each CPU is most likely detrimental to performance.\n" "\n" "Can be increased on the fly, but decreases require a restart to " - "take effect.", - /* l-text */ "", - /* func */ NULL + "take effect." ) /* actual location mgt_pool.c */ @@ -1403,9 +1253,7 @@ PARAM( "\n" "This sets the number of requests we will queue, waiting for an " "available thread. Above this limit sessions will be dropped " - "instead of queued.", - /* l-text */ "", - /* func */ NULL + "instead of queued." ) /* actual location mgt_pool.c */ @@ -1423,9 +1271,7 @@ PARAM( "(request/fetch etc).\n" "This parameters defines the maximum number of jobs a worker " "thread may handle, before it is forced to dump its accumulated " - "stats into the global counters.", - /* l-text */ "", - /* func */ NULL + "stats into the global counters." ) #endif @@ -1452,9 +1298,7 @@ PARAM( " request headers.\n\n" "This parameter is particularly relevant for HTTP1 keepalive " " connections which are closed unless the next request is received" - " before this timeout is reached.", - /* l-text */ "", - /* func */ NULL + " before this timeout is reached." ) #undef XYZZY @@ -1473,9 +1317,7 @@ PARAM( "within the first 100 msec of the previous request completing.\n" "Setting this too high results in worker threads not doing " "anything for their keep, setting it too low just means that more " - "sessions take a detour around the waiter.", - /* l-text */ "", - /* func */ NULL + "sessions take a detour around the waiter." ) #if 0 @@ -1489,9 +1331,7 @@ PARAM( /* units */ "bool", /* flags */ 0, /* s-text */ - "Allow inline C code in VCL.", - /* l-text */ "", - /* func */ NULL + "Allow inline C code in VCL." ) /* actual location mgt_param_tbl.c */ @@ -1504,9 +1344,7 @@ PARAM( /* units */ "bool", /* flags */ 0, /* s-text */ - "Unreferenced VCL objects result in error.", - /* l-text */ "", - /* func */ NULL + "Unreferenced VCL objects result in error." ) /* actual location mgt_param_tbl.c */ @@ -1520,9 +1358,7 @@ PARAM( /* flags */ 0, /* s-text */ "Allow '/' in vmod & include paths.\n" - "Allow 'import ... from ...'.", - /* l-text */ "", - /* func */ NULL + "Allow 'import ... from ...'." ) #endif @@ -1536,9 +1372,7 @@ PARAM( /* flags */ 0, /* s-text */ "How long a VCL is kept warm after being replaced as the " - "active VCL (granularity approximately 30 seconds).", - /* l-text */ "", - /* func */ NULL + "active VCL (granularity approximately 30 seconds)." ) PARAM( @@ -1553,9 +1387,7 @@ PARAM( "Behaviour when attempting to exceed max_vcl loaded VCL.\n" "\n* 0 - Ignore max_vcl parameter.\n" "\n* 1 - Issue warning.\n" - "\n* 2 - Refuse loading VCLs.", - /* l-text */ "", - /* func */ NULL + "\n* 2 - Refuse loading VCLs." ) PARAM( @@ -1568,9 +1400,7 @@ PARAM( /* flags */ 0, /* s-text */ "Threshold of loaded VCL programs. (VCL labels are not counted.)" - " Parameter max_vcl_handling determines behaviour.", - /* l-text */ "", - /* func */ NULL + " Parameter max_vcl_handling determines behaviour." ) PARAM( @@ -1583,9 +1413,7 @@ PARAM( /* flags */ 0, /* s-text */ "How long VSM memory is kept warm after a deallocation " - "(granularity approximately 2 seconds).", - /* l-text */ "", - /* func */ NULL + "(granularity approximately 2 seconds)." ) PARAM( @@ -1604,9 +1432,7 @@ PARAM( "Setting this too high costs memory, setting it too low will cause " "more VSL flushes and likely increase lock-contention on the VSL " "mutex.\n\n" - "The minimum tracks the vsl_reclen parameter + 12 bytes.", - /* l-text */ "", - /* func */ NULL + "The minimum tracks the vsl_reclen parameter + 12 bytes." ) #if 0 @@ -1624,9 +1450,7 @@ PARAM( " default Set default value\n" "\n" "Use +/- prefix in front of VSL tag name to unmask/mask " - "individual VSL messages.", - /* l-text */ "", - /* func */ NULL + "individual VSL messages." ) #endif @@ -1640,9 +1464,7 @@ PARAM( /* flags */ 0, /* s-text */ "Maximum number of bytes in SHM log record.\n\n" - "The maximum tracks the vsl_buffer parameter - 12 bytes.", - /* l-text */ "", - /* func */ NULL + "The maximum tracks the vsl_buffer parameter - 12 bytes." ) PARAM( @@ -1657,9 +1479,7 @@ PARAM( "The amount of space to allocate for the VSL fifo buffer in the " "VSM memory segment. If you make this too small, " "varnish{ncsa|log} etc will not be able to keep up. Making it too " - "large just costs memory resources.", - /* l-text */ "", - /* func */ NULL + "large just costs memory resources." ) PARAM( @@ -1672,9 +1492,7 @@ PARAM( /* flags */ 0, /* s-text */ "DEPRECATED: This parameter is ignored.\n" - "There is no global limit on amount of shared memory now.", - /* l-text */ "", - /* func */ NULL + "There is no global limit on amount of shared memory now." ) #if 0 @@ -1688,9 +1506,7 @@ PARAM( /* units */ NULL, /* flags */ MUST_RESTART| WIZARD, /* s-text */ - "Select the waiter kernel interface.", - /* l-text */ "", - /* func */ NULL + "Select the waiter kernel interface." ) #endif @@ -1704,9 +1520,7 @@ PARAM( /* flags */ DELAYED_EFFECT, /* s-text */ "Bytes of HTTP protocol workspace for backend HTTP req/resp. If " - "larger than 4k, use a multiple of 4k for VM efficiency.", - /* l-text */ "", - /* func */ NULL + "larger than 4k, use a multiple of 4k for VM efficiency." ) PARAM( @@ -1723,9 +1537,7 @@ PARAM( "For HTTP/2 compliance this must be at least 20k, in order to " "receive fullsize (=16k) frames from the client. That usually " "happens only in POST/PUT bodies. For other traffic-patterns " - "smaller values work just fine.", - /* l-text */ "", - /* func */ NULL + "smaller values work just fine." ) PARAM( @@ -1739,9 +1551,7 @@ PARAM( /* s-text */ "Allocation size for session structure and workspace. The " "workspace is primarily used for TCP connection addresses. If " - "larger than 4k, use a multiple of 4k for VM efficiency.", - /* l-text */ "", - /* func */ NULL + "larger than 4k, use a multiple of 4k for VM efficiency." ) PARAM( @@ -1761,9 +1571,7 @@ PARAM( "syscalls, setting it too high just wastes space. ~0.1k + " "UIO_MAXIOV * sizeof(struct iovec) (typically = ~16k for 64bit) " "is considered the maximum sensible value under any known " - "circumstances (excluding exotic vmod use).", - /* l-text */ "", - /* func */ NULL + "circumstances (excluding exotic vmod use)." ) PARAM( @@ -1777,9 +1585,7 @@ PARAM( /* s-text */ "HTTP2 Receive Window low water mark.\n" "We try to keep the window at least this big\n" - "Only affects incoming request bodies (ie: POST, PUT etc.)", - /* l-text */ "", - /* func */ NULL + "Only affects incoming request bodies (ie: POST, PUT etc.)" ) PARAM( @@ -1793,9 +1599,7 @@ PARAM( /* s-text */ "HTTP2 Receive Window Increments.\n" "How big credits we send in WINDOW_UPDATE frames\n" - "Only affects incoming request bodies (ie: POST, PUT etc.)", - /* l-text */ "", - /* func */ NULL + "Only affects incoming request bodies (ie: POST, PUT etc.)" ) PARAM( @@ -1809,9 +1613,7 @@ PARAM( /* s-text */ "HTTP2 header table size.\n" "This is the size that will be used for the HPACK dynamic\n" - "decoding table.", - /* l-text */ "", - /* func */ NULL + "decoding table." ) PARAM( @@ -1825,9 +1627,7 @@ PARAM( /* s-text */ "HTTP2 Maximum number of concurrent streams.\n" "This is the number of requests that can be active\n" - "at the same time for a single HTTP2 connection.", - /* l-text */ "", - /* func */ NULL + "at the same time for a single HTTP2 connection." ) PARAM( @@ -1839,9 +1639,7 @@ PARAM( /* units */ "bytes", /* flags */ 0, /* s-text */ - "HTTP2 initial flow control window size.", - /* l-text */ "", - /* func */ NULL + "HTTP2 initial flow control window size." ) PARAM( @@ -1853,9 +1651,7 @@ PARAM( /* units */ "bytes", /* flags */ 0, /* s-text */ - "HTTP2 maximum per frame payload size we are willing to accept.", - /* l-text */ "", - /* func */ NULL + "HTTP2 maximum per frame payload size we are willing to accept." ) PARAM( @@ -1867,9 +1663,7 @@ PARAM( /* units */ "bytes", /* flags */ 0, /* s-text */ - "HTTP2 maximum size of an uncompressed header list.", - /* l-text */ "", - /* func */ NULL + "HTTP2 maximum size of an uncompressed header list." ) #undef PARAM From dridi.boukelmoune at gmail.com Mon Mar 2 14:09:08 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 14:09:08 +0000 (UTC) Subject: [master] 134212c28 Turn the PARAM() macro into a vararg Message-ID: <20200302140908.937B3ABBE9@lists.varnish-cache.org> commit 134212c28475c6c3ac36e411fef3efcbc6feae25 Author: Dridi Boukelmoune Date: Mon Mar 2 11:27:54 2020 +0100 Turn the PARAM() macro into a vararg The two places where the parameters table is used have very distinct needs, and even among parameters themselves we will need some degree variation. diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h index 0fdeaaefd..6d1630c2e 100644 --- a/bin/varnishd/common/common_param.h +++ b/bin/varnishd/common/common_param.h @@ -82,7 +82,8 @@ struct params { #define ptyp_uint unsigned #define ptyp_vsl_buffer unsigned #define ptyp_vsl_reclen unsigned -#define PARAM(nm, ty, mi, ma, de, un, fl, st) ptyp_##ty nm; +#define PARAM(nm, ty, ...) \ + ptyp_##ty nm; #include #undef ptyp_bool #undef ptyp_bytes diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 4f417cd76..09bff349a 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -45,8 +45,9 @@ "\tmax_age\tmax age of free element." struct parspec mgt_parspec[] = { -#define PARAM(nm, ty, mi, ma, de, un, fl, st) \ - { #nm, tweak_##ty, &mgt_param.nm, mi, ma, st, fl, de, un }, +#define PARAM(nm, ty, mi, ma, de, un, fl, st, ...) \ + { #nm, tweak_##ty, &mgt_param.nm, mi, ma, st, fl, de, un, \ + __VA_ARGS__ }, #include "tbl/params.h" { "cc_command", tweak_string, &mgt_cc_cmd, diff --git a/include/tbl/params.h b/include/tbl/params.h index db734ffcb..63492f9f3 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -27,7 +27,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * PARAM(nm, ty, mi, ma, de, un, fl, st) + * PARAM(nm, ty, mi, ma, de, un, fl, st[, dyn_min_reason, dyn_max_reason]) */ /*lint -save -e525 -e539 */ From dridi.boukelmoune at gmail.com Mon Mar 2 14:09:08 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 14:09:08 +0000 (UTC) Subject: [master] 7b1188543 Document vsl_buffer and vsl_reclen dependencies Message-ID: <20200302140908.BC190ABBED@lists.varnish-cache.org> commit 7b1188543730a1e654241f21c71fbee79e8c7897 Author: Dridi Boukelmoune Date: Mon Mar 2 11:31:05 2020 +0100 Document vsl_buffer and vsl_reclen dependencies Now that PARAM() is a vararg macro, we can afford having the dynamic bounds reasons only for relevant parameters without bloating the rest of the table. diff --git a/include/tbl/params.h b/include/tbl/params.h index 63492f9f3..4acf14910 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1431,8 +1431,8 @@ PARAM( "and workspace_backend will have to be adjusted by the same amount.\n\n" "Setting this too high costs memory, setting it too low will cause " "more VSL flushes and likely increase lock-contention on the VSL " - "mutex.\n\n" - "The minimum tracks the vsl_reclen parameter + 12 bytes." + "mutex.", + /* dyn_min_reason */ "vsl_reclen + 12 bytes" ) #if 0 @@ -1463,8 +1463,9 @@ PARAM( /* units */ "bytes", /* flags */ 0, /* s-text */ - "Maximum number of bytes in SHM log record.\n\n" - "The maximum tracks the vsl_buffer parameter - 12 bytes." + "Maximum number of bytes in SHM log record.", + /* dyn_min_reason */ NULL, + /* dyn_max_reason */ "vsl_buffer - 12 bytes" ) PARAM( From dridi.boukelmoune at gmail.com Mon Mar 2 14:09:08 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 14:09:08 +0000 (UTC) Subject: [master] f3bbf0424 Use struct parspec field names in params.h Message-ID: <20200302140908.E5D36ABBF1@lists.varnish-cache.org> commit f3bbf04241b87a2b8543f553a3ee23665ef31889 Author: Dridi Boukelmoune Date: Mon Mar 2 11:33:00 2020 +0100 Use struct parspec field names in params.h I didn't change their shorthand counterparts (for example st for s-text) because they will go away soon enough. diff --git a/include/tbl/params.h b/include/tbl/params.h index 4acf14910..41c0c4d7e 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -43,13 +43,13 @@ #endif PARAM( /* name */ accept_filter, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "on", + /* def */ "on", /* units */ "bool", /* flags */ XYZZY, - /* s-text */ + /* descr */ "Enable kernel accept-filters. This may require a kernel module to " "be loaded to have an effect when enabled.\n\n" "Enabling accept_filter may prevent some requests to reach Varnish " @@ -61,13 +61,13 @@ PARAM( PARAM( /* name */ acceptor_sleep_decay, - /* typ */ double, + /* type */ double, /* min */ "0", /* max */ "1", - /* default */ "0.9", + /* def */ "0.9", /* units */ NULL, /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "If we run out of resources, such as file descriptors or worker " "threads, the acceptor will sleep between accepts.\n" "This parameter (multiplicatively) reduce the sleep duration for " @@ -76,13 +76,13 @@ PARAM( PARAM( /* name */ acceptor_sleep_incr, - /* typ */ timeout, + /* type */ timeout, /* min */ "0", /* max */ "1", - /* default */ "0", + /* def */ "0", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "If we run out of resources, such as file descriptors or worker " "threads, the acceptor will sleep between accepts.\n" "This parameter control how much longer we sleep, each time we " @@ -91,13 +91,13 @@ PARAM( PARAM( /* name */ acceptor_sleep_max, - /* typ */ timeout, + /* type */ timeout, /* min */ "0", /* max */ "10", - /* default */ "0.05", + /* def */ "0.05", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "If we run out of resources, such as file descriptors or worker " "threads, the acceptor will sleep between accepts.\n" "This parameter limits how long it can sleep between attempts to " @@ -106,25 +106,25 @@ PARAM( PARAM( /* name */ auto_restart, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "on", + /* def */ "on", /* units */ "bool", /* flags */ 0, - /* s-text */ + /* descr */ "Automatically restart the child/worker process if it dies." ) PARAM( /* name */ ban_dups, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "on", + /* def */ "on", /* units */ "bool", /* flags */ 0, - /* s-text */ + /* descr */ "Eliminate older identical bans when a new ban is added. This saves " "CPU cycles by not comparing objects to identical bans.\n" "This is a waste of time if you have many bans which are never " @@ -133,13 +133,13 @@ PARAM( PARAM( /* name */ ban_cutoff, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "0", + /* def */ "0", /* units */ "bans", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Expurge long tail content from the cache to keep the number of bans " "below this value. 0 disables.\n\n" "When this parameter is set to a non-zero value, the ban lurker " @@ -163,13 +163,13 @@ PARAM( PARAM( /* name */ ban_lurker_age, - /* typ */ timeout, + /* type */ timeout, /* min */ "0", /* max */ NULL, - /* default */ "60", + /* def */ "60", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "The ban lurker will ignore bans until they are this old. " "When a ban is added, the active traffic will be tested against it " "as part of object lookup. Because many applications issue bans in " @@ -180,13 +180,13 @@ PARAM( PARAM( /* name */ ban_lurker_batch, - /* typ */ uint, + /* type */ uint, /* min */ "1", /* max */ NULL, - /* default */ "1000", + /* def */ "1000", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "The ban lurker sleeps ${ban_lurker_sleep} after examining this " "many objects." " Use this to pace the ban-lurker if it eats too many resources." @@ -194,13 +194,13 @@ PARAM( PARAM( /* name */ ban_lurker_sleep, - /* typ */ timeout, + /* type */ timeout, /* min */ "0", /* max */ NULL, - /* default */ "0.010", + /* def */ "0.010", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "How long the ban lurker sleeps after examining ${ban_lurker_batch} " "objects." " Use this to pace the ban-lurker if it eats too many resources.\n" @@ -209,26 +209,26 @@ PARAM( PARAM( /* name */ ban_lurker_holdoff, - /* typ */ timeout, + /* type */ timeout, /* min */ "0", /* max */ NULL, - /* default */ "0.010", + /* def */ "0.010", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "How long the ban lurker sleeps when giving way to lookup" " due to lock contention." ) PARAM( /* name */ first_byte_timeout, - /* typ */ timeout, + /* type */ timeout, /* min */ "0", /* max */ NULL, - /* default */ "60", + /* def */ "60", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "Default timeout for receiving first byte from backend. We only " "wait for this many seconds for the first byte before giving up.\n" "VCL can override this default value for each backend and backend " @@ -238,13 +238,13 @@ PARAM( PARAM( /* name */ between_bytes_timeout, - /* typ */ timeout, + /* type */ timeout, /* min */ "0", /* max */ NULL, - /* default */ "60", + /* def */ "60", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "We only wait for this many seconds between bytes received from " "the backend before giving up the fetch.\n" "VCL values, per backend or per backend request take precedence.\n" @@ -253,25 +253,25 @@ PARAM( PARAM( /* name */ backend_idle_timeout, - /* typ */ timeout, + /* type */ timeout, /* min */ "1", /* max */ NULL, - /* default */ "60", + /* def */ "60", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "Timeout before we close unused backend connections." ) PARAM( /* name */ backend_local_error_holddown, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "10.000", + /* def */ "10.000", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "When connecting to backends, certain error codes " "(EADDRNOTAVAIL, EACCESS, EPERM) signal a local resource shortage " "or configuration issue for which retrying connection attempts " @@ -283,13 +283,13 @@ PARAM( PARAM( /* name */ backend_remote_error_holddown, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "0.250", + /* def */ "0.250", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "When connecting to backends, certain error codes (ECONNREFUSED, " "ENETUNREACH) signal fundamental connection issues such as the backend " "not accepting connections or routing problems for which repeated " @@ -300,13 +300,13 @@ PARAM( PARAM( /* name */ cli_limit, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "128b", /* max */ "99999999b", - /* default */ "48k", + /* def */ "48k", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "Maximum size of CLI response. If the response exceeds this " "limit, the response code will be 201 instead of 200 and the last " "line will indicate the truncation." @@ -314,52 +314,52 @@ PARAM( PARAM( /* name */ cli_timeout, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "60.000", + /* def */ "60.000", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "Timeout for the childs replies to CLI requests from the " "mgt_param." ) PARAM( /* name */ clock_skew, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "10", + /* def */ "10", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "How much clockskew we are willing to accept between the backend " "and our own clock." ) PARAM( /* name */ clock_step, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "1.000", + /* def */ "1.000", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "How much observed clock step we are willing to accept before " "we panic." ) PARAM( /* name */ connect_timeout, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "3.500", + /* def */ "3.500", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "Default connection timeout for backend connections. We only try " "to connect to the backend for this many seconds before giving up. " "VCL can override this default value for each backend and backend " @@ -368,13 +368,13 @@ PARAM( PARAM( /* name */ critbit_cooloff, - /* typ */ timeout, + /* type */ timeout, /* min */ "60.000", /* max */ "254.000", - /* default */ "180.000", + /* def */ "180.000", /* units */ "seconds", /* flags */ WIZARD, - /* s-text */ + /* descr */ "How long the critbit hasher keeps deleted objheads on the cooloff " "list." ) @@ -384,13 +384,13 @@ PARAM( /* see tbl/debug_bits.h */ PARAM( /* name */ debug, - /* typ */ debug, + /* type */ debug, /* min */ NULL, /* max */ NULL, - /* default */ NULL, + /* def */ NULL, /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Enable/Disable various kinds of debugging.\n" " none Disable all debugging\n" "\n" @@ -411,13 +411,13 @@ PARAM( PARAM( /* name */ default_grace, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "10.000", + /* def */ "10.000", /* units */ "seconds", /* flags */ OBJ_STICKY, - /* s-text */ + /* descr */ "Default grace period. We will deliver an object this long after " "it has expired, provided another thread is attempting to get a " "new copy." @@ -425,13 +425,13 @@ PARAM( PARAM( /* name */ default_keep, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "0.000", + /* def */ "0.000", /* units */ "seconds", /* flags */ OBJ_STICKY, - /* s-text */ + /* descr */ "Default keep period. We will keep a useless object around this " "long, making it available for conditional backend fetches. That " "means that the object will be removed from the cache at the end " @@ -440,26 +440,26 @@ PARAM( PARAM( /* name */ default_ttl, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "120.000", + /* def */ "120.000", /* units */ "seconds", /* flags */ OBJ_STICKY, - /* s-text */ + /* descr */ "The TTL assigned to objects if neither the backend nor the VCL " "code assigns one." ) PARAM( /* name */ http1_iovs, - /* typ */ uint, + /* type */ uint, /* min */ "5", /* max */ "1024", // XXX stringify IOV_MAX - /* default */ "64", + /* def */ "64", /* units */ "struct iovec (=16 bytes)", /* flags */ WIZARD, - /* s-text */ + /* descr */ "Number of io vectors to allocate for HTTP1 protocol transmission." " A HTTP1 header needs 7 + 2 per HTTP header field." " Allocated from workspace_thread." @@ -470,13 +470,13 @@ PARAM( /* See tbl/feature_bits.h */ PARAM( /* name */ feature, - /* typ */ feature, + /* type */ feature, /* min */ NULL, /* max */ NULL, - /* default */ NULL, + /* def */ NULL, /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Enable/Disable various minor features.\n" " none Disable all features.\n" "\n" @@ -493,13 +493,13 @@ PARAM( PARAM( /* name */ fetch_chunksize, - /* typ */ bytes, + /* type */ bytes, /* min */ "4k", /* max */ NULL, - /* default */ "16k", + /* def */ "16k", /* units */ "bytes", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "The default chunksize used by fetcher. This should be bigger than " "the majority of objects with short TTLs.\n" "Internal limits in the storage_file module makes increases above " @@ -508,26 +508,26 @@ PARAM( PARAM( /* name */ fetch_maxchunksize, - /* typ */ bytes, + /* type */ bytes, /* min */ "64k", /* max */ NULL, - /* default */ "0.25G", + /* def */ "0.25G", /* units */ "bytes", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "The maximum chunksize we attempt to allocate from storage. Making " "this too large may cause delays and storage fragmentation." ) PARAM( /* name */ gzip_buffer, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "2k", /* max */ NULL, - /* default */ "32k", + /* def */ "32k", /* units */ "bytes", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Size of malloc buffer used for gzip processing.\n" "These buffers are used for in-transit data, for instance " "gunzip'ed data being sent to a client.Making this space to small " @@ -537,38 +537,38 @@ PARAM( PARAM( /* name */ gzip_level, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ "9", - /* default */ "6", + /* def */ "6", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Gzip compression level: 0=debug, 1=fast, 9=best" ) PARAM( /* name */ gzip_memlevel, - /* typ */ uint, + /* type */ uint, /* min */ "1", /* max */ "9", - /* default */ "8", + /* def */ "8", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Gzip memory level 1=slow/least, 9=fast/most compression.\n" "Memory impact is 1=1k, 2=2k, ... 9=256k." ) PARAM( /* name */ http_gzip_support, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "on", + /* def */ "on", /* units */ "bool", /* flags */ 0, - /* s-text */ + /* descr */ "Enable gzip support. When enabled Varnish request compressed " "objects from the backend and store them compressed. If a client " "does not support gzip encoding Varnish will uncompress compressed " @@ -587,13 +587,13 @@ PARAM( PARAM( /* name */ http_max_hdr, - /* typ */ uint, + /* type */ uint, /* min */ "32", /* max */ "65535", - /* default */ "64", + /* def */ "64", /* units */ "header lines", /* flags */ 0, - /* s-text */ + /* descr */ "Maximum number of HTTP header lines we allow in " "{req|resp|bereq|beresp}.http (obj.http is autosized to the exact " "number of headers).\n" @@ -603,38 +603,38 @@ PARAM( PARAM( /* name */ http_range_support, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "on", + /* def */ "on", /* units */ "bool", /* flags */ 0, - /* s-text */ + /* descr */ "Enable support for HTTP Range headers." ) PARAM( /* name */ http_req_hdr_len, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "40b", /* max */ NULL, - /* default */ "8k", + /* def */ "8k", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "Maximum length of any HTTP client request header we will allow. " "The limit is inclusive its continuation lines." ) PARAM( /* name */ http_req_size, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "0.25k", /* max */ NULL, - /* default */ "32k", + /* def */ "32k", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "Maximum number of bytes of HTTP client request we will deal with. " " This is a limit on all bytes up to the double blank line which " "ends the HTTP request.\n" @@ -645,26 +645,26 @@ PARAM( PARAM( /* name */ http_resp_hdr_len, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "40b", /* max */ NULL, - /* default */ "8k", + /* def */ "8k", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "Maximum length of any HTTP backend response header we will allow. " " The limit is inclusive its continuation lines." ) PARAM( /* name */ http_resp_size, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "0.25k", /* max */ NULL, - /* default */ "32k", + /* def */ "32k", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "Maximum number of bytes of HTTP backend response we will deal " "with. This is a limit on all bytes up to the double blank line " "which ends the HTTP response.\n" @@ -684,13 +684,13 @@ PARAM( #endif PARAM( /* name */ idle_send_timeout, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "60.000", + /* def */ "60.000", /* units */ "seconds", /* flags */ XYZZY, - /* s-text */ + /* descr */ "Send timeout for individual pieces of data on client connections." " May get extended if 'send_timeout' applies.\n\n" "When this timeout is hit, the session is closed.\n\n" @@ -701,25 +701,25 @@ PARAM( PARAM( /* name */ listen_depth, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "1024", + /* def */ "1024", /* units */ "connections", /* flags */ MUST_RESTART, - /* s-text */ + /* descr */ "Listen queue depth." ) PARAM( /* name */ lru_interval, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "2.000", + /* def */ "2.000", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Grace period before object moves on LRU list.\n" "Objects are only moved to the front of the LRU list if they have " "not been moved there already inside this timeout period. This " @@ -729,49 +729,49 @@ PARAM( PARAM( /* name */ max_esi_depth, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "5", + /* def */ "5", /* units */ "levels", /* flags */ 0, - /* s-text */ + /* descr */ "Maximum depth of esi:include processing." ) PARAM( /* name */ max_restarts, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "4", + /* def */ "4", /* units */ "restarts", /* flags */ 0, - /* s-text */ + /* descr */ "Upper limit on how many times a request can restart." ) PARAM( /* name */ max_retries, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "4", + /* def */ "4", /* units */ "retries", /* flags */ 0, - /* s-text */ + /* descr */ "Upper limit on how many times a backend fetch can retry." ) PARAM( /* name */ nuke_limit, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "50", + /* def */ "50", /* units */ "allocations", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Maximum number of objects we attempt to nuke in order to make " "space for a object body." ) @@ -780,13 +780,13 @@ PARAM( /* actual location mgt_param_tbl.c */ PARAM( /* name */ pcre_match_limit, - /* typ */ uint, + /* type */ uint, /* min */ "1", /* max */ NULL, - /* default */ "1.000", + /* def */ "1.000", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "The limit for the number of internal matching function calls in " "a pcre_exec() execution." ) @@ -794,13 +794,13 @@ PARAM( /* actual location mgt_param_tbl.c */ PARAM( /* name */ pcre_match_limit_recursion, - /* typ */ uint, + /* type */ uint, /* min */ "1", /* max */ NULL, - /* default */ "1.000", + /* def */ "1.000", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "The limit for the number of internal matching function " "recursions in a pcre_exec() execution." ) @@ -808,13 +808,13 @@ PARAM( PARAM( /* name */ ping_interval, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "3", + /* def */ "3", /* units */ "seconds", /* flags */ MUST_RESTART, - /* s-text */ + /* descr */ "Interval between pings from parent to child.\n" "Zero will disable pinging entirely, which makes it possible to " "attach a debugger to the child." @@ -822,25 +822,25 @@ PARAM( PARAM( /* name */ pipe_sess_max, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "0", + /* def */ "0", /* units */ "connections", /* flags */ 0, - /* s-text */ + /* descr */ "Maximum number of sessions dedicated to pipe transactions." ) PARAM( /* name */ pipe_timeout, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "60.000", + /* def */ "60.000", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "Idle timeout for PIPE sessions. If nothing have been received in " "either direction for this many seconds, the session is closed." ) @@ -849,13 +849,13 @@ PARAM( /* actual location mgt_param_tbl.c */ PARAM( /* name */ pool_req, - /* typ */ poolparam, + /* type */ poolparam, /* min */ NULL, /* max */ NULL, - /* default */ "10,100,10", + /* def */ "10,100,10", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Parameters for per worker pool request memory pool.\n" MEMPOOL_TEXT ) @@ -863,13 +863,13 @@ PARAM( /* actual location mgt_param_tbl.c */ PARAM( /* name */ pool_sess, - /* typ */ poolparam, + /* type */ poolparam, /* min */ NULL, /* max */ NULL, - /* default */ "10,100,10", + /* def */ "10,100,10", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Parameters for per worker pool session memory pool.\n" MEMPOOL_TEXT ) @@ -877,13 +877,13 @@ PARAM( /* actual location mgt_param_tbl.c */ PARAM( /* name */ pool_vbo, - /* typ */ poolparam, + /* type */ poolparam, /* min */ NULL, /* max */ NULL, - /* default */ "10,100,10", + /* def */ "10,100,10", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Parameters for backend object fetch memory pool.\n" MEMPOOL_TEXT ) @@ -891,26 +891,26 @@ PARAM( PARAM( /* name */ prefer_ipv6, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "off", + /* def */ "off", /* units */ "bool", /* flags */ 0, - /* s-text */ + /* descr */ "Prefer IPv6 address when connecting to backends which have both " "IPv4 and IPv6 addresses." ) PARAM( /* name */ rush_exponent, - /* typ */ uint, + /* type */ uint, /* min */ "2", /* max */ NULL, - /* default */ "3", + /* def */ "3", /* units */ "requests per request", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "How many parked request we start for each completed request on " "the object.\n" "NB: Even with the implict delay of delivery, this parameter " @@ -928,13 +928,13 @@ PARAM( #endif PARAM( /* name */ send_timeout, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "600.000", + /* def */ "600.000", /* units */ "seconds", /* flags */ XYZZY, - /* s-text */ + /* descr */ "Total timeout for ordinary HTTP1 responses. Does not apply to some" " internally generated errors and pipe mode.\n\n" "When 'idle_send_timeout' is hit while sending an HTTP1 response, the" @@ -946,39 +946,39 @@ PARAM( PARAM( /* name */ shortlived, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "10.000", + /* def */ "10.000", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "Objects created with (ttl+grace+keep) shorter than this are " "always put in transient storage." ) PARAM( /* name */ sigsegv_handler, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "on", + /* def */ "on", /* units */ "bool", /* flags */ MUST_RESTART, - /* s-text */ + /* descr */ "Install a signal handler which tries to dump debug information on " "segmentation faults, bus errors and abort signals." ) PARAM( /* name */ syslog_cli_traffic, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "on", + /* def */ "on", /* units */ "bool", /* flags */ 0, - /* s-text */ + /* descr */ "Log all CLI traffic to syslog(LOG_INFO)." ) @@ -989,13 +989,13 @@ PARAM( #endif PARAM( /* name */ tcp_fastopen, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "off", + /* def */ "off", /* units */ "bool", /* flags */ XYZZY, - /* s-text */ + /* descr */ "Enable TCP Fast Open extension." ) #undef XYZZY @@ -1007,26 +1007,26 @@ PARAM( #endif PARAM( /* name */ tcp_keepalive_intvl, - /* typ */ timeout, + /* type */ timeout, /* min */ "1", /* max */ "100", - /* default */ "", + /* def */ "", /* units */ "seconds", /* flags */ XYZZY, - /* s-text */ + /* descr */ "The number of seconds between TCP keep-alive probes. " "Ignored for Unix domain sockets." ) PARAM( /* name */ tcp_keepalive_probes, - /* typ */ uint, + /* type */ uint, /* min */ "1", /* max */ "100", - /* default */ "", + /* def */ "", /* units */ "probes", /* flags */ XYZZY, - /* s-text */ + /* descr */ "The maximum number of TCP keep-alive probes to send before giving " "up and killing the connection if no response is obtained from the " "other end. Ignored for Unix domain sockets." @@ -1034,13 +1034,13 @@ PARAM( PARAM( /* name */ tcp_keepalive_time, - /* typ */ timeout, + /* type */ timeout, /* min */ "1", /* max */ "7200", - /* default */ "", + /* def */ "", /* units */ "seconds", /* flags */ XYZZY, - /* s-text */ + /* descr */ "The number of seconds a connection needs to be idle before TCP " "begins sending out keep-alive probes. " "Ignored for Unix domain sockets." @@ -1051,13 +1051,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_add_delay, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "0.000", + /* def */ "0.000", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Wait at least this long after creating a thread.\n" "\n" "Some (buggy) systems may need a short (sub-second) delay between " @@ -1070,13 +1070,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_watchdog, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.1", /* max */ NULL, - /* default */ "60.000", + /* def */ "60.000", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Thread queue stuck watchdog.\n" "\n" "If no queued work have been released for this long," @@ -1086,13 +1086,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_destroy_delay, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.010", /* max */ NULL, - /* default */ "1.000", + /* def */ "1.000", /* units */ "seconds", /* flags */ DELAYED_EFFECT| EXPERIMENTAL, - /* s-text */ + /* descr */ "Wait this long after destroying a thread.\n" "This controls the decay of thread pools when idle(-ish)." ) @@ -1100,13 +1100,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_fail_delay, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.010", /* max */ NULL, - /* default */ "0.200", + /* def */ "0.200", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Wait at least this long after a failed thread creation before " "trying to create another thread.\n" "\n" @@ -1125,13 +1125,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_max, - /* typ */ thread_pool_max, + /* type */ thread_pool_max, /* min */ "100", /* max */ NULL, - /* default */ "5000", + /* def */ "5000", /* units */ "threads", /* flags */ DELAYED_EFFECT, - /* s-text */ + /* descr */ "The maximum number of worker threads in each pool.\n" "\n" "Do not set this higher than you have to, since excess worker " @@ -1142,13 +1142,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_min, - /* typ */ thread_pool_min, + /* type */ thread_pool_min, /* min */ NULL, /* max */ "5000", - /* default */ "100", + /* def */ "100", /* units */ "threads", /* flags */ DELAYED_EFFECT, - /* s-text */ + /* descr */ "The minimum number of worker threads in each pool.\n" "\n" "Increasing this may help ramp up faster from low load situations " @@ -1159,13 +1159,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_reserve, - /* typ */ thread_pool_reserve, + /* type */ thread_pool_reserve, /* min */ NULL, /* max */ NULL, - /* default */ "0", + /* def */ "0", /* units */ "threads", /* flags */ DELAYED_EFFECT| EXPERIMENTAL, - /* s-text */ + /* descr */ "The number of worker threads reserved for vital tasks " "in each pool.\n" "\n" @@ -1186,13 +1186,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_stack, - /* typ */ bytes, + /* type */ bytes, /* min */ "2k", /* max */ NULL, - /* default */ "56k", + /* def */ "56k", /* units */ "bytes", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Worker thread stack size.\n" "This will likely be rounded up to a multiple of 4k (or whatever " "the page_size might be) by the kernel." @@ -1201,13 +1201,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_timeout, - /* typ */ timeout, + /* type */ timeout, /* min */ "10.000", /* max */ NULL, - /* default */ "300.000", + /* def */ "300.000", /* units */ "seconds", /* flags */ DELAYED_EFFECT| EXPERIMENTAL, - /* s-text */ + /* descr */ "Thread idle threshold.\n" "\n" "Threads in excess of thread_pool_min, which have been idle for at " @@ -1217,13 +1217,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_pools, - /* typ */ uint, + /* type */ uint, /* min */ "1", /* max */ NULL, - /* default */ "2", + /* def */ "2", /* units */ "pools", /* flags */ DELAYED_EFFECT| EXPERIMENTAL, - /* s-text */ + /* descr */ "Number of worker thread pools.\n" "\n" "Increasing the number of worker pools decreases lock " @@ -1242,13 +1242,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_queue_limit, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "20", + /* def */ "20", /* units */ NULL, /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Permitted request queue length per thread-pool.\n" "\n" "This sets the number of requests we will queue, waiting for an " @@ -1259,13 +1259,13 @@ PARAM( /* actual location mgt_pool.c */ PARAM( /* name */ thread_stats_rate, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "10", + /* def */ "10", /* units */ "requests", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "Worker threads accumulate statistics, and dump these into the " "global stats counters if the lock is free when they finish a job " "(request/fetch etc).\n" @@ -1286,13 +1286,13 @@ PARAM( #endif PARAM( /* name */ timeout_idle, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "5.000", + /* def */ "5.000", /* units */ "seconds", /* flags */ XYZZY, - /* s-text */ + /* descr */ "Idle timeout for client connections.\n\n" "A connection is considered idle until we have received the full" " request headers.\n\n" @@ -1304,13 +1304,13 @@ PARAM( PARAM( /* name */ timeout_linger, - /* typ */ timeout, + /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* default */ "0.050", + /* def */ "0.050", /* units */ "seconds", /* flags */ EXPERIMENTAL, - /* s-text */ + /* descr */ "How long the worker thread lingers on an idle session before " "handing it over to the waiter.\n" "When sessions are reused, as much as half of all reuses happen " @@ -1324,39 +1324,39 @@ PARAM( /* actual location mgt_param_tbl.c */ PARAM( /* name */ vcc_allow_inline_c, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "off", + /* def */ "off", /* units */ "bool", /* flags */ 0, - /* s-text */ + /* descr */ "Allow inline C code in VCL." ) /* actual location mgt_param_tbl.c */ PARAM( /* name */ vcc_err_unref, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "on", + /* def */ "on", /* units */ "bool", /* flags */ 0, - /* s-text */ + /* descr */ "Unreferenced VCL objects result in error." ) /* actual location mgt_param_tbl.c */ PARAM( /* name */ vcc_unsafe_path, - /* typ */ bool, + /* type */ bool, /* min */ NULL, /* max */ NULL, - /* default */ "on", + /* def */ "on", /* units */ "bool", /* flags */ 0, - /* s-text */ + /* descr */ "Allow '/' in vmod & include paths.\n" "Allow 'import ... from ...'." ) @@ -1364,26 +1364,26 @@ PARAM( PARAM( /* name */ vcl_cooldown, - /* typ */ timeout, + /* type */ timeout, /* min */ "1.000", /* max */ NULL, - /* default */ "600.000", + /* def */ "600.000", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "How long a VCL is kept warm after being replaced as the " "active VCL (granularity approximately 30 seconds)." ) PARAM( /* name */ max_vcl_handling, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ "2", - /* default */ "1", + /* def */ "1", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Behaviour when attempting to exceed max_vcl loaded VCL.\n" "\n* 0 - Ignore max_vcl parameter.\n" "\n* 1 - Issue warning.\n" @@ -1392,39 +1392,39 @@ PARAM( PARAM( /* name */ max_vcl, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "100", + /* def */ "100", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Threshold of loaded VCL programs. (VCL labels are not counted.)" " Parameter max_vcl_handling determines behaviour." ) PARAM( /* name */ vsm_free_cooldown, - /* typ */ timeout, + /* type */ timeout, /* min */ "10.000", /* max */ "600.000", - /* default */ "60.000", + /* def */ "60.000", /* units */ "seconds", /* flags */ 0, - /* s-text */ + /* descr */ "How long VSM memory is kept warm after a deallocation " "(granularity approximately 2 seconds)." ) PARAM( /* name */ vsl_buffer, - /* typ */ vsl_buffer, + /* type */ vsl_buffer, /* min */ "267", /* max */ NULL, - /* default */ "4k", + /* def */ "4k", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "Bytes of (req-/backend-)workspace dedicated to buffering VSL " "records.\n" "When this parameter is adjusted, most likely workspace_client " @@ -1439,13 +1439,13 @@ PARAM( /* actual location mgt_param_bits.c*/ PARAM( /* name */ vsl_mask, - /* typ */ vsl_mask, + /* type */ vsl_mask, /* min */ NULL, /* max */ NULL, - /* default */ "default", + /* def */ "default", /* units */ NULL, /* flags */ 0, - /* s-text */ + /* descr */ "Mask individual VSL messages from being logged.\n" " default Set default value\n" "\n" @@ -1456,13 +1456,13 @@ PARAM( PARAM( /* name */ vsl_reclen, - /* typ */ vsl_reclen, + /* type */ vsl_reclen, /* min */ "16b", /* max */ NULL, - /* default */ "255b", + /* def */ "255b", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "Maximum number of bytes in SHM log record.", /* dyn_min_reason */ NULL, /* dyn_max_reason */ "vsl_buffer - 12 bytes" @@ -1470,13 +1470,13 @@ PARAM( PARAM( /* name */ vsl_space, - /* typ */ bytes, + /* type */ bytes, /* min */ "1M", /* max */ "4G", - /* default */ "80M", + /* def */ "80M", /* units */ "bytes", /* flags */ MUST_RESTART, - /* s-text */ + /* descr */ "The amount of space to allocate for the VSL fifo buffer in the " "VSM memory segment. If you make this too small, " "varnish{ncsa|log} etc will not be able to keep up. Making it too " @@ -1485,13 +1485,13 @@ PARAM( PARAM( /* name */ vsm_space, - /* typ */ bytes, + /* type */ bytes, /* min */ "1M", /* max */ "1G", - /* default */ "1M", + /* def */ "1M", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "DEPRECATED: This parameter is ignored.\n" "There is no global limit on amount of shared memory now." ) @@ -1500,39 +1500,39 @@ PARAM( /* see mgt_waiter.c */ PARAM( /* name */ waiter, - /* typ */ waiter, + /* type */ waiter, /* min */ NULL, /* max */ NULL, - /* default */ "kqueue (possible values: kqueue, poll)", + /* def */ "kqueue (possible values: kqueue, poll)", /* units */ NULL, /* flags */ MUST_RESTART| WIZARD, - /* s-text */ + /* descr */ "Select the waiter kernel interface." ) #endif PARAM( /* name */ workspace_backend, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "1k", /* max */ NULL, - /* default */ "64k", + /* def */ "64k", /* units */ "bytes", /* flags */ DELAYED_EFFECT, - /* s-text */ + /* descr */ "Bytes of HTTP protocol workspace for backend HTTP req/resp. If " "larger than 4k, use a multiple of 4k for VM efficiency." ) PARAM( /* name */ workspace_client, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "9k", /* max */ NULL, - /* default */ "64k", + /* def */ "64k", /* units */ "bytes", /* flags */ DELAYED_EFFECT, - /* s-text */ + /* descr */ "Bytes of HTTP protocol workspace for clients HTTP req/resp. Use a " "multiple of 4k for VM efficiency.\n" "For HTTP/2 compliance this must be at least 20k, in order to " @@ -1543,13 +1543,13 @@ PARAM( PARAM( /* name */ workspace_session, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "0.25k", /* max */ NULL, - /* default */ "0.75k", + /* def */ "0.75k", /* units */ "bytes", /* flags */ DELAYED_EFFECT, - /* s-text */ + /* descr */ "Allocation size for session structure and workspace. The " "workspace is primarily used for TCP connection addresses. If " "larger than 4k, use a multiple of 4k for VM efficiency." @@ -1557,13 +1557,13 @@ PARAM( PARAM( /* name */ workspace_thread, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "0.25k", /* max */ "8k", - /* default */ "2k", + /* def */ "2k", /* units */ "bytes", /* flags */ DELAYED_EFFECT, - /* s-text */ + /* descr */ "Bytes of auxiliary workspace per thread.\n" "This workspace is used for certain temporary data structures " "during the operation of a worker thread.\n" @@ -1577,13 +1577,13 @@ PARAM( PARAM( /* name */ h2_rx_window_low_water, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "65535", /* max */ "1G", - /* default */ "10M", + /* def */ "10M", /* units */ "bytes", /* flags */ WIZARD, - /* s-text */ + /* descr */ "HTTP2 Receive Window low water mark.\n" "We try to keep the window at least this big\n" "Only affects incoming request bodies (ie: POST, PUT etc.)" @@ -1591,13 +1591,13 @@ PARAM( PARAM( /* name */ h2_rx_window_increment, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "1M", /* max */ "1G", - /* default */ "1M", + /* def */ "1M", /* units */ "bytes", /* flags */ WIZARD, - /* s-text */ + /* descr */ "HTTP2 Receive Window Increments.\n" "How big credits we send in WINDOW_UPDATE frames\n" "Only affects incoming request bodies (ie: POST, PUT etc.)" @@ -1605,13 +1605,13 @@ PARAM( PARAM( /* name */ h2_header_table_size, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "0b", /* max */ NULL, - /* default */ "4k", + /* def */ "4k", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "HTTP2 header table size.\n" "This is the size that will be used for the HPACK dynamic\n" "decoding table." @@ -1619,13 +1619,13 @@ PARAM( PARAM( /* name */ h2_max_concurrent_streams, - /* typ */ uint, + /* type */ uint, /* min */ "0", /* max */ NULL, - /* default */ "100", + /* def */ "100", /* units */ "streams", /* flags */ 0, - /* s-text */ + /* descr */ "HTTP2 Maximum number of concurrent streams.\n" "This is the number of requests that can be active\n" "at the same time for a single HTTP2 connection." @@ -1633,37 +1633,37 @@ PARAM( PARAM( /* name */ h2_initial_window_size, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "0", /* max */ "2147483647b", - /* default */ "65535b", + /* def */ "65535b", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "HTTP2 initial flow control window size." ) PARAM( /* name */ h2_max_frame_size, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "16k", /* max */ "16777215b", - /* default */ "16k", + /* def */ "16k", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "HTTP2 maximum per frame payload size we are willing to accept." ) PARAM( /* name */ h2_max_header_list_size, - /* typ */ bytes_u, + /* type */ bytes_u, /* min */ "0b", /* max */ NULL, - /* default */ "2147483647b", + /* def */ "2147483647b", /* units */ "bytes", /* flags */ 0, - /* s-text */ + /* descr */ "HTTP2 maximum size of an uncompressed header list." ) From dridi.boukelmoune at gmail.com Mon Mar 2 14:09:09 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 14:09:09 +0000 (UTC) Subject: [master] da279c2ea Move def and units fields up in struct parspec Message-ID: <20200302140909.069B6ABBF5@lists.varnish-cache.org> commit da279c2eaf4003737c022852244717886edc0c65 Author: Dridi Boukelmoune Date: Mon Mar 2 11:36:03 2020 +0100 Move def and units fields up in struct parspec It's a bit weird that the default value and the unit is separated from the min and max. This aligns struct parspec field order with the PARAM() macro for these two fields. In static parspec definitions, the flags field can now be omitted if there are no flags, and the parameter is not subject to dynamic bounds. diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index 98d47f77d..e3a68fbee 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -43,6 +43,8 @@ struct parspec { volatile void *priv; const char *min; const char *max; + const char *def; + const char *units; const char *descr; int flags; #define DELAYED_EFFECT (1<<0) @@ -55,9 +57,6 @@ struct parspec { #define ONLY_ROOT (1<<7) #define NOT_IMPLEMENTED (1<<8) - const char *def; - const char *units; - const char *dyn_min_reason; const char *dyn_max_reason; char *dyn_min; diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index 14adeba6d..cf39ee027 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -252,27 +252,32 @@ tweak_feature(struct vsb *vsb, const struct parspec *par, const char *arg) */ struct parspec VSL_parspec[] = { - { "vsl_mask", tweak_vsl_mask, NULL, NULL, NULL, + { "vsl_mask", tweak_vsl_mask, NULL, + NULL, NULL, "default", + NULL, "Mask individual VSL messages from being logged.\n" "\tdefault\tSet default value\n" "\nUse +/- prefix in front of VSL tag name to unmask/mask " - "individual VSL messages.", - 0, "default", "" }, - { "debug", tweak_debug, NULL, NULL, NULL, + "individual VSL messages." }, + { "debug", tweak_debug, NULL, + NULL, NULL, "none", + NULL, "Enable/Disable various kinds of debugging.\n" "\tnone\tDisable all debugging\n\n" "Use +/- prefix to set/reset individual bits:" #define DEBUG_BIT(U, l, d) "\n\t" #l "\t" d #include "tbl/debug_bits.h" #undef DEBUG_BIT - , 0, "none", "" }, - { "feature", tweak_feature, NULL, NULL, NULL, + }, + { "feature", tweak_feature, NULL, + NULL, NULL, "none", + NULL, "Enable/Disable various minor features.\n" "\tnone\tDisable all features.\n\n" "Use +/- prefix to enable/disable individual feature:" #define FEATURE_BIT(U, l, d, ld) "\n\t" #l "\t" d #include "tbl/feature_bits.h" #undef FEATURE_BIT - , 0, "none", "" }, + }, { NULL, NULL, NULL } }; diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 09bff349a..0f5f087d0 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -46,66 +46,59 @@ struct parspec mgt_parspec[] = { #define PARAM(nm, ty, mi, ma, de, un, fl, st, ...) \ - { #nm, tweak_##ty, &mgt_param.nm, mi, ma, st, fl, de, un, \ + { #nm, tweak_##ty, &mgt_param.nm, mi, ma, de, un, st, fl, \ __VA_ARGS__ }, #include "tbl/params.h" { "cc_command", tweak_string, &mgt_cc_cmd, - NULL, NULL, + NULL, NULL, VCC_CC, + NULL, "Command used for compiling the C source code to a " "dlopen(3) loadable object. Any occurrence of %s in " "the string will be replaced with the source file name, " "and %o will be replaced with the output file name.", - MUST_RELOAD, - VCC_CC , NULL }, + MUST_RELOAD }, { "vcl_path", tweak_string, &mgt_vcl_path, - NULL, NULL, + NULL, NULL, VARNISH_VCL_DIR, + NULL, "Directory (or colon separated list of directories) " "from which relative VCL filenames (vcl.load and " "include) are to be found. By default Varnish searches " "VCL files in both the system configuration and shared " "data directories to allow packages to drop their VCL " "files in a standard location where relative includes " - "would work.", - 0, - VARNISH_VCL_DIR, - NULL }, + "would work." }, { "vmod_path", tweak_string, &mgt_vmod_path, - NULL, NULL, + NULL, NULL, VARNISH_VMOD_DIR, + NULL, "Directory (or colon separated list of directories) " - "where VMODs are to be found.", - 0, - VARNISH_VMOD_DIR, - NULL }, + "where VMODs are to be found." }, { "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref, - NULL, NULL, - "Unreferenced VCL objects result in error.", - 0, - "on", "bool" }, + NULL, NULL, "on", + "bool", + "Unreferenced VCL objects result in error." }, { "vcc_allow_inline_c", tweak_bool, &mgt_vcc_allow_inline_c, - NULL, NULL, - "Allow inline C code in VCL.", - 0, - "off", "bool" }, + NULL, NULL, "off", + "bool", + "Allow inline C code in VCL." }, { "vcc_unsafe_path", tweak_bool, &mgt_vcc_unsafe_path, - NULL, NULL, + NULL, NULL, "on", + "bool", "Allow '/' in vmod & include paths.\n" - "Allow 'import ... from ...'.", - 0, - "on", "bool" }, + "Allow 'import ... from ...'." }, { "pcre_match_limit", tweak_uint, &mgt_param.vre_limits.match, - "1", NULL, + "1", NULL, "10000", + NULL, "The limit for the number of calls to the internal match()" " function in pcre_exec().\n\n" "(See: PCRE_EXTRA_MATCH_LIMIT in pcre docs.)\n\n" "This parameter limits how much CPU time" - " regular expression matching can soak up.", - 0, - "10000", ""}, + " regular expression matching can soak up." }, { "pcre_match_limit_recursion", tweak_uint, &mgt_param.vre_limits.match_recursion, - "1", NULL, + "1", NULL, "20", + NULL, "The recursion depth-limit for the internal match() function" " in a pcre_exec().\n\n" "(See: PCRE_EXTRA_MATCH_LIMIT_RECURSION in pcre docs.)\n\n" @@ -116,27 +109,22 @@ struct parspec mgt_parspec[] = { " matching failures.\n\n" "Matching failures will show up in the log as VCL_Error" " messages with regexp errors -27 or -21.\n\n" - "Testcase r01576 can be useful when tuning this parameter.", - 0, - "20", ""}, + "Testcase r01576 can be useful when tuning this parameter." }, { "pool_req", tweak_poolparam, &mgt_param.req_pool, - NULL, NULL, + NULL, NULL, "10,100,10", + NULL, "Parameters for per worker pool request memory pool.\n\n" - MEMPOOL_TEXT, - 0, - "10,100,10", ""}, + MEMPOOL_TEXT }, { "pool_sess", tweak_poolparam, &mgt_param.sess_pool, - NULL, NULL, + NULL, NULL, "10,100,10", + NULL, "Parameters for per worker pool session memory pool.\n\n" - MEMPOOL_TEXT, - 0, - "10,100,10", ""}, + MEMPOOL_TEXT }, { "pool_vbo", tweak_poolparam, &mgt_param.vbo_pool, - NULL, NULL, + NULL, NULL, "10,100,10", + NULL, "Parameters for backend object fetch memory pool.\n\n" - MEMPOOL_TEXT, - 0, - "10,100,10", ""}, + MEMPOOL_TEXT }, { NULL, NULL, NULL } }; diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index dc8b847c8..4d30ca427 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -89,7 +89,8 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par, struct parspec WRK_parspec[] = { { "thread_pools", tweak_uint, &mgt_param.wthread_pools, - "1", NULL, + "1", NULL, "2", + "pools", "Number of worker thread pools.\n" "\n" "Increasing the number of worker pools decreases lock " @@ -103,21 +104,20 @@ struct parspec WRK_parspec[] = { "\n" "Can be increased on the fly, but decreases require a " "restart to take effect.", - EXPERIMENTAL | DELAYED_EFFECT, - "2", "pools" }, + EXPERIMENTAL | DELAYED_EFFECT }, { "thread_pool_max", tweak_thread_pool_max, &mgt_param.wthread_max, - NULL, NULL, + NULL, NULL, "5000", + "threads", "The maximum number of worker threads in each pool.\n" "\n" "Do not set this higher than you have to, since excess " "worker threads soak up RAM and CPU and generally just get " "in the way of getting work done.", DELAYED_EFFECT, - "5000", "threads", "thread_pool_min" }, { "thread_pool_min", tweak_thread_pool_min, &mgt_param.wthread_min, - "5", // TASK_QUEUE__END - NULL, + "5" /* TASK_QUEUE__END */, NULL, "100", + "threads", "The minimum number of worker threads in each pool.\n" "\n" "Increasing this may help ramp up faster from low load " @@ -127,11 +127,11 @@ struct parspec WRK_parspec[] = { "but this parameter is strongly recommended to be " "at least 10", // 2 * TASK_QUEUE__END DELAYED_EFFECT, - "100", "threads", NULL, "thread_pool_max" }, { "thread_pool_reserve", tweak_uint, &mgt_param.wthread_reserve, - NULL, NULL, + NULL, NULL, "0", + "threads", "The number of worker threads reserved for vital tasks " "in each pool.\n" "\n" @@ -147,37 +147,37 @@ struct parspec WRK_parspec[] = { "Default is 0 to auto-tune (5% of thread_pool_min).\n" "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", DELAYED_EFFECT, - "0", "threads", NULL, "95% of thread_pool_min" }, { "thread_pool_timeout", tweak_timeout, &mgt_param.wthread_timeout, - "10", NULL, + "10", NULL, "300", + "seconds", "Thread idle threshold.\n" "\n" "Threads in excess of thread_pool_min, which have been idle " "for at least this long, will be destroyed.", - EXPERIMENTAL | DELAYED_EFFECT, - "300", "seconds" }, + EXPERIMENTAL | DELAYED_EFFECT }, { "thread_pool_watchdog", tweak_timeout, &mgt_param.wthread_watchdog, - "0.1", NULL, + "0.1", NULL, "60", + "seconds", "Thread queue stuck watchdog.\n" "\n" "If no queued work have been released for this long," " the worker process panics itself.", - EXPERIMENTAL, - "60", "seconds" }, + EXPERIMENTAL }, { "thread_pool_destroy_delay", tweak_timeout, &mgt_param.wthread_destroy_delay, - "0.01", NULL, + "0.01", NULL, "1", + "seconds", "Wait this long after destroying a thread.\n" "\n" "This controls the decay of thread pools when idle(-ish).", - EXPERIMENTAL | DELAYED_EFFECT, - "1", "seconds" }, + EXPERIMENTAL | DELAYED_EFFECT }, { "thread_pool_add_delay", tweak_timeout, &mgt_param.wthread_add_delay, - "0", NULL, + "0", NULL, "0", + "seconds", "Wait at least this long after creating a thread.\n" "\n" "Some (buggy) systems may need a short (sub-second) " @@ -186,11 +186,11 @@ struct parspec WRK_parspec[] = { "'threads_failed' counter grow too much.\n" "\n" "Setting this too high results in insufficient worker threads.", - EXPERIMENTAL, - "0", "seconds" }, + EXPERIMENTAL }, { "thread_pool_fail_delay", tweak_timeout, &mgt_param.wthread_fail_delay, - "10e-3", NULL, + "10e-3", NULL, "0.2", + "seconds", "Wait at least this long after a failed thread creation " "before trying to create another thread.\n" "\n" @@ -205,31 +205,31 @@ struct parspec WRK_parspec[] = { "It may also help to increase thread_pool_timeout and " "thread_pool_min, to reduce the rate at which treads are " "destroyed and later recreated.", - EXPERIMENTAL, - "0.2", "seconds" }, + EXPERIMENTAL }, { "thread_stats_rate", tweak_uint, &mgt_param.wthread_stats_rate, - "0", NULL, + "0", NULL, "10", + "requests", "Worker threads accumulate statistics, and dump these into " "the global stats counters if the lock is free when they " "finish a job (request/fetch etc.)\n" "This parameters defines the maximum number of jobs " "a worker thread may handle, before it is forced to dump " "its accumulated stats into the global counters.", - EXPERIMENTAL, - "10", "requests" }, + EXPERIMENTAL }, { "thread_queue_limit", tweak_uint, &mgt_param.wthread_queue_limit, - "0", NULL, + "0", NULL, "20", + NULL, "Permitted request queue length per thread-pool.\n" "\n" "This sets the number of requests we will queue, waiting " "for an available thread. Above this limit sessions will " "be dropped instead of queued.", - EXPERIMENTAL, - "20", "" }, + EXPERIMENTAL }, { "thread_pool_stack", tweak_bytes, &mgt_param.wthread_stacksize, - NULL, NULL, + NULL, NULL, NULL, // default set in mgt_main.c + "bytes", "Worker thread stack size.\n" "This will likely be rounded up to a multiple of 4k" " (or whatever the page_size might be) by the kernel.\n" @@ -264,7 +264,6 @@ struct parspec WRK_parspec[] = { " overflow occurs. Setting it in 150%-200%" " increments is recommended until stack overflows" " cease to occur.", - DELAYED_EFFECT, - NULL, "bytes" }, // default set in mgt_main.c + DELAYED_EFFECT }, { NULL, NULL, NULL } }; From dridi.boukelmoune at gmail.com Mon Mar 2 14:09:09 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 14:09:09 +0000 (UTC) Subject: [master] ed09c766a Document thread_pool_stack's default value Message-ID: <20200302140909.2C60CABBF9@lists.varnish-cache.org> commit ed09c766a5692a6f24b40b58703ecded4df5e264 Author: Dridi Boukelmoune Date: Mon Mar 2 11:38:29 2020 +0100 Document thread_pool_stack's default value It turns out it's not just minimum and maximum values that can be dynamic. diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 1ce83dee6..27610d57c 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -766,15 +766,16 @@ MCF_DumpRstParam(void) if (pp->units != NULL && *pp->units != '\0') printf("\t* Units: %s\n", pp->units); - printf("\t* Default: %s\n", pp->def); - if (pp->dyn_min_reason != NULL) - printf("\t* Minimum: %s\n", pp->dyn_min_reason); - else if (pp->min != NULL) - printf("\t* Minimum: %s\n", pp->min); - if (pp->dyn_max_reason != NULL) - printf("\t* Maximum: %s\n", pp->dyn_max_reason); - else if (pp->max != NULL) - printf("\t* Maximum: %s\n", pp->max); +#define MCF_DYN_REASON(lbl, nm) \ + if (pp->dyn_ ## nm ## _reason != NULL) \ + printf("\t* " #lbl ": %s\n", \ + pp->dyn_ ## nm ## _reason); \ + else if (pp->nm != NULL) \ + printf("\t* " #lbl ": %s\n", pp->nm); + MCF_DYN_REASON(Default, def); + MCF_DYN_REASON(Minimum, min); + MCF_DYN_REASON(Maximum, max); +#undef MCF_DYN_REASON /* * XXX: we should mark the params with one/two flags * XXX: that say if ->min/->max are valid, so we diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index e3a68fbee..a23616686 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -59,6 +59,7 @@ struct parspec { const char *dyn_min_reason; const char *dyn_max_reason; + const char *dyn_def_reason; char *dyn_min; char *dyn_max; char *dyn_def; diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 4d30ca427..63b3ede40 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -228,7 +228,7 @@ struct parspec WRK_parspec[] = { EXPERIMENTAL }, { "thread_pool_stack", tweak_bytes, &mgt_param.wthread_stacksize, - NULL, NULL, NULL, // default set in mgt_main.c + NULL, NULL, NULL, // default set in mgt_param.c "bytes", "Worker thread stack size.\n" "This will likely be rounded up to a multiple of 4k" @@ -264,6 +264,7 @@ struct parspec WRK_parspec[] = { " overflow occurs. Setting it in 150%-200%" " increments is recommended until stack overflows" " cease to occur.", - DELAYED_EFFECT }, + DELAYED_EFFECT, + NULL, NULL, "sysconf(_SC_THREAD_STACK_MIN)" }, { NULL, NULL, NULL } }; From dridi.boukelmoune at gmail.com Mon Mar 2 14:09:09 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 14:09:09 +0000 (UTC) Subject: [master] a4b894913 Use the struct parspec order in params.h Message-ID: <20200302140909.46756ABBFD@lists.varnish-cache.org> commit a4b89491360d533faf3471dbb0d6c5859411b375 Author: Dridi Boukelmoune Date: Mon Mar 2 11:42:13 2020 +0100 Use the struct parspec order in params.h And with that trailing zero-flags can be omitted. diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 0f5f087d0..ea4da117a 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -45,9 +45,7 @@ "\tmax_age\tmax age of free element." struct parspec mgt_parspec[] = { -#define PARAM(nm, ty, mi, ma, de, un, fl, st, ...) \ - { #nm, tweak_##ty, &mgt_param.nm, mi, ma, de, un, st, fl, \ - __VA_ARGS__ }, +#define PARAM(nm, ty, ...) { #nm, tweak_##ty, &mgt_param.nm, __VA_ARGS__ }, #include "tbl/params.h" { "cc_command", tweak_string, &mgt_cc_cmd, diff --git a/include/tbl/params.h b/include/tbl/params.h index 41c0c4d7e..d115531f1 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -27,7 +27,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * PARAM(nm, ty, mi, ma, de, un, fl, st[, dyn_min_reason, dyn_max_reason]) + * PARAM(nm, ty, ...) + * + * Variable arguments refer to struct parspec fields from min to + * dyn_def_reason. */ /*lint -save -e525 -e539 */ @@ -48,14 +51,14 @@ PARAM( /* max */ NULL, /* def */ "on", /* units */ "bool", - /* flags */ XYZZY, /* descr */ "Enable kernel accept-filters. This may require a kernel module to " "be loaded to have an effect when enabled.\n\n" "Enabling accept_filter may prevent some requests to reach Varnish " "in the first place. Malformed requests may go unnoticed and not " "increase the client_req_400 counter. GET or HEAD requests with a " - "body may be blocked altogether." + "body may be blocked altogether.", + /* flags */ XYZZY ) #undef XYZZY @@ -66,12 +69,12 @@ PARAM( /* max */ "1", /* def */ "0.9", /* units */ NULL, - /* flags */ EXPERIMENTAL, /* descr */ "If we run out of resources, such as file descriptors or worker " "threads, the acceptor will sleep between accepts.\n" "This parameter (multiplicatively) reduce the sleep duration for " - "each successful accept. (ie: 0.9 = reduce by 10%)" + "each successful accept. (ie: 0.9 = reduce by 10%)", + /* flags */ EXPERIMENTAL ) PARAM( @@ -81,12 +84,12 @@ PARAM( /* max */ "1", /* def */ "0", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "If we run out of resources, such as file descriptors or worker " "threads, the acceptor will sleep between accepts.\n" "This parameter control how much longer we sleep, each time we " - "fail to accept a new connection." + "fail to accept a new connection.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -96,12 +99,12 @@ PARAM( /* max */ "10", /* def */ "0.05", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "If we run out of resources, such as file descriptors or worker " "threads, the acceptor will sleep between accepts.\n" "This parameter limits how long it can sleep between attempts to " - "accept new connections." + "accept new connections.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -111,7 +114,6 @@ PARAM( /* max */ NULL, /* def */ "on", /* units */ "bool", - /* flags */ 0, /* descr */ "Automatically restart the child/worker process if it dies." ) @@ -123,7 +125,6 @@ PARAM( /* max */ NULL, /* def */ "on", /* units */ "bool", - /* flags */ 0, /* descr */ "Eliminate older identical bans when a new ban is added. This saves " "CPU cycles by not comparing objects to identical bans.\n" @@ -138,7 +139,6 @@ PARAM( /* max */ NULL, /* def */ "0", /* units */ "bans", - /* flags */ EXPERIMENTAL, /* descr */ "Expurge long tail content from the cache to keep the number of bans " "below this value. 0 disables.\n\n" @@ -158,7 +158,8 @@ PARAM( "additional latency due to request ban testing is in the order of " "ban_cutoff / rate(bans_lurker_tests_tested). For example, for " "rate(bans_lurker_tests_tested) = 2M/s and a tolerable latency of " - "100ms, a good value for ban_cutoff may be 200K." + "100ms, a good value for ban_cutoff may be 200K.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -168,7 +169,6 @@ PARAM( /* max */ NULL, /* def */ "60", /* units */ "seconds", - /* flags */ 0, /* descr */ "The ban lurker will ignore bans until they are this old. " "When a ban is added, the active traffic will be tested against it " @@ -185,7 +185,6 @@ PARAM( /* max */ NULL, /* def */ "1000", /* units */ NULL, - /* flags */ 0, /* descr */ "The ban lurker sleeps ${ban_lurker_sleep} after examining this " "many objects." @@ -199,7 +198,6 @@ PARAM( /* max */ NULL, /* def */ "0.010", /* units */ "seconds", - /* flags */ 0, /* descr */ "How long the ban lurker sleeps after examining ${ban_lurker_batch} " "objects." @@ -214,10 +212,10 @@ PARAM( /* max */ NULL, /* def */ "0.010", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "How long the ban lurker sleeps when giving way to lookup" - " due to lock contention." + " due to lock contention.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -227,7 +225,6 @@ PARAM( /* max */ NULL, /* def */ "60", /* units */ "seconds", - /* flags */ 0, /* descr */ "Default timeout for receiving first byte from backend. We only " "wait for this many seconds for the first byte before giving up.\n" @@ -243,7 +240,6 @@ PARAM( /* max */ NULL, /* def */ "60", /* units */ "seconds", - /* flags */ 0, /* descr */ "We only wait for this many seconds between bytes received from " "the backend before giving up the fetch.\n" @@ -258,7 +254,6 @@ PARAM( /* max */ NULL, /* def */ "60", /* units */ "seconds", - /* flags */ 0, /* descr */ "Timeout before we close unused backend connections." ) @@ -270,7 +265,6 @@ PARAM( /* max */ NULL, /* def */ "10.000", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "When connecting to backends, certain error codes " "(EADDRNOTAVAIL, EACCESS, EPERM) signal a local resource shortage " @@ -278,7 +272,8 @@ PARAM( "may worsen the situation due to the complexity of the operations " "involved in the kernel.\n" "This parameter prevents repeated connection attempts for the " - "configured duration." + "configured duration.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -288,14 +283,14 @@ PARAM( /* max */ NULL, /* def */ "0.250", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "When connecting to backends, certain error codes (ECONNREFUSED, " "ENETUNREACH) signal fundamental connection issues such as the backend " "not accepting connections or routing problems for which repeated " "connection attempts are considered useless\n" "This parameter prevents repeated connection attempts for the " - "configured duration." + "configured duration.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -305,7 +300,6 @@ PARAM( /* max */ "99999999b", /* def */ "48k", /* units */ "bytes", - /* flags */ 0, /* descr */ "Maximum size of CLI response. If the response exceeds this " "limit, the response code will be 201 instead of 200 and the last " @@ -319,7 +313,6 @@ PARAM( /* max */ NULL, /* def */ "60.000", /* units */ "seconds", - /* flags */ 0, /* descr */ "Timeout for the childs replies to CLI requests from the " "mgt_param." @@ -332,7 +325,6 @@ PARAM( /* max */ NULL, /* def */ "10", /* units */ "seconds", - /* flags */ 0, /* descr */ "How much clockskew we are willing to accept between the backend " "and our own clock." @@ -345,7 +337,6 @@ PARAM( /* max */ NULL, /* def */ "1.000", /* units */ "seconds", - /* flags */ 0, /* descr */ "How much observed clock step we are willing to accept before " "we panic." @@ -358,7 +349,6 @@ PARAM( /* max */ NULL, /* def */ "3.500", /* units */ "seconds", - /* flags */ 0, /* descr */ "Default connection timeout for backend connections. We only try " "to connect to the backend for this many seconds before giving up. " @@ -373,10 +363,10 @@ PARAM( /* max */ "254.000", /* def */ "180.000", /* units */ "seconds", - /* flags */ WIZARD, /* descr */ "How long the critbit hasher keeps deleted objheads on the cooloff " - "list." + "list.", + /* flags */ WIZARD ) #if 0 @@ -389,7 +379,6 @@ PARAM( /* max */ NULL, /* def */ NULL, /* units */ NULL, - /* flags */ 0, /* descr */ "Enable/Disable various kinds of debugging.\n" " none Disable all debugging\n" @@ -416,11 +405,11 @@ PARAM( /* max */ NULL, /* def */ "10.000", /* units */ "seconds", - /* flags */ OBJ_STICKY, /* descr */ "Default grace period. We will deliver an object this long after " "it has expired, provided another thread is attempting to get a " - "new copy." + "new copy.", + /* flags */ OBJ_STICKY ) PARAM( @@ -430,12 +419,12 @@ PARAM( /* max */ NULL, /* def */ "0.000", /* units */ "seconds", - /* flags */ OBJ_STICKY, /* descr */ "Default keep period. We will keep a useless object around this " "long, making it available for conditional backend fetches. That " "means that the object will be removed from the cache at the end " - "of ttl+grace+keep." + "of ttl+grace+keep.", + /* flags */ OBJ_STICKY ) PARAM( @@ -445,10 +434,10 @@ PARAM( /* max */ NULL, /* def */ "120.000", /* units */ "seconds", - /* flags */ OBJ_STICKY, /* descr */ "The TTL assigned to objects if neither the backend nor the VCL " - "code assigns one." + "code assigns one.", + /* flags */ OBJ_STICKY ) PARAM( @@ -458,11 +447,11 @@ PARAM( /* max */ "1024", // XXX stringify IOV_MAX /* def */ "64", /* units */ "struct iovec (=16 bytes)", - /* flags */ WIZARD, /* descr */ "Number of io vectors to allocate for HTTP1 protocol transmission." " A HTTP1 header needs 7 + 2 per HTTP header field." - " Allocated from workspace_thread." + " Allocated from workspace_thread.", + /* flags */ WIZARD ) #if 0 @@ -475,7 +464,6 @@ PARAM( /* max */ NULL, /* def */ NULL, /* units */ NULL, - /* flags */ 0, /* descr */ "Enable/Disable various minor features.\n" " none Disable all features.\n" @@ -498,12 +486,12 @@ PARAM( /* max */ NULL, /* def */ "16k", /* units */ "bytes", - /* flags */ EXPERIMENTAL, /* descr */ "The default chunksize used by fetcher. This should be bigger than " "the majority of objects with short TTLs.\n" "Internal limits in the storage_file module makes increases above " - "128kb a dubious idea." + "128kb a dubious idea.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -513,10 +501,10 @@ PARAM( /* max */ NULL, /* def */ "0.25G", /* units */ "bytes", - /* flags */ EXPERIMENTAL, /* descr */ "The maximum chunksize we attempt to allocate from storage. Making " - "this too large may cause delays and storage fragmentation." + "this too large may cause delays and storage fragmentation.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -526,13 +514,13 @@ PARAM( /* max */ NULL, /* def */ "32k", /* units */ "bytes", - /* flags */ EXPERIMENTAL, /* descr */ "Size of malloc buffer used for gzip processing.\n" "These buffers are used for in-transit data, for instance " "gunzip'ed data being sent to a client.Making this space to small " "results in more overhead, writes to sockets etc, making it too " - "big is probably just a waste of memory." + "big is probably just a waste of memory.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -542,7 +530,6 @@ PARAM( /* max */ "9", /* def */ "6", /* units */ NULL, - /* flags */ 0, /* descr */ "Gzip compression level: 0=debug, 1=fast, 9=best" ) @@ -554,7 +541,6 @@ PARAM( /* max */ "9", /* def */ "8", /* units */ NULL, - /* flags */ 0, /* descr */ "Gzip memory level 1=slow/least, 9=fast/most compression.\n" "Memory impact is 1=1k, 2=2k, ... 9=256k." @@ -567,7 +553,6 @@ PARAM( /* max */ NULL, /* def */ "on", /* units */ "bool", - /* flags */ 0, /* descr */ "Enable gzip support. When enabled Varnish request compressed " "objects from the backend and store them compressed. If a client " @@ -592,7 +577,6 @@ PARAM( /* max */ "65535", /* def */ "64", /* units */ "header lines", - /* flags */ 0, /* descr */ "Maximum number of HTTP header lines we allow in " "{req|resp|bereq|beresp}.http (obj.http is autosized to the exact " @@ -608,7 +592,6 @@ PARAM( /* max */ NULL, /* def */ "on", /* units */ "bool", - /* flags */ 0, /* descr */ "Enable support for HTTP Range headers." ) @@ -620,7 +603,6 @@ PARAM( /* max */ NULL, /* def */ "8k", /* units */ "bytes", - /* flags */ 0, /* descr */ "Maximum length of any HTTP client request header we will allow. " "The limit is inclusive its continuation lines." @@ -633,7 +615,6 @@ PARAM( /* max */ NULL, /* def */ "32k", /* units */ "bytes", - /* flags */ 0, /* descr */ "Maximum number of bytes of HTTP client request we will deal with. " " This is a limit on all bytes up to the double blank line which " @@ -650,7 +631,6 @@ PARAM( /* max */ NULL, /* def */ "8k", /* units */ "bytes", - /* flags */ 0, /* descr */ "Maximum length of any HTTP backend response header we will allow. " " The limit is inclusive its continuation lines." @@ -663,7 +643,6 @@ PARAM( /* max */ NULL, /* def */ "32k", /* units */ "bytes", - /* flags */ 0, /* descr */ "Maximum number of bytes of HTTP backend response we will deal " "with. This is a limit on all bytes up to the double blank line " @@ -689,13 +668,13 @@ PARAM( /* max */ NULL, /* def */ "60.000", /* units */ "seconds", - /* flags */ XYZZY, /* descr */ "Send timeout for individual pieces of data on client connections." " May get extended if 'send_timeout' applies.\n\n" "When this timeout is hit, the session is closed.\n\n" "See the man page for `setsockopt(2)` or `socket(7)` under" - " ``SO_SNDTIMEO`` for more information." + " ``SO_SNDTIMEO`` for more information.", + /* flags */ XYZZY ) #undef XYZZY @@ -706,9 +685,9 @@ PARAM( /* max */ NULL, /* def */ "1024", /* units */ "connections", - /* flags */ MUST_RESTART, /* descr */ - "Listen queue depth." + "Listen queue depth.", + /* flags */ MUST_RESTART ) PARAM( @@ -718,13 +697,13 @@ PARAM( /* max */ NULL, /* def */ "2.000", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "Grace period before object moves on LRU list.\n" "Objects are only moved to the front of the LRU list if they have " "not been moved there already inside this timeout period. This " "reduces the amount of lock operations necessary for LRU list " - "access." + "access.", + /* flags */ EXPERIMENTAL ) PARAM( @@ -734,7 +713,6 @@ PARAM( /* max */ NULL, /* def */ "5", /* units */ "levels", - /* flags */ 0, /* descr */ "Maximum depth of esi:include processing." ) @@ -746,7 +724,6 @@ PARAM( /* max */ NULL, /* def */ "4", /* units */ "restarts", - /* flags */ 0, /* descr */ "Upper limit on how many times a request can restart." ) @@ -758,7 +735,6 @@ PARAM( /* max */ NULL, /* def */ "4", /* units */ "retries", - /* flags */ 0, /* descr */ "Upper limit on how many times a backend fetch can retry." ) @@ -770,10 +746,10 @@ PARAM( /* max */ NULL, /* def */ "50", /* units */ "allocations", - /* flags */ EXPERIMENTAL, /* descr */ "Maximum number of objects we attempt to nuke in order to make " - "space for a object body." + "space for a object body.", + /* flags */ EXPERIMENTAL ) #if 0 @@ -785,7 +761,6 @@ PARAM( /* max */ NULL, /* def */ "1.000", /* units */ NULL, - /* flags */ 0, /* descr */ "The limit for the number of internal matching function calls in " "a pcre_exec() execution." @@ -799,7 +774,6 @@ PARAM( /* max */ NULL, /* def */ "1.000", /* units */ NULL, - /* flags */ 0, /* descr */ "The limit for the number of internal matching function " "recursions in a pcre_exec() execution." @@ -813,11 +787,11 @@ PARAM( /* max */ NULL, /* def */ "3", /* units */ "seconds", - /* flags */ MUST_RESTART, /* descr */ "Interval between pings from parent to child.\n" "Zero will disable pinging entirely, which makes it possible to " - "attach a debugger to the child." + "attach a debugger to the child.", + /* flags */ MUST_RESTART ) PARAM( @@ -827,7 +801,6 @@ PARAM( /* max */ NULL, /* def */ "0", /* units */ "connections", - /* flags */ 0, /* descr */ "Maximum number of sessions dedicated to pipe transactions." ) @@ -839,7 +812,6 @@ PARAM( /* max */ NULL, /* def */ "60.000", /* units */ "seconds", - /* flags */ 0, /* descr */ "Idle timeout for PIPE sessions. If nothing have been received in " "either direction for this many seconds, the session is closed." @@ -854,7 +826,6 @@ PARAM( /* max */ NULL, /* def */ "10,100,10", /* units */ NULL, - /* flags */ 0, /* descr */ "Parameters for per worker pool request memory pool.\n" MEMPOOL_TEXT @@ -868,7 +839,6 @@ PARAM( /* max */ NULL, /* def */ "10,100,10", /* units */ NULL, - /* flags */ 0, /* descr */ "Parameters for per worker pool session memory pool.\n" MEMPOOL_TEXT @@ -882,7 +852,6 @@ PARAM( /* max */ NULL, /* def */ "10,100,10", /* units */ NULL, - /* flags */ 0, /* descr */ "Parameters for backend object fetch memory pool.\n" MEMPOOL_TEXT @@ -896,7 +865,6 @@ PARAM( /* max */ NULL, /* def */ "off", /* units */ "bool", - /* flags */ 0, /* descr */ "Prefer IPv6 address when connecting to backends which have both " "IPv4 and IPv6 addresses." @@ -909,12 +877,12 @@ PARAM( /* max */ NULL, /* def */ "3", /* units */ "requests per request", - /* flags */ EXPERIMENTAL, /* descr */ "How many parked request we start for each completed request on " "the object.\n" "NB: Even with the implict delay of delivery, this parameter " - "controls an exponential increase in number of worker threads." + "controls an exponential increase in number of worker threads.", + /* flags */ EXPERIMENTAL ) #if defined(XYZZY) @@ -933,14 +901,14 @@ PARAM( /* max */ NULL, /* def */ "600.000", /* units */ "seconds", - /* flags */ XYZZY, /* descr */ "Total timeout for ordinary HTTP1 responses. Does not apply to some" " internally generated errors and pipe mode.\n\n" "When 'idle_send_timeout' is hit while sending an HTTP1 response, the" " timeout is extended unless the total time already taken for sending" " the response in its entirety exceeds this many seconds.\n\n" - "When this timeout is hit, the session is closed" + "When this timeout is hit, the session is closed", + /* flags */ XYZZY ) #undef XYZZY @@ -951,7 +919,6 @@ PARAM( /* max */ NULL, /* def */ "10.000", /* units */ "seconds", - /* flags */ 0, /* descr */ "Objects created with (ttl+grace+keep) shorter than this are " "always put in transient storage." @@ -964,10 +931,10 @@ PARAM( /* max */ NULL, /* def */ "on", /* units */ "bool", - /* flags */ MUST_RESTART, /* descr */ "Install a signal handler which tries to dump debug information on " - "segmentation faults, bus errors and abort signals." + "segmentation faults, bus errors and abort signals.", + /* flags */ MUST_RESTART ) PARAM( @@ -977,7 +944,6 @@ PARAM( /* max */ NULL, /* def */ "on", /* units */ "bool", - /* flags */ 0, /* descr */ "Log all CLI traffic to syslog(LOG_INFO)." ) @@ -994,9 +960,9 @@ PARAM( /* max */ NULL, /* def */ "off", /* units */ "bool", - /* flags */ XYZZY, /* descr */ - "Enable TCP Fast Open extension." + "Enable TCP Fast Open extension.", + /* flags */ XYZZY ) #undef XYZZY @@ -1012,10 +978,10 @@ PARAM( /* max */ "100", /* def */ "", /* units */ "seconds", - /* flags */ XYZZY, /* descr */ "The number of seconds between TCP keep-alive probes. " - "Ignored for Unix domain sockets." + "Ignored for Unix domain sockets.", + /* flags */ XYZZY ) PARAM( @@ -1025,11 +991,11 @@ PARAM( /* max */ "100", /* def */ "", /* units */ "probes", - /* flags */ XYZZY, /* descr */ "The maximum number of TCP keep-alive probes to send before giving " "up and killing the connection if no response is obtained from the " - "other end. Ignored for Unix domain sockets." + "other end. Ignored for Unix domain sockets.", + /* flags */ XYZZY ) PARAM( @@ -1039,11 +1005,11 @@ PARAM( /* max */ "7200", /* def */ "", /* units */ "seconds", - /* flags */ XYZZY, /* descr */ "The number of seconds a connection needs to be idle before TCP " "begins sending out keep-alive probes. " - "Ignored for Unix domain sockets." + "Ignored for Unix domain sockets.", + /* flags */ XYZZY ) #undef XYZZY @@ -1056,7 +1022,6 @@ PARAM( /* max */ NULL, /* def */ "0.000", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "Wait at least this long after creating a thread.\n" "\n" @@ -1064,7 +1029,8 @@ PARAM( "creating threads.\n" "Set this to a few milliseconds if you see the 'threads_failed' " "counter grow too much.\n" - "Setting this too high results in insufficient worker threads." + "Setting this too high results in insufficient worker threads.", + /* flags */ EXPERIMENTAL ) /* actual location mgt_pool.c */ @@ -1075,12 +1041,12 @@ PARAM( /* max */ NULL, /* def */ "60.000", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "Thread queue stuck watchdog.\n" "\n" "If no queued work have been released for this long," - " the worker process panics itself." + " the worker process panics itself.", + /* flags */ EXPERIMENTAL ) /* actual location mgt_pool.c */ @@ -1091,10 +1057,10 @@ PARAM( /* max */ NULL, /* def */ "1.000", /* units */ "seconds", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL, /* descr */ "Wait this long after destroying a thread.\n" - "This controls the decay of thread pools when idle(-ish)." + "This controls the decay of thread pools when idle(-ish).", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL ) /* actual location mgt_pool.c */ @@ -1105,7 +1071,6 @@ PARAM( /* max */ NULL, /* def */ "0.200", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "Wait at least this long after a failed thread creation before " "trying to create another thread.\n" @@ -1119,7 +1084,8 @@ PARAM( "\n" "It may also help to increase thread_pool_timeout and " "thread_pool_min, to reduce the rate at which treads are destroyed " - "and later recreated." + "and later recreated.", + /* flags */ EXPERIMENTAL ) /* actual location mgt_pool.c */ @@ -1130,13 +1096,13 @@ PARAM( /* max */ NULL, /* def */ "5000", /* units */ "threads", - /* flags */ DELAYED_EFFECT, /* descr */ "The maximum number of worker threads in each pool.\n" "\n" "Do not set this higher than you have to, since excess worker " "threads soak up RAM and CPU and generally just get in the way of " - "getting work done." + "getting work done.", + /* flags */ DELAYED_EFFECT ) /* actual location mgt_pool.c */ @@ -1147,13 +1113,13 @@ PARAM( /* max */ "5000", /* def */ "100", /* units */ "threads", - /* flags */ DELAYED_EFFECT, /* descr */ "The minimum number of worker threads in each pool.\n" "\n" "Increasing this may help ramp up faster from low load situations " "or when threads have expired." - "Minimum is 10 threads." + "Minimum is 10 threads.", + /* flags */ DELAYED_EFFECT ) /* actual location mgt_pool.c */ @@ -1164,7 +1130,6 @@ PARAM( /* max */ NULL, /* def */ "0", /* units */ "threads", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL, /* descr */ "The number of worker threads reserved for vital tasks " "in each pool.\n" @@ -1180,7 +1145,8 @@ PARAM( "unused.\n" "\n" "Default is 0 to auto-tune (currently 5% of thread_pool_min).\n" - "Minimum is 1 otherwise, maximum is 95% of thread_pool_min." + "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL ) /* actual location mgt_pool.c */ @@ -1191,11 +1157,11 @@ PARAM( /* max */ NULL, /* def */ "56k", /* units */ "bytes", - /* flags */ EXPERIMENTAL, /* descr */ "Worker thread stack size.\n" "This will likely be rounded up to a multiple of 4k (or whatever " - "the page_size might be) by the kernel." + "the page_size might be) by the kernel.", + /* flags */ EXPERIMENTAL ) /* actual location mgt_pool.c */ @@ -1206,12 +1172,12 @@ PARAM( /* max */ NULL, /* def */ "300.000", /* units */ "seconds", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL, /* descr */ "Thread idle threshold.\n" "\n" "Threads in excess of thread_pool_min, which have been idle for at " - "least this long, will be destroyed." + "least this long, will be destroyed.", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL ) /* actual location mgt_pool.c */ @@ -1222,7 +1188,6 @@ PARAM( /* max */ NULL, /* def */ "2", /* units */ "pools", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL, /* descr */ "Number of worker thread pools.\n" "\n" @@ -1236,7 +1201,8 @@ PARAM( "pool for each CPU is most likely detrimental to performance.\n" "\n" "Can be increased on the fly, but decreases require a restart to " - "take effect." + "take effect.", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL ) /* actual location mgt_pool.c */ @@ -1247,13 +1213,13 @@ PARAM( /* max */ NULL, /* def */ "20", /* units */ NULL, - /* flags */ EXPERIMENTAL, /* descr */ "Permitted request queue length per thread-pool.\n" "\n" "This sets the number of requests we will queue, waiting for an " "available thread. Above this limit sessions will be dropped " - "instead of queued." + "instead of queued.", + /* flags */ EXPERIMENTAL ) /* actual location mgt_pool.c */ @@ -1264,14 +1230,14 @@ PARAM( /* max */ NULL, /* def */ "10", /* units */ "requests", - /* flags */ EXPERIMENTAL, /* descr */ "Worker threads accumulate statistics, and dump these into the " "global stats counters if the lock is free when they finish a job " "(request/fetch etc).\n" "This parameters defines the maximum number of jobs a worker " "thread may handle, before it is forced to dump its accumulated " - "stats into the global counters." + "stats into the global counters.", + /* flags */ EXPERIMENTAL ) #endif @@ -1291,14 +1257,14 @@ PARAM( /* max */ NULL, /* def */ "5.000", /* units */ "seconds", - /* flags */ XYZZY, /* descr */ "Idle timeout for client connections.\n\n" "A connection is considered idle until we have received the full" " request headers.\n\n" "This parameter is particularly relevant for HTTP1 keepalive " " connections which are closed unless the next request is received" - " before this timeout is reached." + " before this timeout is reached.", + /* flags */ XYZZY ) #undef XYZZY @@ -1309,7 +1275,6 @@ PARAM( /* max */ NULL, /* def */ "0.050", /* units */ "seconds", - /* flags */ EXPERIMENTAL, /* descr */ "How long the worker thread lingers on an idle session before " "handing it over to the waiter.\n" @@ -1317,7 +1282,8 @@ PARAM( "within the first 100 msec of the previous request completing.\n" "Setting this too high results in worker threads not doing " "anything for their keep, setting it too low just means that more " - "sessions take a detour around the waiter." + "sessions take a detour around the waiter.", + /* flags */ EXPERIMENTAL ) #if 0 @@ -1329,7 +1295,6 @@ PARAM( /* max */ NULL, /* def */ "off", /* units */ "bool", - /* flags */ 0, /* descr */ "Allow inline C code in VCL." ) @@ -1342,7 +1307,6 @@ PARAM( /* max */ NULL, /* def */ "on", /* units */ "bool", - /* flags */ 0, /* descr */ "Unreferenced VCL objects result in error." ) @@ -1355,7 +1319,6 @@ PARAM( /* max */ NULL, /* def */ "on", /* units */ "bool", - /* flags */ 0, /* descr */ "Allow '/' in vmod & include paths.\n" "Allow 'import ... from ...'." @@ -1369,7 +1332,6 @@ PARAM( /* max */ NULL, /* def */ "600.000", /* units */ "seconds", - /* flags */ 0, /* descr */ "How long a VCL is kept warm after being replaced as the " "active VCL (granularity approximately 30 seconds)." @@ -1382,7 +1344,6 @@ PARAM( /* max */ "2", /* def */ "1", /* units */ NULL, - /* flags */ 0, /* descr */ "Behaviour when attempting to exceed max_vcl loaded VCL.\n" "\n* 0 - Ignore max_vcl parameter.\n" @@ -1397,7 +1358,6 @@ PARAM( /* max */ NULL, /* def */ "100", /* units */ NULL, - /* flags */ 0, /* descr */ "Threshold of loaded VCL programs. (VCL labels are not counted.)" " Parameter max_vcl_handling determines behaviour." @@ -1410,7 +1370,6 @@ PARAM( /* max */ "600.000", /* def */ "60.000", /* units */ "seconds", - /* flags */ 0, /* descr */ "How long VSM memory is kept warm after a deallocation " "(granularity approximately 2 seconds)." @@ -1423,7 +1382,6 @@ PARAM( /* max */ NULL, /* def */ "4k", /* units */ "bytes", - /* flags */ 0, /* descr */ "Bytes of (req-/backend-)workspace dedicated to buffering VSL " "records.\n" @@ -1432,6 +1390,7 @@ PARAM( "Setting this too high costs memory, setting it too low will cause " "more VSL flushes and likely increase lock-contention on the VSL " "mutex.", + /* flags */ 0, /* dyn_min_reason */ "vsl_reclen + 12 bytes" ) @@ -1444,7 +1403,6 @@ PARAM( /* max */ NULL, /* def */ "default", /* units */ NULL, - /* flags */ 0, /* descr */ "Mask individual VSL messages from being logged.\n" " default Set default value\n" @@ -1461,9 +1419,9 @@ PARAM( /* max */ NULL, /* def */ "255b", /* units */ "bytes", - /* flags */ 0, /* descr */ "Maximum number of bytes in SHM log record.", + /* flags */ 0, /* dyn_min_reason */ NULL, /* dyn_max_reason */ "vsl_buffer - 12 bytes" ) @@ -1475,12 +1433,12 @@ PARAM( /* max */ "4G", /* def */ "80M", /* units */ "bytes", - /* flags */ MUST_RESTART, /* descr */ "The amount of space to allocate for the VSL fifo buffer in the " "VSM memory segment. If you make this too small, " "varnish{ncsa|log} etc will not be able to keep up. Making it too " - "large just costs memory resources." + "large just costs memory resources.", + /* flags */ MUST_RESTART ) PARAM( @@ -1490,7 +1448,6 @@ PARAM( /* max */ "1G", /* def */ "1M", /* units */ "bytes", - /* flags */ 0, /* descr */ "DEPRECATED: This parameter is ignored.\n" "There is no global limit on amount of shared memory now." @@ -1505,9 +1462,9 @@ PARAM( /* max */ NULL, /* def */ "kqueue (possible values: kqueue, poll)", /* units */ NULL, - /* flags */ MUST_RESTART| WIZARD, /* descr */ - "Select the waiter kernel interface." + "Select the waiter kernel interface.", + /* flags */ MUST_RESTART| WIZARD ) #endif @@ -1518,10 +1475,10 @@ PARAM( /* max */ NULL, /* def */ "64k", /* units */ "bytes", - /* flags */ DELAYED_EFFECT, /* descr */ "Bytes of HTTP protocol workspace for backend HTTP req/resp. If " - "larger than 4k, use a multiple of 4k for VM efficiency." + "larger than 4k, use a multiple of 4k for VM efficiency.", + /* flags */ DELAYED_EFFECT ) PARAM( @@ -1531,14 +1488,14 @@ PARAM( /* max */ NULL, /* def */ "64k", /* units */ "bytes", - /* flags */ DELAYED_EFFECT, /* descr */ "Bytes of HTTP protocol workspace for clients HTTP req/resp. Use a " "multiple of 4k for VM efficiency.\n" "For HTTP/2 compliance this must be at least 20k, in order to " "receive fullsize (=16k) frames from the client. That usually " "happens only in POST/PUT bodies. For other traffic-patterns " - "smaller values work just fine." + "smaller values work just fine.", + /* flags */ DELAYED_EFFECT ) PARAM( @@ -1548,11 +1505,11 @@ PARAM( /* max */ NULL, /* def */ "0.75k", /* units */ "bytes", - /* flags */ DELAYED_EFFECT, /* descr */ "Allocation size for session structure and workspace. The " "workspace is primarily used for TCP connection addresses. If " - "larger than 4k, use a multiple of 4k for VM efficiency." + "larger than 4k, use a multiple of 4k for VM efficiency.", + /* flags */ DELAYED_EFFECT ) PARAM( @@ -1562,7 +1519,6 @@ PARAM( /* max */ "8k", /* def */ "2k", /* units */ "bytes", - /* flags */ DELAYED_EFFECT, /* descr */ "Bytes of auxiliary workspace per thread.\n" "This workspace is used for certain temporary data structures " @@ -1572,7 +1528,8 @@ PARAM( "syscalls, setting it too high just wastes space. ~0.1k + " "UIO_MAXIOV * sizeof(struct iovec) (typically = ~16k for 64bit) " "is considered the maximum sensible value under any known " - "circumstances (excluding exotic vmod use)." + "circumstances (excluding exotic vmod use).", + /* flags */ DELAYED_EFFECT ) PARAM( @@ -1582,11 +1539,11 @@ PARAM( /* max */ "1G", /* def */ "10M", /* units */ "bytes", - /* flags */ WIZARD, /* descr */ "HTTP2 Receive Window low water mark.\n" "We try to keep the window at least this big\n" - "Only affects incoming request bodies (ie: POST, PUT etc.)" + "Only affects incoming request bodies (ie: POST, PUT etc.)", + /* flags */ WIZARD ) PARAM( @@ -1596,11 +1553,11 @@ PARAM( /* max */ "1G", /* def */ "1M", /* units */ "bytes", - /* flags */ WIZARD, /* descr */ "HTTP2 Receive Window Increments.\n" "How big credits we send in WINDOW_UPDATE frames\n" - "Only affects incoming request bodies (ie: POST, PUT etc.)" + "Only affects incoming request bodies (ie: POST, PUT etc.)", + /* flags */ WIZARD ) PARAM( @@ -1610,7 +1567,6 @@ PARAM( /* max */ NULL, /* def */ "4k", /* units */ "bytes", - /* flags */ 0, /* descr */ "HTTP2 header table size.\n" "This is the size that will be used for the HPACK dynamic\n" @@ -1624,7 +1580,6 @@ PARAM( /* max */ NULL, /* def */ "100", /* units */ "streams", - /* flags */ 0, /* descr */ "HTTP2 Maximum number of concurrent streams.\n" "This is the number of requests that can be active\n" @@ -1638,7 +1593,6 @@ PARAM( /* max */ "2147483647b", /* def */ "65535b", /* units */ "bytes", - /* flags */ 0, /* descr */ "HTTP2 initial flow control window size." ) @@ -1650,7 +1604,6 @@ PARAM( /* max */ "16777215b", /* def */ "16k", /* units */ "bytes", - /* flags */ 0, /* descr */ "HTTP2 maximum per frame payload size we are willing to accept." ) @@ -1662,7 +1615,6 @@ PARAM( /* max */ NULL, /* def */ "2147483647b", /* units */ "bytes", - /* flags */ 0, /* descr */ "HTTP2 maximum size of an uncompressed header list." ) From dridi.boukelmoune at gmail.com Mon Mar 2 14:09:09 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 14:09:09 +0000 (UTC) Subject: [master] 53efadd03 Retire the waiter parameter for good Message-ID: <20200302140909.620DEABC01@lists.varnish-cache.org> commit 53efadd03be31979c6087d062d4a4c44cdc7929b Author: Dridi Boukelmoune Date: Mon Mar 2 11:42:52 2020 +0100 Retire the waiter parameter for good It was already gone for the 4.1.0 release, replaced by the varnishd -W option. diff --git a/include/tbl/params.h b/include/tbl/params.h index d115531f1..2c2b3e83b 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1453,21 +1453,6 @@ PARAM( "There is no global limit on amount of shared memory now." ) -#if 0 -/* see mgt_waiter.c */ -PARAM( - /* name */ waiter, - /* type */ waiter, - /* min */ NULL, - /* max */ NULL, - /* def */ "kqueue (possible values: kqueue, poll)", - /* units */ NULL, - /* descr */ - "Select the waiter kernel interface.", - /* flags */ MUST_RESTART| WIZARD -) -#endif - PARAM( /* name */ workspace_backend, /* type */ bytes_u, From dridi.boukelmoune at gmail.com Mon Mar 2 14:09:09 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 14:09:09 +0000 (UTC) Subject: [master] 8d63eb9fa Group impotent PARAM() macros at the end of the table Message-ID: <20200302140909.7AB22ABC07@lists.varnish-cache.org> commit 8d63eb9fa74df799867ccc7133276834c788651a Author: Dridi Boukelmoune Date: Mon Mar 2 11:48:19 2020 +0100 Group impotent PARAM() macros at the end of the table I have a plan to help them fit in the table, and regrouping them will make matters simpler. Much better diff with the --patience option. diff --git a/include/tbl/params.h b/include/tbl/params.h index 2c2b3e83b..cf07a1052 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -369,35 +369,6 @@ PARAM( /* flags */ WIZARD ) -#if 0 -/* actual location mgt_param_bits.c*/ -/* see tbl/debug_bits.h */ -PARAM( - /* name */ debug, - /* type */ debug, - /* min */ NULL, - /* max */ NULL, - /* def */ NULL, - /* units */ NULL, - /* descr */ - "Enable/Disable various kinds of debugging.\n" - " none Disable all debugging\n" - "\n" - "Use +/- prefix to set/reset individual bits:\n" - " req_state VSL Request state engine\n" - " workspace VSL Workspace operations\n" - " waiter VSL Waiter internals\n" - " waitinglist VSL Waitinglist events\n" - " syncvsl Make VSL synchronous\n" - " hashedge Edge cases in Hash\n" - " vclrel Rapid VCL release\n" - " lurker VSL Ban lurker\n" - " esi_chop Chop ESI fetch to bits\n" - " flush_head Flush after http1 head\n" - " vtc_mode Varnishtest Mode" -) -#endif - PARAM( /* name */ default_grace, /* type */ timeout, @@ -454,31 +425,6 @@ PARAM( /* flags */ WIZARD ) -#if 0 -/* actual location mgt_param_bits.c*/ -/* See tbl/feature_bits.h */ -PARAM( - /* name */ feature, - /* type */ feature, - /* min */ NULL, - /* max */ NULL, - /* def */ NULL, - /* units */ NULL, - /* descr */ - "Enable/Disable various minor features.\n" - " none Disable all features.\n" - "\n" - "Use +/- prefix to enable/disable individual feature:\n" - " short_panic Short panic message.\n" - " wait_silo Wait for persistent silo.\n" - " no_coredump No coredumps.\n" - " esi_ignore_https Treat HTTPS as HTTP in ESI:includes\n" - " esi_disable_xml_check Don't check of body looks like XML\n" - " esi_ignore_other_elements Ignore non-esi XML-elements\n" - " esi_remove_bom Remove UTF-8 BOM" -) -#endif - PARAM( /* name */ fetch_chunksize, /* type */ bytes, @@ -752,34 +698,6 @@ PARAM( /* flags */ EXPERIMENTAL ) -#if 0 -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pcre_match_limit, - /* type */ uint, - /* min */ "1", - /* max */ NULL, - /* def */ "1.000", - /* units */ NULL, - /* descr */ - "The limit for the number of internal matching function calls in " - "a pcre_exec() execution." -) - -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pcre_match_limit_recursion, - /* type */ uint, - /* min */ "1", - /* max */ NULL, - /* def */ "1.000", - /* units */ NULL, - /* descr */ - "The limit for the number of internal matching function " - "recursions in a pcre_exec() execution." -) -#endif - PARAM( /* name */ ping_interval, /* type */ uint, @@ -817,47 +735,6 @@ PARAM( "either direction for this many seconds, the session is closed." ) -#if 0 -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pool_req, - /* type */ poolparam, - /* min */ NULL, - /* max */ NULL, - /* def */ "10,100,10", - /* units */ NULL, - /* descr */ - "Parameters for per worker pool request memory pool.\n" - MEMPOOL_TEXT -) - -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pool_sess, - /* type */ poolparam, - /* min */ NULL, - /* max */ NULL, - /* def */ "10,100,10", - /* units */ NULL, - /* descr */ - "Parameters for per worker pool session memory pool.\n" - MEMPOOL_TEXT -) - -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pool_vbo, - /* type */ poolparam, - /* min */ NULL, - /* max */ NULL, - /* def */ "10,100,10", - /* units */ NULL, - /* descr */ - "Parameters for backend object fetch memory pool.\n" - MEMPOOL_TEXT -) -#endif - PARAM( /* name */ prefer_ipv6, /* type */ bool, @@ -1013,596 +890,707 @@ PARAM( ) #undef XYZZY -#if 0 -/* actual location mgt_pool.c */ +#if defined(XYZZY) + #error "Temporary macro XYZZY already defined" +#endif + +#if defined(SO_RCVTIMEO_WORKS) + #define XYZZY 0 +#else + #define XYZZY NOT_IMPLEMENTED +#endif PARAM( - /* name */ thread_pool_add_delay, + /* name */ timeout_idle, /* type */ timeout, /* min */ "0.000", /* max */ NULL, - /* def */ "0.000", + /* def */ "5.000", /* units */ "seconds", /* descr */ - "Wait at least this long after creating a thread.\n" - "\n" - "Some (buggy) systems may need a short (sub-second) delay between " - "creating threads.\n" - "Set this to a few milliseconds if you see the 'threads_failed' " - "counter grow too much.\n" - "Setting this too high results in insufficient worker threads.", - /* flags */ EXPERIMENTAL + "Idle timeout for client connections.\n\n" + "A connection is considered idle until we have received the full" + " request headers.\n\n" + "This parameter is particularly relevant for HTTP1 keepalive " + " connections which are closed unless the next request is received" + " before this timeout is reached.", + /* flags */ XYZZY ) +#undef XYZZY -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_pool_watchdog, + /* name */ timeout_linger, /* type */ timeout, - /* min */ "0.1", + /* min */ "0.000", /* max */ NULL, - /* def */ "60.000", + /* def */ "0.050", /* units */ "seconds", /* descr */ - "Thread queue stuck watchdog.\n" - "\n" - "If no queued work have been released for this long," - " the worker process panics itself.", + "How long the worker thread lingers on an idle session before " + "handing it over to the waiter.\n" + "When sessions are reused, as much as half of all reuses happen " + "within the first 100 msec of the previous request completing.\n" + "Setting this too high results in worker threads not doing " + "anything for their keep, setting it too low just means that more " + "sessions take a detour around the waiter.", /* flags */ EXPERIMENTAL ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_pool_destroy_delay, + /* name */ vcl_cooldown, /* type */ timeout, - /* min */ "0.010", + /* min */ "1.000", /* max */ NULL, - /* def */ "1.000", + /* def */ "600.000", /* units */ "seconds", /* descr */ - "Wait this long after destroying a thread.\n" - "This controls the decay of thread pools when idle(-ish).", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL + "How long a VCL is kept warm after being replaced as the " + "active VCL (granularity approximately 30 seconds)." ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_pool_fail_delay, - /* type */ timeout, - /* min */ "0.010", - /* max */ NULL, - /* def */ "0.200", - /* units */ "seconds", + /* name */ max_vcl_handling, + /* type */ uint, + /* min */ "0", + /* max */ "2", + /* def */ "1", + /* units */ NULL, /* descr */ - "Wait at least this long after a failed thread creation before " - "trying to create another thread.\n" - "\n" - "Failure to create a worker thread is often a sign that the end " - "is near, because the process is running out of some resource. " - "This delay tries to not rush the end on needlessly.\n" - "\n" - "If thread creation failures are a problem, check that " - "thread_pool_max is not too high.\n" - "\n" - "It may also help to increase thread_pool_timeout and " - "thread_pool_min, to reduce the rate at which treads are destroyed " - "and later recreated.", - /* flags */ EXPERIMENTAL + "Behaviour when attempting to exceed max_vcl loaded VCL.\n" + "\n* 0 - Ignore max_vcl parameter.\n" + "\n* 1 - Issue warning.\n" + "\n* 2 - Refuse loading VCLs." ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_pool_max, - /* type */ thread_pool_max, - /* min */ "100", + /* name */ max_vcl, + /* type */ uint, + /* min */ "0", /* max */ NULL, - /* def */ "5000", - /* units */ "threads", + /* def */ "100", + /* units */ NULL, /* descr */ - "The maximum number of worker threads in each pool.\n" - "\n" - "Do not set this higher than you have to, since excess worker " - "threads soak up RAM and CPU and generally just get in the way of " - "getting work done.", - /* flags */ DELAYED_EFFECT + "Threshold of loaded VCL programs. (VCL labels are not counted.)" + " Parameter max_vcl_handling determines behaviour." ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_pool_min, - /* type */ thread_pool_min, - /* min */ NULL, - /* max */ "5000", - /* def */ "100", - /* units */ "threads", + /* name */ vsm_free_cooldown, + /* type */ timeout, + /* min */ "10.000", + /* max */ "600.000", + /* def */ "60.000", + /* units */ "seconds", /* descr */ - "The minimum number of worker threads in each pool.\n" - "\n" - "Increasing this may help ramp up faster from low load situations " - "or when threads have expired." - "Minimum is 10 threads.", - /* flags */ DELAYED_EFFECT + "How long VSM memory is kept warm after a deallocation " + "(granularity approximately 2 seconds)." ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_pool_reserve, - /* type */ thread_pool_reserve, - /* min */ NULL, + /* name */ vsl_buffer, + /* type */ vsl_buffer, + /* min */ "267", /* max */ NULL, - /* def */ "0", - /* units */ "threads", + /* def */ "4k", + /* units */ "bytes", /* descr */ - "The number of worker threads reserved for vital tasks " - "in each pool.\n" - "\n" - "Tasks may require other tasks to complete (for example, " - "client requests may require backend requests). This reserve " - "is to ensure that such tasks still get to run even under high " - "load.\n" - "\n" - "Increasing the reserve may help setups with a high number of " - "backend requests at the expense of client performance. " - "Setting it too high will waste resources by keeping threads " - "unused.\n" - "\n" - "Default is 0 to auto-tune (currently 5% of thread_pool_min).\n" - "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL + "Bytes of (req-/backend-)workspace dedicated to buffering VSL " + "records.\n" + "When this parameter is adjusted, most likely workspace_client " + "and workspace_backend will have to be adjusted by the same amount.\n\n" + "Setting this too high costs memory, setting it too low will cause " + "more VSL flushes and likely increase lock-contention on the VSL " + "mutex.", + /* flags */ 0, + /* dyn_min_reason */ "vsl_reclen + 12 bytes" ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_pool_stack, - /* type */ bytes, - /* min */ "2k", + /* name */ vsl_reclen, + /* type */ vsl_reclen, + /* min */ "16b", /* max */ NULL, - /* def */ "56k", + /* def */ "255b", /* units */ "bytes", /* descr */ - "Worker thread stack size.\n" - "This will likely be rounded up to a multiple of 4k (or whatever " - "the page_size might be) by the kernel.", - /* flags */ EXPERIMENTAL + "Maximum number of bytes in SHM log record.", + /* flags */ 0, + /* dyn_min_reason */ NULL, + /* dyn_max_reason */ "vsl_buffer - 12 bytes" ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_pool_timeout, - /* type */ timeout, - /* min */ "10.000", - /* max */ NULL, - /* def */ "300.000", - /* units */ "seconds", + /* name */ vsl_space, + /* type */ bytes, + /* min */ "1M", + /* max */ "4G", + /* def */ "80M", + /* units */ "bytes", /* descr */ - "Thread idle threshold.\n" - "\n" - "Threads in excess of thread_pool_min, which have been idle for at " - "least this long, will be destroyed.", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL + "The amount of space to allocate for the VSL fifo buffer in the " + "VSM memory segment. If you make this too small, " + "varnish{ncsa|log} etc will not be able to keep up. Making it too " + "large just costs memory resources.", + /* flags */ MUST_RESTART ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_pools, - /* type */ uint, - /* min */ "1", + /* name */ vsm_space, + /* type */ bytes, + /* min */ "1M", + /* max */ "1G", + /* def */ "1M", + /* units */ "bytes", + /* descr */ + "DEPRECATED: This parameter is ignored.\n" + "There is no global limit on amount of shared memory now." +) + +PARAM( + /* name */ workspace_backend, + /* type */ bytes_u, + /* min */ "1k", /* max */ NULL, - /* def */ "2", - /* units */ "pools", + /* def */ "64k", + /* units */ "bytes", /* descr */ - "Number of worker thread pools.\n" - "\n" - "Increasing the number of worker pools decreases lock " - "contention. Each worker pool also has a thread accepting " - "new connections, so for very high rates of incoming new " - "connections on systems with many cores, increasing the " - "worker pools may be required.\n" - "\n" - "Too many pools waste CPU and RAM resources, and more than one " - "pool for each CPU is most likely detrimental to performance.\n" - "\n" - "Can be increased on the fly, but decreases require a restart to " - "take effect.", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL + "Bytes of HTTP protocol workspace for backend HTTP req/resp. If " + "larger than 4k, use a multiple of 4k for VM efficiency.", + /* flags */ DELAYED_EFFECT ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_queue_limit, - /* type */ uint, - /* min */ "0", + /* name */ workspace_client, + /* type */ bytes_u, + /* min */ "9k", /* max */ NULL, - /* def */ "20", - /* units */ NULL, + /* def */ "64k", + /* units */ "bytes", /* descr */ - "Permitted request queue length per thread-pool.\n" - "\n" - "This sets the number of requests we will queue, waiting for an " - "available thread. Above this limit sessions will be dropped " - "instead of queued.", - /* flags */ EXPERIMENTAL + "Bytes of HTTP protocol workspace for clients HTTP req/resp. Use a " + "multiple of 4k for VM efficiency.\n" + "For HTTP/2 compliance this must be at least 20k, in order to " + "receive fullsize (=16k) frames from the client. That usually " + "happens only in POST/PUT bodies. For other traffic-patterns " + "smaller values work just fine.", + /* flags */ DELAYED_EFFECT ) -/* actual location mgt_pool.c */ PARAM( - /* name */ thread_stats_rate, - /* type */ uint, - /* min */ "0", + /* name */ workspace_session, + /* type */ bytes_u, + /* min */ "0.25k", /* max */ NULL, - /* def */ "10", - /* units */ "requests", + /* def */ "0.75k", + /* units */ "bytes", /* descr */ - "Worker threads accumulate statistics, and dump these into the " - "global stats counters if the lock is free when they finish a job " - "(request/fetch etc).\n" - "This parameters defines the maximum number of jobs a worker " - "thread may handle, before it is forced to dump its accumulated " - "stats into the global counters.", - /* flags */ EXPERIMENTAL + "Allocation size for session structure and workspace. The " + "workspace is primarily used for TCP connection addresses. If " + "larger than 4k, use a multiple of 4k for VM efficiency.", + /* flags */ DELAYED_EFFECT ) -#endif -#if defined(XYZZY) - #error "Temporary macro XYZZY already defined" -#endif +PARAM( + /* name */ workspace_thread, + /* type */ bytes_u, + /* min */ "0.25k", + /* max */ "8k", + /* def */ "2k", + /* units */ "bytes", + /* descr */ + "Bytes of auxiliary workspace per thread.\n" + "This workspace is used for certain temporary data structures " + "during the operation of a worker thread.\n" + "One use is for the IO-vectors used during delivery. Setting " + "this parameter too low may increase the number of writev() " + "syscalls, setting it too high just wastes space. ~0.1k + " + "UIO_MAXIOV * sizeof(struct iovec) (typically = ~16k for 64bit) " + "is considered the maximum sensible value under any known " + "circumstances (excluding exotic vmod use).", + /* flags */ DELAYED_EFFECT +) -#if defined(SO_RCVTIMEO_WORKS) - #define XYZZY 0 -#else - #define XYZZY NOT_IMPLEMENTED -#endif PARAM( - /* name */ timeout_idle, - /* type */ timeout, - /* min */ "0.000", + /* name */ h2_rx_window_low_water, + /* type */ bytes_u, + /* min */ "65535", + /* max */ "1G", + /* def */ "10M", + /* units */ "bytes", + /* descr */ + "HTTP2 Receive Window low water mark.\n" + "We try to keep the window at least this big\n" + "Only affects incoming request bodies (ie: POST, PUT etc.)", + /* flags */ WIZARD +) + +PARAM( + /* name */ h2_rx_window_increment, + /* type */ bytes_u, + /* min */ "1M", + /* max */ "1G", + /* def */ "1M", + /* units */ "bytes", + /* descr */ + "HTTP2 Receive Window Increments.\n" + "How big credits we send in WINDOW_UPDATE frames\n" + "Only affects incoming request bodies (ie: POST, PUT etc.)", + /* flags */ WIZARD +) + +PARAM( + /* name */ h2_header_table_size, + /* type */ bytes_u, + /* min */ "0b", /* max */ NULL, - /* def */ "5.000", - /* units */ "seconds", + /* def */ "4k", + /* units */ "bytes", /* descr */ - "Idle timeout for client connections.\n\n" - "A connection is considered idle until we have received the full" - " request headers.\n\n" - "This parameter is particularly relevant for HTTP1 keepalive " - " connections which are closed unless the next request is received" - " before this timeout is reached.", - /* flags */ XYZZY + "HTTP2 header table size.\n" + "This is the size that will be used for the HPACK dynamic\n" + "decoding table." ) -#undef XYZZY PARAM( - /* name */ timeout_linger, - /* type */ timeout, - /* min */ "0.000", + /* name */ h2_max_concurrent_streams, + /* type */ uint, + /* min */ "0", /* max */ NULL, - /* def */ "0.050", - /* units */ "seconds", + /* def */ "100", + /* units */ "streams", + /* descr */ + "HTTP2 Maximum number of concurrent streams.\n" + "This is the number of requests that can be active\n" + "at the same time for a single HTTP2 connection." +) + +PARAM( + /* name */ h2_initial_window_size, + /* type */ bytes_u, + /* min */ "0", + /* max */ "2147483647b", + /* def */ "65535b", + /* units */ "bytes", /* descr */ - "How long the worker thread lingers on an idle session before " - "handing it over to the waiter.\n" - "When sessions are reused, as much as half of all reuses happen " - "within the first 100 msec of the previous request completing.\n" - "Setting this too high results in worker threads not doing " - "anything for their keep, setting it too low just means that more " - "sessions take a detour around the waiter.", - /* flags */ EXPERIMENTAL + "HTTP2 initial flow control window size." ) -#if 0 -/* actual location mgt_param_tbl.c */ PARAM( - /* name */ vcc_allow_inline_c, - /* type */ bool, - /* min */ NULL, - /* max */ NULL, - /* def */ "off", - /* units */ "bool", + /* name */ h2_max_frame_size, + /* type */ bytes_u, + /* min */ "16k", + /* max */ "16777215b", + /* def */ "16k", + /* units */ "bytes", /* descr */ - "Allow inline C code in VCL." + "HTTP2 maximum per frame payload size we are willing to accept." ) -/* actual location mgt_param_tbl.c */ PARAM( - /* name */ vcc_err_unref, - /* type */ bool, - /* min */ NULL, + /* name */ h2_max_header_list_size, + /* type */ bytes_u, + /* min */ "0b", /* max */ NULL, - /* def */ "on", - /* units */ "bool", + /* def */ "2147483647b", + /* units */ "bytes", /* descr */ - "Unreferenced VCL objects result in error." + "HTTP2 maximum size of an uncompressed header list." ) -/* actual location mgt_param_tbl.c */ +#if 0 +/* actual location mgt_param_bits.c*/ +/* see tbl/debug_bits.h */ PARAM( - /* name */ vcc_unsafe_path, - /* type */ bool, + /* name */ debug, + /* type */ debug, /* min */ NULL, /* max */ NULL, - /* def */ "on", - /* units */ "bool", + /* def */ NULL, + /* units */ NULL, /* descr */ - "Allow '/' in vmod & include paths.\n" - "Allow 'import ... from ...'." + "Enable/Disable various kinds of debugging.\n" + " none Disable all debugging\n" + "\n" + "Use +/- prefix to set/reset individual bits:\n" + " req_state VSL Request state engine\n" + " workspace VSL Workspace operations\n" + " waiter VSL Waiter internals\n" + " waitinglist VSL Waitinglist events\n" + " syncvsl Make VSL synchronous\n" + " hashedge Edge cases in Hash\n" + " vclrel Rapid VCL release\n" + " lurker VSL Ban lurker\n" + " esi_chop Chop ESI fetch to bits\n" + " flush_head Flush after http1 head\n" + " vtc_mode Varnishtest Mode" ) -#endif +/* actual location mgt_param_bits.c*/ +/* See tbl/feature_bits.h */ PARAM( - /* name */ vcl_cooldown, - /* type */ timeout, - /* min */ "1.000", + /* name */ feature, + /* type */ feature, + /* min */ NULL, /* max */ NULL, - /* def */ "600.000", - /* units */ "seconds", + /* def */ NULL, + /* units */ NULL, /* descr */ - "How long a VCL is kept warm after being replaced as the " - "active VCL (granularity approximately 30 seconds)." + "Enable/Disable various minor features.\n" + " none Disable all features.\n" + "\n" + "Use +/- prefix to enable/disable individual feature:\n" + " short_panic Short panic message.\n" + " wait_silo Wait for persistent silo.\n" + " no_coredump No coredumps.\n" + " esi_ignore_https Treat HTTPS as HTTP in ESI:includes\n" + " esi_disable_xml_check Don't check of body looks like XML\n" + " esi_ignore_other_elements Ignore non-esi XML-elements\n" + " esi_remove_bom Remove UTF-8 BOM" ) +/* actual location mgt_param_tbl.c */ PARAM( - /* name */ max_vcl_handling, + /* name */ pcre_match_limit, /* type */ uint, - /* min */ "0", - /* max */ "2", - /* def */ "1", + /* min */ "1", + /* max */ NULL, + /* def */ "1.000", /* units */ NULL, /* descr */ - "Behaviour when attempting to exceed max_vcl loaded VCL.\n" - "\n* 0 - Ignore max_vcl parameter.\n" - "\n* 1 - Issue warning.\n" - "\n* 2 - Refuse loading VCLs." + "The limit for the number of internal matching function calls in " + "a pcre_exec() execution." ) +/* actual location mgt_param_tbl.c */ PARAM( - /* name */ max_vcl, + /* name */ pcre_match_limit_recursion, /* type */ uint, - /* min */ "0", + /* min */ "1", /* max */ NULL, - /* def */ "100", + /* def */ "1.000", /* units */ NULL, /* descr */ - "Threshold of loaded VCL programs. (VCL labels are not counted.)" - " Parameter max_vcl_handling determines behaviour." + "The limit for the number of internal matching function " + "recursions in a pcre_exec() execution." ) +/* actual location mgt_param_tbl.c */ PARAM( - /* name */ vsm_free_cooldown, - /* type */ timeout, - /* min */ "10.000", - /* max */ "600.000", - /* def */ "60.000", - /* units */ "seconds", + /* name */ pool_req, + /* type */ poolparam, + /* min */ NULL, + /* max */ NULL, + /* def */ "10,100,10", + /* units */ NULL, /* descr */ - "How long VSM memory is kept warm after a deallocation " - "(granularity approximately 2 seconds)." + "Parameters for per worker pool request memory pool.\n" + MEMPOOL_TEXT ) +/* actual location mgt_param_tbl.c */ PARAM( - /* name */ vsl_buffer, - /* type */ vsl_buffer, - /* min */ "267", + /* name */ pool_sess, + /* type */ poolparam, + /* min */ NULL, /* max */ NULL, - /* def */ "4k", - /* units */ "bytes", + /* def */ "10,100,10", + /* units */ NULL, /* descr */ - "Bytes of (req-/backend-)workspace dedicated to buffering VSL " - "records.\n" - "When this parameter is adjusted, most likely workspace_client " - "and workspace_backend will have to be adjusted by the same amount.\n\n" - "Setting this too high costs memory, setting it too low will cause " - "more VSL flushes and likely increase lock-contention on the VSL " - "mutex.", - /* flags */ 0, - /* dyn_min_reason */ "vsl_reclen + 12 bytes" + "Parameters for per worker pool session memory pool.\n" + MEMPOOL_TEXT ) -#if 0 -/* actual location mgt_param_bits.c*/ +/* actual location mgt_param_tbl.c */ PARAM( - /* name */ vsl_mask, - /* type */ vsl_mask, + /* name */ pool_vbo, + /* type */ poolparam, /* min */ NULL, /* max */ NULL, - /* def */ "default", + /* def */ "10,100,10", /* units */ NULL, /* descr */ - "Mask individual VSL messages from being logged.\n" - " default Set default value\n" + "Parameters for backend object fetch memory pool.\n" + MEMPOOL_TEXT +) + +/* actual location mgt_pool.c */ +PARAM( + /* name */ thread_pool_add_delay, + /* type */ timeout, + /* min */ "0.000", + /* max */ NULL, + /* def */ "0.000", + /* units */ "seconds", + /* descr */ + "Wait at least this long after creating a thread.\n" "\n" - "Use +/- prefix in front of VSL tag name to unmask/mask " - "individual VSL messages." + "Some (buggy) systems may need a short (sub-second) delay between " + "creating threads.\n" + "Set this to a few milliseconds if you see the 'threads_failed' " + "counter grow too much.\n" + "Setting this too high results in insufficient worker threads.", + /* flags */ EXPERIMENTAL ) -#endif +/* actual location mgt_pool.c */ PARAM( - /* name */ vsl_reclen, - /* type */ vsl_reclen, - /* min */ "16b", + /* name */ thread_pool_watchdog, + /* type */ timeout, + /* min */ "0.1", /* max */ NULL, - /* def */ "255b", - /* units */ "bytes", + /* def */ "60.000", + /* units */ "seconds", /* descr */ - "Maximum number of bytes in SHM log record.", - /* flags */ 0, - /* dyn_min_reason */ NULL, - /* dyn_max_reason */ "vsl_buffer - 12 bytes" + "Thread queue stuck watchdog.\n" + "\n" + "If no queued work have been released for this long," + " the worker process panics itself.", + /* flags */ EXPERIMENTAL ) +/* actual location mgt_pool.c */ PARAM( - /* name */ vsl_space, - /* type */ bytes, - /* min */ "1M", - /* max */ "4G", - /* def */ "80M", - /* units */ "bytes", + /* name */ thread_pool_destroy_delay, + /* type */ timeout, + /* min */ "0.010", + /* max */ NULL, + /* def */ "1.000", + /* units */ "seconds", /* descr */ - "The amount of space to allocate for the VSL fifo buffer in the " - "VSM memory segment. If you make this too small, " - "varnish{ncsa|log} etc will not be able to keep up. Making it too " - "large just costs memory resources.", - /* flags */ MUST_RESTART + "Wait this long after destroying a thread.\n" + "This controls the decay of thread pools when idle(-ish).", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL ) +/* actual location mgt_pool.c */ PARAM( - /* name */ vsm_space, - /* type */ bytes, - /* min */ "1M", - /* max */ "1G", - /* def */ "1M", - /* units */ "bytes", + /* name */ thread_pool_fail_delay, + /* type */ timeout, + /* min */ "0.010", + /* max */ NULL, + /* def */ "0.200", + /* units */ "seconds", /* descr */ - "DEPRECATED: This parameter is ignored.\n" - "There is no global limit on amount of shared memory now." + "Wait at least this long after a failed thread creation before " + "trying to create another thread.\n" + "\n" + "Failure to create a worker thread is often a sign that the end " + "is near, because the process is running out of some resource. " + "This delay tries to not rush the end on needlessly.\n" + "\n" + "If thread creation failures are a problem, check that " + "thread_pool_max is not too high.\n" + "\n" + "It may also help to increase thread_pool_timeout and " + "thread_pool_min, to reduce the rate at which treads are destroyed " + "and later recreated.", + /* flags */ EXPERIMENTAL ) +/* actual location mgt_pool.c */ PARAM( - /* name */ workspace_backend, - /* type */ bytes_u, - /* min */ "1k", + /* name */ thread_pool_max, + /* type */ thread_pool_max, + /* min */ "100", /* max */ NULL, - /* def */ "64k", - /* units */ "bytes", + /* def */ "5000", + /* units */ "threads", /* descr */ - "Bytes of HTTP protocol workspace for backend HTTP req/resp. If " - "larger than 4k, use a multiple of 4k for VM efficiency.", + "The maximum number of worker threads in each pool.\n" + "\n" + "Do not set this higher than you have to, since excess worker " + "threads soak up RAM and CPU and generally just get in the way of " + "getting work done.", /* flags */ DELAYED_EFFECT ) +/* actual location mgt_pool.c */ PARAM( - /* name */ workspace_client, - /* type */ bytes_u, - /* min */ "9k", - /* max */ NULL, - /* def */ "64k", - /* units */ "bytes", + /* name */ thread_pool_min, + /* type */ thread_pool_min, + /* min */ NULL, + /* max */ "5000", + /* def */ "100", + /* units */ "threads", /* descr */ - "Bytes of HTTP protocol workspace for clients HTTP req/resp. Use a " - "multiple of 4k for VM efficiency.\n" - "For HTTP/2 compliance this must be at least 20k, in order to " - "receive fullsize (=16k) frames from the client. That usually " - "happens only in POST/PUT bodies. For other traffic-patterns " - "smaller values work just fine.", + "The minimum number of worker threads in each pool.\n" + "\n" + "Increasing this may help ramp up faster from low load situations " + "or when threads have expired." + "Minimum is 10 threads.", /* flags */ DELAYED_EFFECT ) +/* actual location mgt_pool.c */ PARAM( - /* name */ workspace_session, - /* type */ bytes_u, - /* min */ "0.25k", + /* name */ thread_pool_reserve, + /* type */ thread_pool_reserve, + /* min */ NULL, /* max */ NULL, - /* def */ "0.75k", - /* units */ "bytes", + /* def */ "0", + /* units */ "threads", /* descr */ - "Allocation size for session structure and workspace. The " - "workspace is primarily used for TCP connection addresses. If " - "larger than 4k, use a multiple of 4k for VM efficiency.", - /* flags */ DELAYED_EFFECT + "The number of worker threads reserved for vital tasks " + "in each pool.\n" + "\n" + "Tasks may require other tasks to complete (for example, " + "client requests may require backend requests). This reserve " + "is to ensure that such tasks still get to run even under high " + "load.\n" + "\n" + "Increasing the reserve may help setups with a high number of " + "backend requests at the expense of client performance. " + "Setting it too high will waste resources by keeping threads " + "unused.\n" + "\n" + "Default is 0 to auto-tune (currently 5% of thread_pool_min).\n" + "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL ) +/* actual location mgt_pool.c */ PARAM( - /* name */ workspace_thread, - /* type */ bytes_u, - /* min */ "0.25k", - /* max */ "8k", - /* def */ "2k", + /* name */ thread_pool_stack, + /* type */ bytes, + /* min */ "2k", + /* max */ NULL, + /* def */ "56k", /* units */ "bytes", /* descr */ - "Bytes of auxiliary workspace per thread.\n" - "This workspace is used for certain temporary data structures " - "during the operation of a worker thread.\n" - "One use is for the IO-vectors used during delivery. Setting " - "this parameter too low may increase the number of writev() " - "syscalls, setting it too high just wastes space. ~0.1k + " - "UIO_MAXIOV * sizeof(struct iovec) (typically = ~16k for 64bit) " - "is considered the maximum sensible value under any known " - "circumstances (excluding exotic vmod use).", - /* flags */ DELAYED_EFFECT + "Worker thread stack size.\n" + "This will likely be rounded up to a multiple of 4k (or whatever " + "the page_size might be) by the kernel.", + /* flags */ EXPERIMENTAL ) +/* actual location mgt_pool.c */ PARAM( - /* name */ h2_rx_window_low_water, - /* type */ bytes_u, - /* min */ "65535", - /* max */ "1G", - /* def */ "10M", - /* units */ "bytes", + /* name */ thread_pool_timeout, + /* type */ timeout, + /* min */ "10.000", + /* max */ NULL, + /* def */ "300.000", + /* units */ "seconds", /* descr */ - "HTTP2 Receive Window low water mark.\n" - "We try to keep the window at least this big\n" - "Only affects incoming request bodies (ie: POST, PUT etc.)", - /* flags */ WIZARD + "Thread idle threshold.\n" + "\n" + "Threads in excess of thread_pool_min, which have been idle for at " + "least this long, will be destroyed.", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL ) +/* actual location mgt_pool.c */ PARAM( - /* name */ h2_rx_window_increment, - /* type */ bytes_u, - /* min */ "1M", - /* max */ "1G", - /* def */ "1M", - /* units */ "bytes", + /* name */ thread_pools, + /* type */ uint, + /* min */ "1", + /* max */ NULL, + /* def */ "2", + /* units */ "pools", /* descr */ - "HTTP2 Receive Window Increments.\n" - "How big credits we send in WINDOW_UPDATE frames\n" - "Only affects incoming request bodies (ie: POST, PUT etc.)", - /* flags */ WIZARD + "Number of worker thread pools.\n" + "\n" + "Increasing the number of worker pools decreases lock " + "contention. Each worker pool also has a thread accepting " + "new connections, so for very high rates of incoming new " + "connections on systems with many cores, increasing the " + "worker pools may be required.\n" + "\n" + "Too many pools waste CPU and RAM resources, and more than one " + "pool for each CPU is most likely detrimental to performance.\n" + "\n" + "Can be increased on the fly, but decreases require a restart to " + "take effect.", + /* flags */ DELAYED_EFFECT| EXPERIMENTAL ) +/* actual location mgt_pool.c */ PARAM( - /* name */ h2_header_table_size, - /* type */ bytes_u, - /* min */ "0b", + /* name */ thread_queue_limit, + /* type */ uint, + /* min */ "0", /* max */ NULL, - /* def */ "4k", - /* units */ "bytes", + /* def */ "20", + /* units */ NULL, /* descr */ - "HTTP2 header table size.\n" - "This is the size that will be used for the HPACK dynamic\n" - "decoding table." + "Permitted request queue length per thread-pool.\n" + "\n" + "This sets the number of requests we will queue, waiting for an " + "available thread. Above this limit sessions will be dropped " + "instead of queued.", + /* flags */ EXPERIMENTAL ) +/* actual location mgt_pool.c */ PARAM( - /* name */ h2_max_concurrent_streams, + /* name */ thread_stats_rate, /* type */ uint, /* min */ "0", /* max */ NULL, - /* def */ "100", - /* units */ "streams", + /* def */ "10", + /* units */ "requests", /* descr */ - "HTTP2 Maximum number of concurrent streams.\n" - "This is the number of requests that can be active\n" - "at the same time for a single HTTP2 connection." + "Worker threads accumulate statistics, and dump these into the " + "global stats counters if the lock is free when they finish a job " + "(request/fetch etc).\n" + "This parameters defines the maximum number of jobs a worker " + "thread may handle, before it is forced to dump its accumulated " + "stats into the global counters.", + /* flags */ EXPERIMENTAL ) +/* actual location mgt_param_tbl.c */ PARAM( - /* name */ h2_initial_window_size, - /* type */ bytes_u, - /* min */ "0", - /* max */ "2147483647b", - /* def */ "65535b", - /* units */ "bytes", + /* name */ vcc_allow_inline_c, + /* type */ bool, + /* min */ NULL, + /* max */ NULL, + /* def */ "off", + /* units */ "bool", /* descr */ - "HTTP2 initial flow control window size." + "Allow inline C code in VCL." ) +/* actual location mgt_param_tbl.c */ PARAM( - /* name */ h2_max_frame_size, - /* type */ bytes_u, - /* min */ "16k", - /* max */ "16777215b", - /* def */ "16k", - /* units */ "bytes", + /* name */ vcc_err_unref, + /* type */ bool, + /* min */ NULL, + /* max */ NULL, + /* def */ "on", + /* units */ "bool", /* descr */ - "HTTP2 maximum per frame payload size we are willing to accept." + "Unreferenced VCL objects result in error." ) +/* actual location mgt_param_tbl.c */ PARAM( - /* name */ h2_max_header_list_size, - /* type */ bytes_u, - /* min */ "0b", + /* name */ vcc_unsafe_path, + /* type */ bool, + /* min */ NULL, /* max */ NULL, - /* def */ "2147483647b", - /* units */ "bytes", + /* def */ "on", + /* units */ "bool", /* descr */ - "HTTP2 maximum size of an uncompressed header list." + "Allow '/' in vmod & include paths.\n" + "Allow 'import ... from ...'." +) + +/* actual location mgt_param_bits.c*/ +PARAM( + /* name */ vsl_mask, + /* type */ vsl_mask, + /* min */ NULL, + /* max */ NULL, + /* def */ "default", + /* units */ NULL, + /* descr */ + "Mask individual VSL messages from being logged.\n" + " default Set default value\n" + "\n" + "Use +/- prefix in front of VSL tag name to unmask/mask " + "individual VSL messages." ) +#endif #undef PARAM From nils.goroll at uplex.de Mon Mar 2 15:22:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Mar 2020 15:22:06 +0000 (UTC) Subject: [master] a6051ba75 centralize ctx->method in vcl_send_event Message-ID: <20200302152206.50940AE498@lists.varnish-cache.org> commit a6051ba75c477a8a235842710e66c1825187b026 Author: Nils Goroll Date: Thu Feb 13 09:30:40 2020 +0100 centralize ctx->method in vcl_send_event vcl_event dictates ctx->method, so we set it where we need it. Note: This is in preparation of the disabled AZ(ctx->msg) for cold events, but we are not there yet. Ref #2902 diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index c1c11e6a1..a093e572c 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -160,7 +160,7 @@ VCL_Rel_CliCtx(struct vrt_ctx **ctx) /*--------------------------------------------------------------------*/ static int -vcl_send_event(VRT_CTX, enum vcl_event_e ev) +vcl_send_event(struct vrt_ctx *ctx, enum vcl_event_e ev) { int r; @@ -168,23 +168,35 @@ vcl_send_event(VRT_CTX, enum vcl_event_e ev) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); - assert(ev == VCL_EVENT_LOAD || - ev == VCL_EVENT_WARM || - ev == VCL_EVENT_COLD || - ev == VCL_EVENT_DISCARD); - assert(ev != VCL_EVENT_LOAD || ctx->method == VCL_MET_INIT); - assert(ev != VCL_EVENT_DISCARD || ctx->method == VCL_MET_FINI); + + AZ(ctx->method); + switch (ev) { + case VCL_EVENT_LOAD: + ctx->method = VCL_MET_INIT; + /* FALLTHROUGH */ + case VCL_EVENT_WARM: + AN(ctx->msg); + break; + case VCL_EVENT_DISCARD: + ctx->method = VCL_MET_FINI; + /* FALLTHROUGH */ + case VCL_EVENT_COLD: + // XXX AZ(ctx->msg); + break; + default: + WRONG("vcl_event"); + } + AN(ctx->handling); *ctx->handling = 0; AN(ctx->ws); - if (ev == VCL_EVENT_LOAD || ev == VCL_EVENT_WARM) - AN(ctx->msg); - VCL_TaskEnter(cli_task_privs); r = ctx->vcl->conf->event_vcl(ctx, ev); VCL_TaskLeave(cli_task_privs); + ctx->method = 0; + if (r && (ev == VCL_EVENT_COLD || ev == VCL_EVENT_DISCARD)) WRONG("A VMOD cannot fail COLD or DISCARD events"); @@ -505,7 +517,7 @@ vcl_print_refs(VRT_CTX) } static int -vcl_set_state(VRT_CTX, const char *state) +vcl_set_state(struct vrt_ctx *ctx, const char *state) { struct vcl *vcl; int i = 0; @@ -587,10 +599,8 @@ vcl_cancel_load(struct vrt_ctx *ctx, struct cli *cli, VCL_Rel_CliCtx(&ctx); ctx = VCL_Get_CliCtx(0); ctx->vcl = vcl; - ctx->method = VCL_MET_FINI; ctx->syntax = ctx->vcl->conf->syntax; AZ(vcl_send_event(ctx, VCL_EVENT_DISCARD)); - ctx->method = 0; vcl_KillBackends(vcl); free(vcl->loaded_name); VCL_Close(&vcl); @@ -629,9 +639,7 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx, ctx->syntax = ctx->vcl->conf->syntax; VSB_clear(ctx->msg); - ctx->method = VCL_MET_INIT; i = vcl_send_event(ctx, VCL_EVENT_LOAD); - ctx->method = 0; if (i || *ctx->handling != VCL_RET_OK) { vcl_cancel_load(ctx, cli, name, "initialization"); return; @@ -671,7 +679,6 @@ VCL_Poll(void) ctx = VCL_Get_CliCtx(1); ctx->vcl = vcl; ctx->syntax = ctx->vcl->conf->syntax; - ctx->method = 0; (void)vcl_set_state(ctx, "0"); VCL_Rel_CliCtx(&ctx); } @@ -683,9 +690,7 @@ VCL_Poll(void) ctx = VCL_Get_CliCtx(0); ctx->vcl = vcl; ctx->syntax = ctx->vcl->conf->syntax; - ctx->method = VCL_MET_FINI; AZ(vcl_send_event(ctx, VCL_EVENT_DISCARD)); - ctx->method = 0; vcl_KillBackends(vcl); free(vcl->loaded_name); VCL_Close(&vcl); From nils.goroll at uplex.de Mon Mar 2 15:22:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Mar 2020 15:22:06 +0000 (UTC) Subject: [master] 3982ed51d refactor vcl_load for separate vsb for VSL_Open Message-ID: <20200302152206.7038DAE49B@lists.varnish-cache.org> commit 3982ed51d9a38fb6b08677fde7c41fc8ad2276f7 Author: Nils Goroll Date: Thu Feb 13 19:23:14 2020 +0100 refactor vcl_load for separate vsb for VSL_Open In preparation for the next commit, we avoid using the ctx->msg vsl and use a separate vsl for the VCL_Open step of vcl_load. Ref #2902 diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index a093e572c..dba873d0b 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -611,6 +611,7 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx, const char *name, const char *fn, const char *state) { struct vcl *vcl; + struct vsb *msg; int i; ASSERT_CLI(); @@ -618,14 +619,18 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx, vcl = vcl_find(name); AZ(vcl); - vcl = VCL_Open(fn, ctx->msg); + msg = VSB_new_auto(); + vcl = VCL_Open(fn, msg); + AZ(VSB_finish(ctx->msg)); if (vcl == NULL) { - AZ(VSB_finish(ctx->msg)); VCLI_SetResult(cli, CLIS_PARAM); VCLI_Out(cli, "%s", VSB_data(ctx->msg)); + VSB_destroy(&msg); return; } + VSB_destroy(&msg); + vcl->loaded_name = strdup(name); XXXAN(vcl->loaded_name); VTAILQ_INIT(&vcl->director_list); From nils.goroll at uplex.de Mon Mar 2 15:22:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Mar 2020 15:22:06 +0000 (UTC) Subject: [master] 3cfd35136 Fix ctx->msg for CLI COLD events and simplify VRT_CTX management Message-ID: <20200302152206.9CF62AE4A0@lists.varnish-cache.org> commit 3cfd35136d3e80a81507e3d5cc11193c8ebbf343 Author: Nils Goroll Date: Thu Feb 13 21:12:10 2020 +0100 Fix ctx->msg for CLI COLD events and simplify VRT_CTX management This commit adds the final bit to fix #2902: As discussed in the ticket, there should be no VRT_CTX msg vsb present for COLD events, yet existing code did provide it in some cases. This commit message comments on changes in cache_vcl.c in order of appearance. Existing code kept the CLI VRT_CTX around for longer than it strictly needed to just because access to the error message in ctx->msg was still required. In order to be able to simplify management of the CLI VRT_CTX, we change VCL_Rel_CliCtx() to return the ctx->msg IFF it was requested by a true argument to VCL_Get_CliCtx() Regarding #2902, the main issue with existing code was that the decision on whether or not to request a ctx->msg was separated from where we could actually make that decision: It is (only) in vcl_send_event() where we know which event is going to be issued, which, in turn, determines if we are going to need a ctx->msg. Thus, we move the VRT_CTX management for all of the CLI operations into one place in vcl_send_event() and change the infrastructure around it to handle the vcl and the error message vsb instead of a VRT_CTX. This allows us to (finally) ensure that VCL_EVENT_COLD does never have a ctx->msg error vsb, asserted by AZ(havemsg) in the switch/case statement in vcl_send_event(). In vcl_send_event(), we also assert that if a vcl subroutine was called in the case of a LOAD or DISCARD event, only vcl_init {} for a LOAD event may return anything but OK. vcl_set_state() contains the two use cases of vcl_send_event(): When we know that an event may not fail, we also assert that there is no msg vsb returned, and if it could, we keep it, as it is still required by its callers. vcl_cancel_load() shows how the new infrastructure simplifies the code: Rather than having to re-setup the VRT_CTX for the DISCARD event, we simply output the error and have vcl_send_event() set up the proper handling for that event. Basically the same pattern repeats in vcl_load(), VCL_Poll() and vcl_cli_state(): In these functions, we only look after error handling and output. Fixes #2902 diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index b0adeb172..fe5b132b9 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -344,7 +344,7 @@ do_list(struct cli *cli, struct director *d, void *priv) d->vdir->methods->list(ctx, d, la->vsb, la->p, 0); VSB_printf(la->vsb, "\n"); - VCL_Rel_CliCtx(&ctx); + AZ(VCL_Rel_CliCtx(&ctx)); AZ(ctx); return (0); @@ -389,7 +389,7 @@ do_list_json(struct cli *cli, struct director *d, void *priv) VSB_indent(cli->sb, -2); VCLI_Out(cli, "}"); - VCL_Rel_CliCtx(&ctx); + AZ(VCL_Rel_CliCtx(&ctx)); AZ(ctx); return (0); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index e7fc7515f..6e98f425d 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -430,7 +430,7 @@ const char *VCL_Method_Name(unsigned); void VCL_Bo2Ctx(struct vrt_ctx *, struct busyobj *); void VCL_Req2Ctx(struct vrt_ctx *, struct req *); struct vrt_ctx *VCL_Get_CliCtx(int); -void VCL_Rel_CliCtx(struct vrt_ctx **); +struct vsb *VCL_Rel_CliCtx(struct vrt_ctx **); #define VCL_MET_MAC(l,u,t,b) \ void VCL_##l##_method(struct vcl *, struct worker *, struct req *, \ diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index dba873d0b..fe8758a6b 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -140,62 +140,91 @@ VCL_Get_CliCtx(int msg) return (&ctx_cli); } -void +/* + * releases CLI ctx + * + * returns finished error msg vsb if VCL_Get_CliCtx(1) was called + * + * caller needs to VSB_destroy a non-NULL return value + * + */ +struct vsb * VCL_Rel_CliCtx(struct vrt_ctx **ctx) { + struct vsb *r = NULL; ASSERT_CLI(); assert(*ctx == &ctx_cli); AN((*ctx)->handling); - if (ctx_cli.msg) - VSB_destroy(&ctx_cli.msg); + if (ctx_cli.msg) { + TAKE_OBJ_NOTNULL(r, &ctx_cli.msg, VSB_MAGIC); + AZ(VSB_finish(r)); + } if (ctx_cli.vsl) VSL_Flush(ctx_cli.vsl, 0); WS_Assert(ctx_cli.ws); WS_Reset(&ws_cli, ws_snapshot_cli); INIT_OBJ(*ctx, VRT_CTX_MAGIC); *ctx = NULL; + + return (r); } /*--------------------------------------------------------------------*/ static int -vcl_send_event(struct vrt_ctx *ctx, enum vcl_event_e ev) +vcl_send_event(struct vcl *vcl, enum vcl_event_e ev, struct vsb **msg) { - int r; + int r, havemsg = 0; + unsigned method = 0; + struct vrt_ctx *ctx; ASSERT_CLI(); - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); - AZ(ctx->method); + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC); + AN(msg); + AZ(*msg); + switch (ev) { case VCL_EVENT_LOAD: - ctx->method = VCL_MET_INIT; + method = VCL_MET_INIT; /* FALLTHROUGH */ case VCL_EVENT_WARM: - AN(ctx->msg); + havemsg = 1; break; case VCL_EVENT_DISCARD: - ctx->method = VCL_MET_FINI; + method = VCL_MET_FINI; /* FALLTHROUGH */ case VCL_EVENT_COLD: - // XXX AZ(ctx->msg); + AZ(havemsg); break; default: WRONG("vcl_event"); } + ctx = VCL_Get_CliCtx(havemsg); + AN(ctx->handling); - *ctx->handling = 0; + AZ(*ctx->handling); AN(ctx->ws); + ctx->vcl = vcl; + ctx->syntax = ctx->vcl->conf->syntax; + ctx->method = method; + VCL_TaskEnter(cli_task_privs); r = ctx->vcl->conf->event_vcl(ctx, ev); VCL_TaskLeave(cli_task_privs); - ctx->method = 0; + /* if the warm event did not get to vcl_init, vcl_fini + * won't be run, so handling may be zero */ + if (method && *ctx->handling && *ctx->handling != VCL_RET_OK) { + assert(ev == VCL_EVENT_LOAD); + r = 1; + } + + *msg = VCL_Rel_CliCtx(&ctx); if (r && (ev == VCL_EVENT_COLD || ev == VCL_EVENT_DISCARD)) WRONG("A VMOD cannot fail COLD or DISCARD events"); @@ -499,39 +528,37 @@ VCL_TestLoad(const char *fn) /*--------------------------------------------------------------------*/ -static void -vcl_print_refs(VRT_CTX) +static struct vsb * +vcl_print_refs(const struct vcl *vcl) { - struct vcl *vcl; + struct vsb *msg; struct vclref *ref; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); - AN(ctx->msg); - vcl = ctx->vcl; - VSB_printf(ctx->msg, "VCL %s is waiting for:", vcl->loaded_name); + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + msg = VSB_new_auto(); + + VSB_printf(msg, "VCL %s is waiting for:", vcl->loaded_name); Lck_Lock(&vcl_mtx); - VTAILQ_FOREACH(ref, &ctx->vcl->ref_list, list) - VSB_printf(ctx->msg, "\n\t- %s", ref->desc); + VTAILQ_FOREACH(ref, &vcl->ref_list, list) + VSB_printf(msg, "\n\t- %s", ref->desc); Lck_Unlock(&vcl_mtx); + AZ(VSB_finish(msg)); + return (msg); } static int -vcl_set_state(struct vrt_ctx *ctx, const char *state) +vcl_set_state(struct vcl *vcl, const char *state, struct vsb **msg) { - struct vcl *vcl; + struct vsb *nomsg = NULL; int i = 0; ASSERT_CLI(); - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); - AN(ctx->handling); - AN(ctx->vcl); + + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); AN(state); - assert(ctx->msg != NULL || *state == '0'); + AN(msg); + AZ(*msg); - vcl = ctx->vcl; AN(vcl->temp); switch (state[0]) { @@ -541,7 +568,8 @@ vcl_set_state(struct vrt_ctx *ctx, const char *state) if (vcl->busy == 0 && vcl->temp->is_warm) { vcl->temp = VTAILQ_EMPTY(&vcl->ref_list) ? VCL_TEMP_COLD : VCL_TEMP_COOLING; - AZ(vcl_send_event(ctx, VCL_EVENT_COLD)); + AZ(vcl_send_event(vcl, VCL_EVENT_COLD, msg)); + AZ(*msg); vcl_BackendEvent(vcl, VCL_EVENT_COLD); } else if (vcl->busy) @@ -559,17 +587,18 @@ vcl_set_state(struct vrt_ctx *ctx, const char *state) vcl->temp = VCL_TEMP_WARM; /* The VCL must first reach a stable cold state */ else if (vcl->temp == VCL_TEMP_COOLING) { - vcl_print_refs(ctx); + *msg = vcl_print_refs(vcl); i = -1; } else { vcl->temp = VCL_TEMP_WARM; - i = vcl_send_event(ctx, VCL_EVENT_WARM); + i = vcl_send_event(vcl, VCL_EVENT_WARM, msg); if (i == 0) { vcl_BackendEvent(vcl, VCL_EVENT_WARM); break; } - AZ(vcl_send_event(ctx, VCL_EVENT_COLD)); + AZ(vcl_send_event(vcl, VCL_EVENT_COLD, &nomsg)); + AZ(nomsg); vcl->temp = VCL_TEMP_COLD; } break; @@ -583,36 +612,33 @@ vcl_set_state(struct vrt_ctx *ctx, const char *state) } static void -vcl_cancel_load(struct vrt_ctx *ctx, struct cli *cli, +vcl_cancel_load(struct vcl *vcl, struct cli *cli, struct vsb *msg, const char *name, const char *step) { - struct vcl *vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC); - AZ(VSB_finish(ctx->msg)); VCLI_SetResult(cli, CLIS_CANT); VCLI_Out(cli, "VCL \"%s\" Failed %s", name, step); - if (VSB_len(ctx->msg)) - VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(ctx->msg)); - VCL_Rel_CliCtx(&ctx); - ctx = VCL_Get_CliCtx(0); - ctx->vcl = vcl; - ctx->syntax = ctx->vcl->conf->syntax; - AZ(vcl_send_event(ctx, VCL_EVENT_DISCARD)); + if (VSB_len(msg)) + VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(msg)); + VSB_destroy(&msg); + + AZ(vcl_send_event(vcl, VCL_EVENT_DISCARD, &msg)); + AZ(msg); + vcl_KillBackends(vcl); free(vcl->loaded_name); VCL_Close(&vcl); } static void -vcl_load(struct cli *cli, struct vrt_ctx *ctx, +vcl_load(struct cli *cli, const char *name, const char *fn, const char *state) { struct vcl *vcl; struct vsb *msg; - int i; ASSERT_CLI(); @@ -621,10 +647,10 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx, msg = VSB_new_auto(); vcl = VCL_Open(fn, msg); - AZ(VSB_finish(ctx->msg)); + AZ(VSB_finish(msg)); if (vcl == NULL) { VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, "%s", VSB_data(ctx->msg)); + VCLI_Out(cli, "%s", VSB_data(msg)); VSB_destroy(&msg); return; } @@ -640,23 +666,20 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx, vcl->temp = VCL_TEMP_INIT; - ctx->vcl = vcl; - ctx->syntax = ctx->vcl->conf->syntax; - - VSB_clear(ctx->msg); - i = vcl_send_event(ctx, VCL_EVENT_LOAD); - if (i || *ctx->handling != VCL_RET_OK) { - vcl_cancel_load(ctx, cli, name, "initialization"); + if (vcl_send_event(vcl, VCL_EVENT_LOAD, &msg)) { + vcl_cancel_load(vcl, cli, msg, name, "initialization"); return; } - assert(*ctx->handling == VCL_RET_OK); - VSB_clear(ctx->msg); - i = vcl_set_state(ctx, state); - if (i) { + VSB_destroy(&msg); + + if (vcl_set_state(vcl, state, &msg)) { assert(*state == '1'); - vcl_cancel_load(ctx, cli, name, "warmup"); + vcl_cancel_load(vcl, cli, msg, name, "warmup"); return; } + if (msg) + VSB_destroy(&msg); + VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name); VTAILQ_INSERT_TAIL(&vcl_head, vcl, list); Lck_Lock(&vcl_mtx); @@ -672,36 +695,27 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx, void VCL_Poll(void) { - struct vrt_ctx *ctx; + struct vsb *nomsg = NULL; struct vcl *vcl, *vcl2; ASSERT_CLI(); VTAILQ_FOREACH_SAFE(vcl, &vcl_head, list, vcl2) { if (vcl->temp == VCL_TEMP_BUSY || - vcl->temp == VCL_TEMP_COOLING) { - // XXX #2902 : cold event to have msg ? - // move ctx into vcl_set_state() ? - ctx = VCL_Get_CliCtx(1); - ctx->vcl = vcl; - ctx->syntax = ctx->vcl->conf->syntax; - (void)vcl_set_state(ctx, "0"); - VCL_Rel_CliCtx(&ctx); - } + vcl->temp == VCL_TEMP_COOLING) + AZ(vcl_set_state(vcl, "0", &nomsg)); + AZ(nomsg); if (vcl->discard && vcl->temp == VCL_TEMP_COLD) { AZ(vcl->busy); assert(vcl != vcl_active); assert(VTAILQ_EMPTY(&vcl->ref_list)); VTAILQ_REMOVE(&vcl_head, vcl, list); - ctx = VCL_Get_CliCtx(0); - ctx->vcl = vcl; - ctx->syntax = ctx->vcl->conf->syntax; - AZ(vcl_send_event(ctx, VCL_EVENT_DISCARD)); + AZ(vcl_send_event(vcl, VCL_EVENT_DISCARD, &nomsg)); + AZ(nomsg); vcl_KillBackends(vcl); free(vcl->loaded_name); VCL_Close(&vcl); VSC_C_main->n_vcl--; VSC_C_main->n_vcl_discard--; - VCL_Rel_CliCtx(&ctx); } } } @@ -794,37 +808,38 @@ vcl_cli_list_json(struct cli *cli, const char * const *av, void *priv) static void v_matchproto_(cli_func_t) vcl_cli_load(struct cli *cli, const char * const *av, void *priv) { - struct vrt_ctx *ctx; AZ(priv); ASSERT_CLI(); - ctx = VCL_Get_CliCtx(1); - vcl_load(cli, ctx, av[2], av[3], av[4]); - VCL_Rel_CliCtx(&ctx); + // XXX move back code from vcl_load? + vcl_load(cli, av[2], av[3], av[4]); } static void v_matchproto_(cli_func_t) vcl_cli_state(struct cli *cli, const char * const *av, void *priv) { - struct vrt_ctx *ctx; + struct vcl *vcl; + struct vsb *msg = NULL; AZ(priv); ASSERT_CLI(); AN(av[2]); AN(av[3]); - ctx = VCL_Get_CliCtx(1); - ctx->vcl = vcl_find(av[2]); - AN(ctx->vcl); - ctx->syntax = ctx->vcl->conf->syntax; - if (vcl_set_state(ctx, av[3])) { - AZ(VSB_finish(ctx->msg)); + + vcl = vcl_find(av[2]); + AN(vcl); + + if (vcl_set_state(vcl, av[3], &msg)) { + CHECK_OBJ_NOTNULL(msg, VSB_MAGIC); + VCLI_SetResult(cli, CLIS_CANT); - VCLI_Out(cli, "Failed ", ctx->vcl->loaded_name, + VCLI_Out(cli, "Failed ", vcl->loaded_name, av[3] + 1); - if (VSB_len(ctx->msg)) - VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(ctx->msg)); + if (VSB_len(msg)) + VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(msg)); } - VCL_Rel_CliCtx(&ctx); + if (msg) + VSB_destroy(&msg); } static void v_matchproto_(cli_func_t) diff --git a/include/vrt.h b/include/vrt.h index 8d06bb7b5..0b54d74e5 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -292,7 +292,6 @@ struct vrt_ctx { * msg is for error messages and exists only for * VCL_EVENT_LOAD * VCL_EVENT_WARM - * VCL_EVENT_COLD */ struct vsb *msg; struct vsl_log *vsl; diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 45234400c..11fddb90a 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -499,7 +499,7 @@ event_cold(VRT_CTX, const struct vmod_priv *priv) pthread_t thread; struct priv_vcl *priv_vcl; - AN(ctx->msg); + AZ(ctx->msg); CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC); From nils.goroll at uplex.de Mon Mar 2 20:06:12 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Mar 2020 20:06:12 +0000 (UTC) Subject: [master] b169266e3 Add VRT_AllocStrandsWS() to allocate strands on a workspace Message-ID: <20200302200612.C95E6BFA6D@lists.varnish-cache.org> commit b169266e3545b6fe395e1614f706ea071262c39e Author: Nils Goroll Date: Tue Feb 11 15:56:20 2020 +0100 Add VRT_AllocStrandsWS() to allocate strands on a workspace So far, we have focused on transitioning vmod arguments to STRANDS. To take the next step and also return STRANDS (which leverages the actual benefit - not copying unmodified strand elements), vmods need to allocate a struct strands on the workspace. This commit adds a utility function for this common task. Implementation note: A single WS_Alloc() with some pointer arithmetic would suffice, but using two results in cleaner code. diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index fe26401eb..b89efccf4 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -180,6 +180,30 @@ VRT_GetHdr(VRT_CTX, VCL_HEADER hs) return (p); } +/*-------------------------------------------------------------------- + * Alloc Strands with space for n elements on workspace + * + * Error handling is deliberately left to the caller + */ + +struct strands * +VRT_AllocStrandsWS(struct ws *ws, int n) +{ + struct strands *s; + const char **p; + + s = WS_Alloc(ws, sizeof *s); + p = WS_Alloc(ws, n * sizeof *p); + + if (s == NULL || p == NULL) + return (NULL); + + s->n = n; + s->p = p; + + return (s); +} + /*-------------------------------------------------------------------- * Build STRANDS from what is essentially a STRING_LIST */ diff --git a/include/vrt.h b/include/vrt.h index 0b54d74e5..2baea19a8 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -60,6 +60,7 @@ * VRT_l_resp_body() changed * VRT_l_beresp_body() changed * VRT_Format_Proxy() added // transitional interface + * VRT_AllocStrandsWS() added * 10.0 (2019-09-15) * VRT_UpperLowerStrands added. * VRT_synth_page now takes STRANDS argument @@ -206,6 +207,9 @@ struct strands { const char **p; }; +struct strands * VRT_AllocStrandsWS(struct ws *, int); + + /* * VCL_BLOB: * From dridi.boukelmoune at gmail.com Mon Mar 2 20:16:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 20:16:07 +0000 (UTC) Subject: [6.3] 03e5baeed Don't ignore OA_GZIPBITS if there is a boc Message-ID: <20200302201607.9336ABFECB@lists.varnish-cache.org> commit 03e5baeed877ce24a09cf80450651bded04d7729 Author: Dridi Boukelmoune Date: Fri Jan 10 20:17:44 2020 +0100 Don't ignore OA_GZIPBITS if there is a boc Under load, client c4 from g00005.vtc may fail with a 200 response instead of the expected 206 partial response. There is a window during which we might still see a boc, but because c4 sets beresp.do_stream to false, the fetch has to be over. To close this race we can instead reference the boc as suggested in #2904 and keep track of the boc state. Refs #3235 diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 46146beb3..d812edc5a 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -42,6 +42,7 @@ #include "cache_varnishd.h" #include "cache_filter.h" +#include "cache_objhead.h" #include "cache_vgz.h" #include "vend.h" @@ -288,6 +289,8 @@ static int v_matchproto_(vdp_init_f) vdp_gunzip_init(struct req *req, void **priv) { struct vgz *vg; + struct boc *boc; + enum boc_state_e bos; const char *p; ssize_t dl; uint64_t u; @@ -309,8 +312,15 @@ vdp_gunzip_init(struct req *req, void **priv) req->resp_len = -1; + boc = HSH_RefBoc(req->objcore); + if (boc != NULL) { + bos = boc->state; + HSH_DerefBoc(req->wrk, req->objcore); + } else + bos = BOS_FINISHED; + /* OA_GZIPBITS is not stable yet */ - if (req->objcore->boc) + if (bos < BOS_FINISHED) return (0); p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl); diff --git a/bin/varnishtest/tests/g00005.vtc b/bin/varnishtest/tests/g00005.vtc index 24b3b6071..74b21de5b 100644 --- a/bin/varnishtest/tests/g00005.vtc +++ b/bin/varnishtest/tests/g00005.vtc @@ -35,7 +35,7 @@ client c1 { varnish v1 -vsl_catchup -client c1 { +client c2 { txreq -hdr "Accept-encoding: gzip;q=0.1" rxresp expect resp.http.content-encoding == "gzip" @@ -49,7 +49,7 @@ varnish v1 -vsl_catchup # is completed before we attempt the range request delay 2 -client c1 { +client c3 { txreq -hdr "Range: bytes=3-5" rxresp expect resp.status == 206 @@ -60,7 +60,7 @@ client c1 { varnish v1 -vsl_catchup -client c1 { +client c4 { txreq -url "/nostreamcachemiss" -hdr "Range: bytes=3-5" rxresp expect resp.status == 206 @@ -71,7 +71,7 @@ client c1 { varnish v1 -vsl_catchup -client c1 { +client c5 { # simple cache miss, no stream, no gunzip txreq -url "/nostream2" -hdr "Range: bytes=3-5" -hdr "Accept-Encoding: gzip" rxresp From dridi.boukelmoune at gmail.com Mon Mar 2 20:16:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 2 Mar 2020 20:16:07 +0000 (UTC) Subject: [6.3] 864611901 Stabilize r2964 Message-ID: <20200302201607.AE97ABFECD@lists.varnish-cache.org> commit 864611901e9d9db61cb3a07b595cda38935ffdcb Author: Dridi Boukelmoune Date: Fri Jan 3 09:53:05 2020 +0100 Stabilize r2964 The fetch may be interrupted before s1 has time to buffer the complete response. Refs #3236 diff --git a/bin/varnishtest/tests/r02964.vtc b/bin/varnishtest/tests/r02964.vtc index 12cc4d603..427b1c3cf 100644 --- a/bin/varnishtest/tests/r02964.vtc +++ b/bin/varnishtest/tests/r02964.vtc @@ -1,6 +1,7 @@ varnishtest "Cancel private busy obj from vcl_deliver" server s1 { + non_fatal rxreq expect req.url == "/hfm" txresp -hdr "HFM: True" -bodylen 65530 From nils.goroll at uplex.de Mon Mar 2 21:05:09 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Mar 2020 21:05:09 +0000 (UTC) Subject: [master] f1c47e486 add v_dont_optimize and enable it for vcl_init / vcl_fini Message-ID: <20200302210509.7FE3D92FD@lists.varnish-cache.org> commit f1c47e4860f213710bd58f0c41fc1f1437d6c461 Author: Nils Goroll Date: Mon Mar 2 21:47:47 2020 +0100 add v_dont_optimize and enable it for vcl_init / vcl_fini We add an attribute macro which, for gcc, will disable compiler optimizations. The intended use are VCL subs which are called exactly once such that any additional compilation effort is wasteful with respect to the overall vcl.load time. We also decorate the vcc-generated C code for vcl_init and vcl_fini with v_dont_optimize, which is simple and easy: void v_dont_optimize v_matchproto_(vcl_func_f) VGC_function_vcl_fini(VRT_CTX) void v_dont_optimize v_matchproto_(vcl_func_f) VGC_function_vcl_init(VRT_CTX) A more difficult improvement is left to be done: Any custom vcl subs which are called from vcl_init and/or vcl_fini _only_ should also be decorated with v_dont_optimize. With the current code base, determining if a custom sub qualifies would require helper code in vcc_xref to check all uses and return the usage union. As #3163 requires a similar mechanism and because we are about to enter the pre-6.4 release freeze, the better option seems to be to implement this "use mask" when/if #3163 gets in and come back to this optimization then. Closes #3230 diff --git a/include/vdef.h b/include/vdef.h index 0a60716b0..a9111feb6 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -100,6 +100,12 @@ # define v_deprecated_ #endif +#if __GNUC_PREREQ__(4,4) // added 2008-07-23 +# define v_dont_optimize __attribute__((optimize("O"))) +#else +# define v_dont_optimize +#endif + /********************************************************************* * Pointer alignment magic */ diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index b7ccc9cf1..1be668bfd 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -151,7 +151,16 @@ vcc_EmitProc(struct vcc *tl, struct proc *p) AZ(VSB_finish(p->prologue)); AZ(VSB_finish(p->body)); Fh(tl, 1, "vcl_func_f %s;\n", VSB_data(p->cname)); - Fc(tl, 1, "\nvoid v_matchproto_(vcl_func_f)\n"); + /* + * TODO: v_dont_optimize for custom subs called from vcl_init/fini only + * + * Needs infrastructure similar to that in #3163 : custom subs need a + * mask containing the builtin subs calling the custom sub. If that is + * no larger than VCL_MET_TASK_H, we can enable v_dont_optimize + */ + Fc(tl, 1, "\nvoid %sv_matchproto_(vcl_func_f)\n", + p->method && p->method->bitval & VCL_MET_TASK_H ? + "v_dont_optimize " : ""); Fc(tl, 1, "%s(VRT_CTX)\n", VSB_data(p->cname)); Fc(tl, 1, "{\n%s\n%s}\n", VSB_data(p->prologue), VSB_data(p->body)); VSB_destroy(&p->body); From dridi.boukelmoune at gmail.com Tue Mar 3 10:02:10 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 3 Mar 2020 10:02:10 +0000 (UTC) Subject: [master] 896151b41 An overflowed workspace must remain overflowed after WS_Reset() Message-ID: <20200303100210.4F7B3A0E16@lists.varnish-cache.org> commit 896151b411bda1674112d859e9bc3eb4ad611038 Author: Nils Goroll Date: Mon Jan 27 16:42:48 2020 +0100 An overflowed workspace must remain overflowed after WS_Reset() We use workspace overflows to signal to bail out for example after a failing `VRT_SetHdr()`. This is a guarantee that if some serious issue occurred during processing, we rather send an error downstream than an incomplete response or the result of incomplete processing. We use the `WS_Snapshot() ... WS_Reset()` pattern as some kind of second order workspace allocation where the called code itself uses `WS_Reserve()`. With this usage pattern, `WS_Reset()` called `ws_ClearOverflow(ws)`, potentially clearing the overflow bit from a previous relevant failure. We now avoid any other unintended clears of the overflow bit by splitting two functions: * WS_Rollback() is now what WS_Reset() used to be: It clears overflows and accepts the zero cookie for a reset-to-start It is only intended for use within varnishd and is thus declared in cache_varnishd.h * WS_Reset() does not touch the overflow bit any longer, ensuring that a once-overflowed workspace stays overflowed `WS_Snapshot()` now returns a magic value which gets recognized by `WS_Reset()` to ensure that the overflowed marker is still present. This serves two purposes: - better debugging and - a safety measure against passing a cookie from an already overflowed workspace to WS_Rollback() Fixes #3194 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 7bc595613..dd51a468e 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -110,8 +110,8 @@ void Bereq_Rollback(struct busyobj *bo) VCL_TaskLeave(bo->privs); VCL_TaskEnter(bo->privs); HTTP_Clone(bo->bereq, bo->bereq0); - WS_Reset(bo->bereq->ws, bo->ws_bo); - WS_Reset(bo->ws, bo->ws_bo); + WS_Rollback(bo->bereq->ws, bo->ws_bo); + WS_Rollback(bo->ws, bo->ws_bo); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 122dfdb6d..d9ad87203 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -200,7 +200,7 @@ Req_Rollback(struct req *req) HTTP_Clone(req->http, req->http0); if (WS_Overflowed(req->ws)) req->wrk->stats->ws_client_overflow++; - WS_Reset(req->ws, req->ws_req); + WS_Rollback(req->ws, req->ws_req); } /*---------------------------------------------------------------------- @@ -251,7 +251,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) if (WS_Overflowed(req->ws)) wrk->stats->ws_client_overflow++; - WS_Reset(req->ws, 0); + WS_Rollback(req->ws, 0); } /*---------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 6e98f425d..a9050a099 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -463,6 +463,9 @@ void VMOD_Panic(struct vsb *); /* cache_wrk.c */ void WRK_Init(void); +/* cache_ws.c */ +void WS_Rollback(struct ws *, uintptr_t); + /* http1/cache_http1_pipe.c */ void V1P_Init(void); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index fe8758a6b..6064a9c08 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -163,7 +163,7 @@ VCL_Rel_CliCtx(struct vrt_ctx **ctx) if (ctx_cli.vsl) VSL_Flush(ctx_cli.vsl, 0); WS_Assert(ctx_cli.ws); - WS_Reset(&ws_cli, ws_snapshot_cli); + WS_Rollback(&ws_cli, ws_snapshot_cli); INIT_OBJ(*ctx, VRT_CTX_MAGIC); *ctx = NULL; diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index d7fb95530..b4ad19b90 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -348,7 +348,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); tp = NULL; - WS_Reset(wrk->aws, 0); + WS_Rollback(wrk->aws, 0); AZ(wrk->vsl); Lck_Lock(&pp->mtx); diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 39a91992f..3da9ba732 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -36,6 +36,8 @@ #include +static const void * const snap_overflowed = &snap_overflowed; + void WS_Assert(const struct ws *ws) { @@ -127,7 +129,12 @@ ws_ClearOverflow(struct ws *ws) } /* - * Reset a WS to start or a given pointer, likely from WS_Snapshot + * Reset a WS to a cookie from WS_Snapshot + * + * for use by any code using cache.h + * + * does not reset the overflow bit and asserts that, if WS_Snapshot had found + * the workspace overflown, the marker is intact */ void @@ -136,20 +143,39 @@ WS_Reset(struct ws *ws, uintptr_t pp) char *p; WS_Assert(ws); + AN(pp); + if (pp == (uintptr_t)snap_overflowed) { + DSL(DBG_WORKSPACE, 0, "WS_Reset(%p, overflowed)", ws); + AN(WS_Overflowed(ws)); + return; + } p = (char *)pp; DSL(DBG_WORKSPACE, 0, "WS_Reset(%p, %p)", ws, p); assert(ws->r == NULL); - if (p == NULL) - ws->f = ws->s; - else { - assert(p >= ws->s); - assert(p <= ws->e); - ws->f = p; - } - ws_ClearOverflow(ws); + assert(p >= ws->s); + assert(p <= ws->e); + ws->f = p; WS_Assert(ws); } +/* + * Reset the WS to a cookie or its start and clears any overflow + * + * for varnishd internal use only + */ + +void +WS_Rollback(struct ws *ws, uintptr_t pp) +{ + WS_Assert(ws); + + if (pp == 0) + pp = (uintptr_t)ws->s; + + ws_ClearOverflow(ws); + WS_Reset(ws, pp); +} + void * WS_Alloc(struct ws *ws, unsigned bytes) { @@ -224,8 +250,12 @@ WS_Snapshot(struct ws *ws) WS_Assert(ws); assert(ws->r == NULL); + if (WS_Overflowed(ws)) { + DSL(DBG_WORKSPACE, 0, "WS_Snapshot(%p) = overflowed", ws); + return ((uintptr_t) snap_overflowed); + } DSL(DBG_WORKSPACE, 0, "WS_Snapshot(%p) = %p", ws, ws->f); - return (ws->f == ws->s ? 0 : (uintptr_t)ws->f); + return ((uintptr_t)ws->f); } /* diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index 88ff2ffd9..a6914d3e6 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -139,7 +139,7 @@ V1L_Close(struct worker *wrk, uint64_t *cnt) *cnt = v1l->cnt; if (v1l->ws->r) WS_Release(v1l->ws, 0); - WS_Reset(v1l->ws, v1l->res); + WS_Rollback(v1l->ws, v1l->res); ZERO_OBJ(v1l, sizeof *v1l); return (sc); } diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index a1cdbf9c1..6996e879a 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -321,8 +321,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) sz, r, &req->acct.resp_hdrbytes); H2_Send_Rel(r2->h2sess, r2); - if (!WS_Overflowed(req->ws)) // XXX: remove if when #3202 is fixed - WS_Reset(req->ws, ss); + WS_Reset(req->ws, ss); /* XXX someone into H2 please add appropriate error handling */ if (sendbody) { diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index fec88569b..543eea868 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -365,7 +365,7 @@ h2_new_session(struct worker *wrk, void *arg) } assert(HTC_S_COMPLETE == H2_prism_complete(h2->htc)); HTC_RxPipeline(h2->htc, h2->htc->rxbuf_b + sizeof(H2_prism)); - WS_Reset(h2->ws, 0); + WS_Rollback(h2->ws, 0); HTC_RxInit(h2->htc, h2->ws); AN(h2->ws->r); VSLb(h2->vsl, SLT_Debug, "H2: Got pu PRISM"); @@ -387,7 +387,7 @@ h2_new_session(struct worker *wrk, void *arg) h2->cond = &wrk->cond; while (h2_rxframe(wrk, h2)) { - WS_Reset(h2->ws, 0); + WS_Rollback(h2->ws, 0); HTC_RxInit(h2->htc, h2->ws); if (WS_Overflowed(h2->ws)) { VSLb(h2->vsl, SLT_Debug, "H2: Empty Rx Workspace"); diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index a9b267458..4eb5526ee 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -145,7 +145,7 @@ vpx_proto1(const struct worker *wrk, const struct req *req) VSL(SLT_Proxy, req->sp->vxid, "1 %s %s %s %s", fld[1], fld[3], fld[2], fld[4]); HTC_RxPipeline(req->htc, q); - WS_Reset(req->htc->ws, 0); + WS_Rollback(req->htc->ws, 0); return (0); } @@ -345,7 +345,7 @@ vpx_proto2(const struct worker *wrk, struct req *req) hdr_len = l + 16L; assert(req->htc->rxbuf_e - req->htc->rxbuf_b >= hdr_len); HTC_RxPipeline(req->htc, req->htc->rxbuf_b + hdr_len); - WS_Reset(req->ws, 0); + WS_Rollback(req->ws, 0); p = (const void *)req->htc->rxbuf_b; d = req->htc->rxbuf_b + 16L; From dridi.boukelmoune at gmail.com Tue Mar 3 10:02:10 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 3 Mar 2020 10:02:10 +0000 (UTC) Subject: [master] 9844c65a7 Test that std.syslog does not clear a workspace overflow Message-ID: <20200303100210.63F64A0E19@lists.varnish-cache.org> commit 9844c65a7f1cb60eb89d801dff5900385caf6498 Author: Nils Goroll Date: Mon Jan 27 17:44:02 2020 +0100 Test that std.syslog does not clear a workspace overflow and check our WS_Snapshot() / WS_Reset() debug messages diff --git a/bin/varnishtest/tests/c00071.vtc b/bin/varnishtest/tests/c00071.vtc index fdb48c4da..7ad563095 100644 --- a/bin/varnishtest/tests/c00071.vtc +++ b/bin/varnishtest/tests/c00071.vtc @@ -11,7 +11,7 @@ server s1 { txresp } -start -varnish v1 -vcl+backend { +varnish v1 -arg "-p debug=+workspace" -vcl+backend { import vtc; import std; sub vcl_deliver { @@ -24,6 +24,7 @@ varnish v1 -vcl+backend { else if (req.url ~ "/baz") { set resp.http.x-foo = regsub(req.url, "baz", "baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaz"); std.log("dummy"); + std.syslog(8 + 7, "vtc: " + 1 + 2); } set resp.http.x-of = vtc.workspace_overflowed(client); } @@ -43,7 +44,7 @@ client c1 { varnish v1 -vsl_catchup -logexpect l1 -v v1 -g raw { +logexpect l1 -v v1 -g vxid -q "vxid == 1006" { expect * 1006 VCL_call {^DELIVER$} expect 0 = LostHeader {^x-foo:$} # std.log does not need workspace @@ -53,6 +54,11 @@ logexpect l1 -v v1 -g raw { expect 6 = Error {^workspace_client overflow} } -start +logexpect l2 -v v1 -g raw { + expect * 0 Debug {^WS_Snapshot.* = overflowed} + expect 12 0 Debug {^WS_Reset.*, overflowed} +} -start + client c2 { txreq -url /baz rxresp @@ -61,6 +67,7 @@ client c2 { } -run logexpect l1 -wait +logexpect l2 -wait varnish v1 -expect client_resp_500 == 2 varnish v1 -expect ws_client_overflow == 2 From nils.goroll at uplex.de Tue Mar 3 10:14:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 3 Mar 2020 10:14:06 +0000 (UTC) Subject: [master] 7d3ffa515 explicitly disable the send deadline on the backend side Message-ID: <20200303101406.8703AA15E2@lists.varnish-cache.org> commit 7d3ffa5159edbbb721276c83f43f05ae32376aa4 Author: Nils Goroll Date: Tue Mar 3 11:03:49 2020 +0100 explicitly disable the send deadline on the backend side As @Dridi and myself concluded, the send_timeout had no effect on backend connections anyway because we never set SO_SNDTIMEO (aka idle_send_timeout on the client side) on backend connections. With the next commit, we will fix the send_timeout on the client side and thus would also enable it for "dripping" writes on the backend side. To preserve existing behavior for the time being, we explicitly disable the timeout (actually deadline) on the backend side. There is ongoing work in progress to rework all of our timeouts for 7.x. Implementation note: if (VTIM_real() > v1l->deadline) evaluates to false for v1l->deadline == NaN Ref #3189 diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index ad70c6868..3892b1544 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -97,9 +97,8 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, } VTCP_blocking(*htc->rfd); /* XXX: we should timeout instead */ - /* XXX: what is the right timeout ? Isn't send_timeout client side? */ - V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, - bo->t_prev + cache_param->send_timeout, 0); + /* XXX: need a send_timeout for the backend side */ + V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, nan(""), 0); hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req); /* Deal with any message-body the request might (still) have */ From nils.goroll at uplex.de Tue Mar 3 10:14:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 3 Mar 2020 10:14:06 +0000 (UTC) Subject: [master] 49e44e390 respect `send_timeout` for "dripping" http1 writes Message-ID: <20200303101406.A7C47A15E5@lists.varnish-cache.org> commit 49e44e390488c57c03d8111fcd43e245933bc151 Author: Nils Goroll Date: Mon Jan 27 14:18:28 2020 +0100 respect `send_timeout` for "dripping" http1 writes Previously, we only checked `v1l->deadline` (which gets initialized from the `send_timeout` session attribute or parameter) only for short writes, so for successful "dripping" http1 writes (streaming from a backend busy object with small chunks), we did not respect the timeout. This patch restructures `V1L_Flush()` to always check the deadline before a write by turning a `while() { ... }` into a `do { ... } while` with the same continuation criteria: `while (i != v1l->liov)` is turned into `if (i == v1l->liov) break;` and `while (i > 0 || errno == EWOULDBLOCK)` is kept to retry short writes. This also reduces the `writev()` call sites to one. Fixes #3189 diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index a6914d3e6..fcb6af3ee 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -200,18 +200,8 @@ V1L_Flush(const struct worker *wrk) v1l->iov[v1l->ciov].iov_len = 0; } - i = writev(*v1l->wfd, v1l->iov, v1l->niov); - if (i > 0) - v1l->cnt += i; - while (i != v1l->liov && (i > 0 || errno == EWOULDBLOCK)) { - /* Remove sent data from start of I/O vector, - * then retry; we hit a timeout, and some data - * may have been sent. - * - * XXX: Add a "minimum sent data per timeout - * counter to prevent slowloris attacks - */ - + i = 0; + do { if (VTIM_real() > v1l->deadline) { VSLb(v1l->vsl, SLT_Debug, "Hit total send timeout, " @@ -221,16 +211,28 @@ V1L_Flush(const struct worker *wrk) break; } + i = writev(*v1l->wfd, v1l->iov, v1l->niov); + if (i > 0) + v1l->cnt += i; + + if (i == v1l->liov) + break; + + /* we hit a timeout, and some data may have been sent: + * Remove sent data from start of I/O vector, then retry + * + * XXX: Add a "minimum sent data per timeout counter to + * prevent slowloris attacks + */ + VSLb(v1l->vsl, SLT_Debug, "Hit idle send timeout, wrote = %zd/%zd; retrying", i, v1l->liov); if (i > 0) v1l_prune(v1l, i); - i = writev(*v1l->wfd, v1l->iov, v1l->niov); - if (i > 0) - v1l->cnt += i; - } + } while (i > 0 || errno == EWOULDBLOCK); + if (i <= 0) { VSLb(v1l->vsl, SLT_Debug, "Write error, retval = %zd, len = %zd, errno = %s", diff --git a/bin/varnishtest/tests/r03189.vtc b/bin/varnishtest/tests/r03189.vtc new file mode 100644 index 000000000..cf3d466c9 --- /dev/null +++ b/bin/varnishtest/tests/r03189.vtc @@ -0,0 +1,28 @@ +varnishtest "h1 send_timeout and streaming of dripping chunks" + +barrier b cond 2 + +server s1 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 1 + delay 1 + non_fatal + barrier b sync + chunkedlen 1 + delay 1 + chunkedlen 0 +} -start + +varnish v1 \ + -arg "-p idle_send_timeout=.1" \ + -arg "-p send_timeout=.8" \ + -vcl+backend { } -start + +client c1 { + txreq + rxresphdrs + rxchunk + barrier b sync + expect_close +} -run diff --git a/bin/varnishtest/tests/s00011.vtc b/bin/varnishtest/tests/s00011.vtc index 50e6e4612..11ff01e36 100644 --- a/bin/varnishtest/tests/s00011.vtc +++ b/bin/varnishtest/tests/s00011.vtc @@ -1,10 +1,8 @@ varnishtest "backend send timeouts" server s1 { + non_fatal rxreq - expect req.method == POST - expect req.body == helloworld - txresp } -start varnish v1 -vcl+backend "" -start @@ -21,5 +19,5 @@ client c1 { delay 2 send world rxresp - expect resp.status == 200 + expect resp.status == 503 } -run From dridi.boukelmoune at gmail.com Tue Mar 3 11:03:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 3 Mar 2020 11:03:07 +0000 (UTC) Subject: [master] 8c91e49d1 More documentation on automade VSCs Message-ID: <20200303110307.16AF5A2B1A@lists.varnish-cache.org> commit 8c91e49d1a516d86a1b84ac990c83c8d8cb3aa77 Author: Dridi Boukelmoune Date: Tue Mar 3 11:59:30 2020 +0100 More documentation on automade VSCs diff --git a/varnish.m4 b/varnish.m4 index 1c10e7659..2ede52bfc 100644 --- a/varnish.m4 +++ b/varnish.m4 @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2018 Varnish Software AS +# Copyright (c) 2016-2020 Varnish Software AS # All rights reserved. # # Author: Dridi Boukelmoune @@ -438,7 +438,7 @@ clean-vsc-$1: # @BUILD_VSC_BAR@ # # They take care of turning foo.vsc and bar.vsc into C code and RST -# documentation. +# documentation. Now let's imagine a vmod_baz uses the counters foo. # # Just like the vcc_*_if.[ch] files, you need to manually add the generated # sources to the appropriate VMODs: @@ -449,9 +449,14 @@ clean-vsc-$1: # VSC_foo.c \ # VSC_foo.h # +# Don't forget to add foo.vsc and bar.vsc to EXTRA_DIST. +# # You can then include the counters documentation somewhere in the VMOD's # VCC descriptor: # +# COUNTERS +# ======== +# # .. include:: VSC_foo.rst # # Doing so, you can add the generated RST as a dependency of the manual in From dridi.boukelmoune at gmail.com Tue Mar 3 11:12:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 3 Mar 2020 11:12:06 +0000 (UTC) Subject: [master] 9db32058b Kill dead assert Message-ID: <20200303111207.053E6A3018@lists.varnish-cache.org> commit 9db32058b4a98738dc26eded0cf5959719dc9fc1 Author: Dridi Boukelmoune Date: Tue Mar 3 12:07:40 2020 +0100 Kill dead assert The havemsg variable is purely local, and only written-then-read once. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 6064a9c08..0c8e221fb 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -175,7 +175,7 @@ VCL_Rel_CliCtx(struct vrt_ctx **ctx) static int vcl_send_event(struct vcl *vcl, enum vcl_event_e ev, struct vsb **msg) { - int r, havemsg = 0; + int r, havemsg; unsigned method = 0; struct vrt_ctx *ctx; @@ -197,7 +197,7 @@ vcl_send_event(struct vcl *vcl, enum vcl_event_e ev, struct vsb **msg) method = VCL_MET_FINI; /* FALLTHROUGH */ case VCL_EVENT_COLD: - AZ(havemsg); + havemsg = 0; break; default: WRONG("vcl_event"); From nils.goroll at uplex.de Tue Mar 3 12:27:40 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 3 Mar 2020 13:27:40 +0100 Subject: [master] 9db32058b Kill dead assert In-Reply-To: <20200303111207.053E6A3018@lists.varnish-cache.org> References: <20200303111207.053E6A3018@lists.varnish-cache.org> Message-ID: it's ok, but actually that bit _was_ intentional On 03/03/2020 12:12, Dridi Boukelmoune wrote: > > commit 9db32058b4a98738dc26eded0cf5959719dc9fc1 > Author: Dridi Boukelmoune > Date: Tue Mar 3 12:07:40 2020 +0100 > > Kill dead assert > > The havemsg variable is purely local, and only written-then-read once. > > diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c > index 6064a9c08..0c8e221fb 100644 > --- a/bin/varnishd/cache/cache_vcl.c > +++ b/bin/varnishd/cache/cache_vcl.c > @@ -175,7 +175,7 @@ VCL_Rel_CliCtx(struct vrt_ctx **ctx) > static int > vcl_send_event(struct vcl *vcl, enum vcl_event_e ev, struct vsb **msg) > { > - int r, havemsg = 0; > + int r, havemsg; > unsigned method = 0; > struct vrt_ctx *ctx; > > @@ -197,7 +197,7 @@ vcl_send_event(struct vcl *vcl, enum vcl_event_e ev, struct vsb **msg) > method = VCL_MET_FINI; > /* FALLTHROUGH */ > case VCL_EVENT_COLD: > - AZ(havemsg); > + havemsg = 0; > break; > default: > WRONG("vcl_event"); > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi at varni.sh Tue Mar 3 13:30:23 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 3 Mar 2020 13:30:23 +0000 Subject: [master] 9db32058b Kill dead assert In-Reply-To: References: <20200303111207.053E6A3018@lists.varnish-cache.org> Message-ID: On Tue, Mar 3, 2020 at 12:27 PM Nils Goroll wrote: > > it's ok, but actually that bit _was_ intentional Sorry, I didn't realize. What about turning enum vcl_event_e into a struct that has fields for both method and have_msg properties? That would be for 7.0 of course... Dridi From nils.goroll at uplex.de Tue Mar 3 13:34:35 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 3 Mar 2020 14:34:35 +0100 Subject: [master] 9db32058b Kill dead assert In-Reply-To: References: <20200303111207.053E6A3018@lists.varnish-cache.org> Message-ID: On 03/03/2020 14:30, Dridi Boukelmoune wrote: > What about turning enum vcl_event_e into a struct that has fields for > both method and have_msg properties? That would be for 7.0 of > course... I do not share the disgust for enums, but that could make the code cleaner/more readable. I would not touch the VMOD/VCL facing side though, the struct could still contain the vcl_event_e to be passed to the vmod event function. Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi at varni.sh Tue Mar 3 13:40:19 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 3 Mar 2020 13:40:19 +0000 Subject: [master] 9db32058b Kill dead assert In-Reply-To: References: <20200303111207.053E6A3018@lists.varnish-cache.org> Message-ID: On Tue, Mar 3, 2020 at 1:34 PM Nils Goroll wrote: > > On 03/03/2020 14:30, Dridi Boukelmoune wrote: > > What about turning enum vcl_event_e into a struct that has fields for > > both method and have_msg properties? That would be for 7.0 of > > course... > > I do not share the disgust for enums, but that could make the code cleaner/more > readable. Disgust was not the driver for this suggestion :) > I would not touch the VMOD/VCL facing side though, the struct could still > contain the vcl_event_e to be passed to the vmod event function. Interesting middle ground! Dridi From dridi.boukelmoune at gmail.com Wed Mar 4 10:09:09 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 4 Mar 2020 10:09:09 +0000 (UTC) Subject: [master] c22ccdcb2 Formally expose VSB_tofile in libvarnishapi Message-ID: <20200304100909.6C86B940C4@lists.varnish-cache.org> commit c22ccdcb2c605a78ecc07ffef694f3e0d5a34cc1 Author: Dridi Boukelmoune Date: Wed Mar 4 10:27:38 2020 +0100 Formally expose VSB_tofile in libvarnishapi Closes #3238 diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map index 4c0acb133..0ba4c8500 100644 --- a/lib/libvarnishapi/libvarnishapi.map +++ b/lib/libvarnishapi/libvarnishapi.map @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2011-2017 Varnish Software AS + * Copyright (c) 2011-2020 Varnish Software AS * All rights reserved. * * Author: Tollef Fog Heen @@ -172,7 +172,16 @@ LIBVARNISHAPI_2.2 { LIBVARNISHAPI_2.3 { /* 2019-09-15 release */ global: + # vcs.c VCS_String; local: *; }; + +LIBVARNISHAPI_2.4 { /* 2020-03-15 release */ + global: + # vsb.c + VSB_tofile; + local: + *; +}; From dridi.boukelmoune at gmail.com Wed Mar 4 10:09:09 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 4 Mar 2020 10:09:09 +0000 (UTC) Subject: [master] a368615f1 Finalize 6.3 release notes and upgrading guide Message-ID: <20200304100909.82588940C8@lists.varnish-cache.org> commit a368615f1039c65691a34468f89911b1f8f64b2e Author: Dridi Boukelmoune Date: Wed Mar 4 11:03:59 2020 +0100 Finalize 6.3 release notes and upgrading guide Better late than never. diff --git a/doc/sphinx/whats-new/changes-6.3.rst b/doc/sphinx/whats-new/changes-6.3.rst index d73a8c930..d51b0789d 100644 --- a/doc/sphinx/whats-new/changes-6.3.rst +++ b/doc/sphinx/whats-new/changes-6.3.rst @@ -1,7 +1,3 @@ -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` - .. _whatsnew_changes_6.3: %%%%%%%%%%%%%%%%%%%%%% diff --git a/doc/sphinx/whats-new/upgrading-6.3.rst b/doc/sphinx/whats-new/upgrading-6.3.rst index b813b9c44..fb9e90b73 100644 --- a/doc/sphinx/whats-new/upgrading-6.3.rst +++ b/doc/sphinx/whats-new/upgrading-6.3.rst @@ -1,19 +1,9 @@ -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` - .. _whatsnew_upgrading_6.3: %%%%%%%%%%%%%%%%%%%%%%%% Upgrading to Varnish 6.3 %%%%%%%%%%%%%%%%%%%%%%%% -**XXX: how to upgrade from previous deployments to this -version. Limited to work that has to be done for an upgrade, new -features are listed in "Changes". Explicitly mention what does *not* -have to be changed, especially in VCL. May include, but is not limited -to:** - For users of many and/or labeled VCLs ===================================== @@ -35,7 +25,6 @@ being used or being labeled), an ``auto`` vcl will remain ``auto``, and a ``cold`` / ``warm`` vcl will change state, but never become ``auto`` implicitly. - For developers and authors of VMODs and API clients =================================================== From dridi.boukelmoune at gmail.com Wed Mar 4 10:15:08 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 4 Mar 2020 10:15:08 +0000 (UTC) Subject: [6.3] 2c9632e4b upgrading 6.3: vcl auto state details Message-ID: <20200304101508.26CA394634@lists.varnish-cache.org> commit 2c9632e4b5b24badee30db38b17e88e00231b1a3 Author: Nils Goroll Date: Mon Feb 24 12:19:27 2020 +0100 upgrading 6.3: vcl auto state details diff --git a/doc/sphinx/whats-new/upgrading-6.3.rst b/doc/sphinx/whats-new/upgrading-6.3.rst index df269e077..b813b9c44 100644 --- a/doc/sphinx/whats-new/upgrading-6.3.rst +++ b/doc/sphinx/whats-new/upgrading-6.3.rst @@ -14,7 +14,27 @@ features are listed in "Changes". Explicitly mention what does *not* have to be changed, especially in VCL. May include, but is not limited to:** -TODO: a word on VCL temperature and the ``auto`` change. +For users of many and/or labeled VCLs +===================================== + +Users of the advanced mechanics behind the ``vcl.state`` CLI command +(most likely used via ``varnishadm``) should be aware of the following +changes, which may require adjustments to (or, more likely, allow for +simplifications of) scripts/programs interfacing with varnish: + +The VCL ``auto`` state has been streamlined. Conceptually, it used to +be a variant of the ``warm`` state which would automatically cool +the vcl. Yet, cooling did not only transition the temperature, but +also the state, so ``auto`` only worked one way - except that +``vcl.use`` or moving a label (by labeling another vcl) would also set +``auto``, so a manual warm/cold setting would get lost. + +Now the ``auto`` state will remain no matter the actual temperature or +labeling, so when a vcl needs to implicitly change temperature (due to +being used or being labeled), an ``auto`` vcl will remain ``auto``, +and a ``cold`` / ``warm`` vcl will change state, but never become +``auto`` implicitly. + For developers and authors of VMODs and API clients =================================================== From dridi.boukelmoune at gmail.com Wed Mar 4 10:15:08 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 4 Mar 2020 10:15:08 +0000 (UTC) Subject: [6.3] b108c7ee9 Finalize 6.3 release notes and upgrading guide Message-ID: <20200304101508.3C2ED94637@lists.varnish-cache.org> commit b108c7ee924b0877fdd3d2eb77c5de3e5bdf62f6 Author: Dridi Boukelmoune Date: Wed Mar 4 11:03:59 2020 +0100 Finalize 6.3 release notes and upgrading guide Better late than never. diff --git a/doc/sphinx/whats-new/changes-6.3.rst b/doc/sphinx/whats-new/changes-6.3.rst index d73a8c930..d51b0789d 100644 --- a/doc/sphinx/whats-new/changes-6.3.rst +++ b/doc/sphinx/whats-new/changes-6.3.rst @@ -1,7 +1,3 @@ -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` - .. _whatsnew_changes_6.3: %%%%%%%%%%%%%%%%%%%%%% diff --git a/doc/sphinx/whats-new/upgrading-6.3.rst b/doc/sphinx/whats-new/upgrading-6.3.rst index b813b9c44..fb9e90b73 100644 --- a/doc/sphinx/whats-new/upgrading-6.3.rst +++ b/doc/sphinx/whats-new/upgrading-6.3.rst @@ -1,19 +1,9 @@ -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` - .. _whatsnew_upgrading_6.3: %%%%%%%%%%%%%%%%%%%%%%%% Upgrading to Varnish 6.3 %%%%%%%%%%%%%%%%%%%%%%%% -**XXX: how to upgrade from previous deployments to this -version. Limited to work that has to be done for an upgrade, new -features are listed in "Changes". Explicitly mention what does *not* -have to be changed, especially in VCL. May include, but is not limited -to:** - For users of many and/or labeled VCLs ===================================== @@ -35,7 +25,6 @@ being used or being labeled), an ``auto`` vcl will remain ``auto``, and a ``cold`` / ``warm`` vcl will change state, but never become ``auto`` implicitly. - For developers and authors of VMODs and API clients =================================================== From dridi.boukelmoune at gmail.com Wed Mar 4 11:21:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 4 Mar 2020 11:21:07 +0000 (UTC) Subject: [master] da552667d Document semi-dynamic default value for accept_filter Message-ID: <20200304112107.1543F962F9@lists.varnish-cache.org> commit da552667d519e127d177ef5dd734820d4fe5164c Author: Dridi Boukelmoune Date: Wed Mar 4 12:17:24 2020 +0100 Document semi-dynamic default value for accept_filter diff --git a/include/tbl/params.h b/include/tbl/params.h index cf07a1052..16ab3d163 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -49,7 +49,7 @@ PARAM( /* type */ bool, /* min */ NULL, /* max */ NULL, - /* def */ "on", + /* def */ "on", /* default adjusted in mgt_param.c */ /* units */ "bool", /* descr */ "Enable kernel accept-filters. This may require a kernel module to " @@ -58,7 +58,10 @@ PARAM( "in the first place. Malformed requests may go unnoticed and not " "increase the client_req_400 counter. GET or HEAD requests with a " "body may be blocked altogether.", - /* flags */ XYZZY + /* flags */ XYZZY, + /* dyn_min_reason */ NULL, + /* dyn_max_reason */ NULL, + /* dyn_def_reason */ "on (if your platform supports accept filters)" ) #undef XYZZY From dridi.boukelmoune at gmail.com Wed Mar 4 11:21:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 4 Mar 2020 11:21:07 +0000 (UTC) Subject: [master] 0e13b705e Don't show an empty "Flags: " list for parameters Message-ID: <20200304112107.2E8AE962FC@lists.varnish-cache.org> commit 0e13b705eafdc2bfa1cc29a660dd75227e179212 Author: Dridi Boukelmoune Date: Wed Mar 4 12:18:57 2020 +0100 Don't show an empty "Flags: " list for parameters diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 27610d57c..bbd8700ec 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -781,7 +781,7 @@ MCF_DumpRstParam(void) * XXX: that say if ->min/->max are valid, so we * XXX: can emit those also in help texts. */ - if (pp->flags) { + if (pp->flags & ~NOT_IMPLEMENTED) { printf("\t* Flags: "); q = ""; From nils.goroll at uplex.de Thu Mar 5 08:58:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Mar 2020 08:58:08 +0000 (UTC) Subject: [master] 7c8a71a7e changelog tlc Message-ID: <20200305085808.89E8D62316@lists.varnish-cache.org> commit 7c8a71a7e34b099fc51da11429f8e4578e0f9342 Author: Nils Goroll Date: Thu Mar 5 09:56:44 2020 +0100 changelog tlc went through commits and dropped notes in running order, stopped at f402e5ff3063026f6ac779295b318da491a934a9 for now (had to switch tasks) diff --git a/doc/changes.rst b/doc/changes.rst index 22c14f6cc..daa78049a 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -38,10 +38,26 @@ NEXT (2020-03-15) * backend ``none`` was added for "no backend". +* ``std.rollback(bereq)`` is now safe to use, fixed bug 3009_ + +* Fixed ``varnishstat``, ``varnishtop`` and ``varnishhist`` handling + INT, TERM and HUP signals (bug 3088_) + +* The hash algorithm of the ``hash`` director was changed, so backend + selection will change once only when upgrading. Users of the + ``hash`` director are advised to consider using the ``shard`` + director, which, amongst other advantages, offers more stable + backend selection through consistent hashing. + +.. do we want to mention VSV00004 ? + * Log records can safely have empty fields or fields containing blanks if they are delimited by "double quotes". This was applied to ``SessError`` and ``Backend_health``. +.. 3109 (vdp gunzip) is probably irrelevant for users because it only + happens with gzip objects inserted bypassing the built-in vfps + * The option ``varnishtest -W`` is gone, the same can be achieved with ``varnishtest -p debug=+witness``. @@ -49,9 +65,25 @@ NEXT (2020-03-15) before the VCL subroutine, now it gets emitted after VCL returns for consistency with ``vcl_deliver {}``. +* Latencies for newly created worker threads to start work on + congested systems have been improved. + +* ``VRB_Iterate()`` signature has changed + +* ``VRT_fail()`` now also works from director code + +* deliberately closing backend requests through ``return(abandon)``, + ``return(fail)`` or ``return(error)`` is no longer accounted as a + fetch failure + +.. slink: continue at f402e5ff3063026f6ac779295b318da491a934a9 + * The ``if-range`` header is now handled, allowing clients to conditionally request a range based on a date or an ETag. +.. _3009: https://github.com/varnishcache/varnish-cache/issues/3009 +.. _3088: https://github.com/varnishcache/varnish-cache/issues/3088 + ================================ Varnish Cache 6.3.0 (2019-09-15) ================================ From dridi.boukelmoune at gmail.com Thu Mar 5 15:32:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 5 Mar 2020 15:32:07 +0000 (UTC) Subject: [master] 9f6df49f6 Speed up Travis CI builds Message-ID: <20200305153207.4C1F39706C@lists.varnish-cache.org> commit 9f6df49f61fb7d8b2deb178d37542319d1de43af Author: Dridi Boukelmoune Date: Thu Mar 5 16:27:01 2020 +0100 Speed up Travis CI builds The longest part is the test suite, so in order to release build machines sooner I hope that 8 concurrent jobs will strick a good balance between speed and resource consumption, considering that test cases mostly wait. That is still below what we do for Circle CI builds with 12 jobs for distcheck tasks. diff --git a/.travis.yml b/.travis.yml index 806d4f45a..f8a432b8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,9 +27,9 @@ jobs: script: &script-common - | if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then - make -j3 distcheck VERBOSE=1 + make -j8 distcheck VERBOSE=1 else - make -j3 check VERBOSE=1 + make -j8 check VERBOSE=1 fi - <<: *test-linux arch: arm64 From dridi at varni.sh Thu Mar 5 16:05:03 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 5 Mar 2020 16:05:03 +0000 Subject: [master] 9f6df49f6 Speed up Travis CI builds In-Reply-To: <20200305153207.4C1F39706C@lists.varnish-cache.org> References: <20200305153207.4C1F39706C@lists.varnish-cache.org> Message-ID: On Thu, Mar 5, 2020 at 3:32 PM Dridi Boukelmoune wrote: > > > commit 9f6df49f61fb7d8b2deb178d37542319d1de43af > Author: Dridi Boukelmoune > Date: Thu Mar 5 16:27:01 2020 +0100 > > Speed up Travis CI builds > > The longest part is the test suite, so in order to release build > machines sooner I hope that 8 concurrent jobs will strick a good > balance between speed and resource consumption, considering that > test cases mostly wait. Before: - 14m for x84_64 jobs - 20m for aarch64 jobs - 1h10 accumulated machine time After: - 6m for x84_64 jobs - 7m30 for aarch64 jobs - 28m accumulated machine time I think both Varnish Cache and Travis CI win. Dridi From nils.goroll at uplex.de Thu Mar 5 21:35:11 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Mar 2020 21:35:11 +0000 (UTC) Subject: [master] 7bebe5888 improve operator documentation Message-ID: <20200305213511.3E1EDA7340@lists.varnish-cache.org> commit 7bebe588852a511f428f857331ec1e0959115801 Author: Nils Goroll Date: Thu Mar 5 21:51:52 2020 +0100 improve operator documentation diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index ffd22bae9..0f9fe4a93 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -40,20 +40,25 @@ The following operators are available in VCL: ``=`` Assignment operator. - ``==`` - Comparison. + ``+=``, ``-=``, ``*=``, ``/=`` + Assign and increment/decrement/multiply/divide operator. - ``~`` - Match. Can either be used with regular expressions or ACLs. + For strings, ``+=`` appends. + + ``==``, ``!=``, ``<``, ``>``, ``<=``, ``>=`` + Comparisons + + ``~``, ``!~`` + Match / non-match. Can either be used with regular expressions or ACLs. ``!`` Negation. - ``&&`` - Logical and. + ``&&`` / ``||`` + Logical and/or. - ``||`` - Logical or. + ``<<``, ``>>`` + left/right bit-shift Conditionals From nils.goroll at uplex.de Thu Mar 5 21:35:11 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Mar 2020 21:35:11 +0000 (UTC) Subject: [master] 70328380b finish my first pass trough commits for the changelog Message-ID: <20200305213511.54EC6A7345@lists.varnish-cache.org> commit 70328380bc60f7b95ed986c6a4624a955288f316 Author: Nils Goroll Date: Thu Mar 5 22:31:44 2020 +0100 finish my first pass trough commits for the changelog this might need more love in places, but I wanted to get my first pass done and pushed before an ICE train arrives in Hamburg. I suggest we aim for a complete and correct changelog first before we rewrite/reformat it for the release- and upgrading notes. Please review and feel free to fix/improve or add additional changes which I left out or might have missed. diff --git a/doc/changes.rst b/doc/changes.rst index daa78049a..01a957933 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -40,8 +40,9 @@ NEXT (2020-03-15) * ``std.rollback(bereq)`` is now safe to use, fixed bug 3009_ -* Fixed ``varnishstat``, ``varnishtop`` and ``varnishhist`` handling - INT, TERM and HUP signals (bug 3088_) +* Fixed ``varnishstat``, ``varnishtop``, ``varnishhist`` and + ``varnishadm`` handling INT, TERM and HUP signals (bugs 3088_ and + 3229_) * The hash algorithm of the ``hash`` director was changed, so backend selection will change once only when upgrading. Users of the @@ -72,17 +73,116 @@ NEXT (2020-03-15) * ``VRT_fail()`` now also works from director code -* deliberately closing backend requests through ``return(abandon)``, +* Deliberately closing backend requests through ``return(abandon)``, ``return(fail)`` or ``return(error)`` is no longer accounted as a fetch failure -.. slink: continue at f402e5ff3063026f6ac779295b318da491a934a9 +* Fixed a bug which could cause probes not to run * The ``if-range`` header is now handled, allowing clients to conditionally request a range based on a date or an ETag. +* (Re)introduced struct reqtop to hold information on the ESI top + request and PRIV_TOP, fixed regression 3019_ + +* Allow numerical expressions in VCL to be negative / negated + +* Add vi-stype CTRL-f / CTRL-b for page down/up to interactive + varnishstat + +* Fixed wrong handling of an out-of-workspae condition in the proxy + vmod and in the workspace allocator, bug 3131_ + +* Raised the minimum for the ``vcl_cooldown`` parameter to 1s to fix + bug 3135_ + +* Improved creation of additional threads when none are available + +* Fixed a race between director creation and the ``backend.list`` CLI + command - see bug 3094_ + +* Added error handling to avoid panics for workspace overflows during + session attribute allocation - bug 3145_ + +* Overloaded the ``+=`` operator to also append to headers + +* Fixed set ``*.body`` commands. + +* Fixed status for truncated CLI responses, bug 3038_ + +* Output VCC warnings also for VCLs loaded via the ``varnishd -f`` + option, see bug 3160_ + +* Improved fetch error handling when stale objects are present in + cache, see bug 3089_ + +* Added a ``Notice`` VSL tag (used for ``varnishlog`` logging) + +.. mention #3176? (backend cooling straightened out) + +* Fixed session close reason reporting and accounting, added + ``rx_close_idle`` counter for separate accounting when + ``timeout_idle`` is reached. + +* Fixed handling of request bodies for backend retries + +* Fix deadlocks when the maximum number of threads has been reached, + in particular with http/2, see 2418_ + +* Add more vcl control over timeouts with ``sess.timeout_linger``, + ``sess.send_timeout`` and ``sess.idle_send_timeout`` + +* Fix panics due to missing EINVAL handling on MacOS, see 1853_ + +* Added ``VSLs()`` and ``VSLbs()`` functions for logging ``STRANDS`` to + VSL + +* Fixed cases where a workspace overflow would not result in a VCL + failure, see 3194_ + +* Added ``WS_VSB_new()`` / ``WS_VSB_finish()`` for VSBs on workspaces + +* Imported ``vmod_cookie`` from `varnish_modules`_ + +* ``body_status`` and ``req_body_status`` have been collapsed into one + type. In particular, the ``REQ_BODY_*`` enums now have been replaced + with ``BS_*``. + +.. mention VSB_QUOTE_GLOB ? + +* Fixed an old regression of the ``Age:`` header for passes, see bug + 3221_ + +* Added ``VRT_AllocStrandsWS()`` as a utility function to allocate + STRANDS on a workspace. + +* Reduced compile time of ``vcl_init{}`` / ``vcl_fini{}`` with gcc, + added ``v_dont_optimize`` attribute macro + +* Fixed a case where ``send_timeout`` would have no effect when + streaming from a backend fetch, see bug 3189_ + + *NOTE* Users upgrading varnish should re-check ``send_timeout`` with + respect to long pass and streaming fetches and watch out for + increased session close rates. + .. _3009: https://github.com/varnishcache/varnish-cache/issues/3009 .. _3088: https://github.com/varnishcache/varnish-cache/issues/3088 +.. _3019: https://github.com/varnishcache/varnish-cache/issues/3019 +.. _3131: https://github.com/varnishcache/varnish-cache/issues/3131 +.. _3135: https://github.com/varnishcache/varnish-cache/issues/3135 +.. _3094: https://github.com/varnishcache/varnish-cache/issues/3094 +.. _3145: https://github.com/varnishcache/varnish-cache/issues/3145 +.. _3038: https://github.com/varnishcache/varnish-cache/issues/3038 +.. _3160: https://github.com/varnishcache/varnish-cache/issues/3160 +.. _3089: https://github.com/varnishcache/varnish-cache/issues/3089 +.. _2418: https://github.com/varnishcache/varnish-cache/issues/2418 +.. _1853: https://github.com/varnishcache/varnish-cache/issues/1853 +.. _3194: https://github.com/varnishcache/varnish-cache/issues/3194 +.. _varnish_modules: https://github.com/varnish/varnish-modules +.. _3221: https://github.com/varnishcache/varnish-cache/issues/3221 +.. _3229: https://github.com/varnishcache/varnish-cache/issues/3229 +.. _3189: https://github.com/varnishcache/varnish-cache/issues/3189 ================================ Varnish Cache 6.3.0 (2019-09-15) From dridi.boukelmoune at gmail.com Fri Mar 6 07:11:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 6 Mar 2020 07:11:07 +0000 (UTC) Subject: [master] dc8a987da VCL doesn't have bit-shift operators Message-ID: <20200306071107.32691BE6EF@lists.varnish-cache.org> commit dc8a987da37e8fd90306c0871dff8297b502349d Author: Dridi Boukelmoune Date: Fri Mar 6 08:10:34 2020 +0100 VCL doesn't have bit-shift operators diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 0f9fe4a93..a532c96fd 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -57,9 +57,6 @@ The following operators are available in VCL: ``&&`` / ``||`` Logical and/or. - ``<<``, ``>>`` - left/right bit-shift - Conditionals ------------ From dridi at varni.sh Fri Mar 6 07:16:23 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Fri, 6 Mar 2020 07:16:23 +0000 Subject: [master] 70328380b finish my first pass trough commits for the changelog In-Reply-To: <20200305213511.54EC6A7345@lists.varnish-cache.org> References: <20200305213511.54EC6A7345@lists.varnish-cache.org> Message-ID: On Thu, Mar 5, 2020 at 9:35 PM Nils Goroll wrote: > > > commit 70328380bc60f7b95ed986c6a4624a955288f316 > Author: Nils Goroll > Date: Thu Mar 5 22:31:44 2020 +0100 > > finish my first pass trough commits for the changelog > > this might need more love in places, but I wanted to get my first > pass done and pushed before an ICE train arrives in Hamburg. > > I suggest we aim for a complete and correct changelog first before we > rewrite/reformat it for the release- and upgrading notes. > > Please review and feel free to fix/improve or add additional changes > which I left out or might have missed. > > diff --git a/doc/changes.rst b/doc/changes.rst > index daa78049a..01a957933 100644 > --- a/doc/changes.rst > +++ b/doc/changes.rst > @@ -40,8 +40,9 @@ NEXT (2020-03-15) > > * ``std.rollback(bereq)`` is now safe to use, fixed bug 3009_ > > -* Fixed ``varnishstat``, ``varnishtop`` and ``varnishhist`` handling > - INT, TERM and HUP signals (bug 3088_) > +* Fixed ``varnishstat``, ``varnishtop``, ``varnishhist`` and > + ``varnishadm`` handling INT, TERM and HUP signals (bugs 3088_ and > + 3229_) > > * The hash algorithm of the ``hash`` director was changed, so backend > selection will change once only when upgrading. Users of the > @@ -72,17 +73,116 @@ NEXT (2020-03-15) > > * ``VRT_fail()`` now also works from director code > > -* deliberately closing backend requests through ``return(abandon)``, > +* Deliberately closing backend requests through ``return(abandon)``, > ``return(fail)`` or ``return(error)`` is no longer accounted as a > fetch failure > > -.. slink: continue at f402e5ff3063026f6ac779295b318da491a934a9 > +* Fixed a bug which could cause probes not to run > > * The ``if-range`` header is now handled, allowing clients to conditionally > request a range based on a date or an ETag. > > +* (Re)introduced struct reqtop to hold information on the ESI top > + request and PRIV_TOP, fixed regression 3019_ I think we can drop the "Re" part, struct reqtop never made it to a release before. I'll do my own sweep later this morning, thanks! > + > +* Allow numerical expressions in VCL to be negative / negated > + > +* Add vi-stype CTRL-f / CTRL-b for page down/up to interactive > + varnishstat > + > +* Fixed wrong handling of an out-of-workspae condition in the proxy > + vmod and in the workspace allocator, bug 3131_ > + > +* Raised the minimum for the ``vcl_cooldown`` parameter to 1s to fix > + bug 3135_ > + > +* Improved creation of additional threads when none are available > + > +* Fixed a race between director creation and the ``backend.list`` CLI > + command - see bug 3094_ > + > +* Added error handling to avoid panics for workspace overflows during > + session attribute allocation - bug 3145_ > + > +* Overloaded the ``+=`` operator to also append to headers > + > +* Fixed set ``*.body`` commands. > + > +* Fixed status for truncated CLI responses, bug 3038_ > + > +* Output VCC warnings also for VCLs loaded via the ``varnishd -f`` > + option, see bug 3160_ > + > +* Improved fetch error handling when stale objects are present in > + cache, see bug 3089_ > + > +* Added a ``Notice`` VSL tag (used for ``varnishlog`` logging) > + > +.. mention #3176? (backend cooling straightened out) > + > +* Fixed session close reason reporting and accounting, added > + ``rx_close_idle`` counter for separate accounting when > + ``timeout_idle`` is reached. > + > +* Fixed handling of request bodies for backend retries > + > +* Fix deadlocks when the maximum number of threads has been reached, > + in particular with http/2, see 2418_ > + > +* Add more vcl control over timeouts with ``sess.timeout_linger``, > + ``sess.send_timeout`` and ``sess.idle_send_timeout`` > + > +* Fix panics due to missing EINVAL handling on MacOS, see 1853_ > + > +* Added ``VSLs()`` and ``VSLbs()`` functions for logging ``STRANDS`` to > + VSL > + > +* Fixed cases where a workspace overflow would not result in a VCL > + failure, see 3194_ > + > +* Added ``WS_VSB_new()`` / ``WS_VSB_finish()`` for VSBs on workspaces > + > +* Imported ``vmod_cookie`` from `varnish_modules`_ > + > +* ``body_status`` and ``req_body_status`` have been collapsed into one > + type. In particular, the ``REQ_BODY_*`` enums now have been replaced > + with ``BS_*``. > + > +.. mention VSB_QUOTE_GLOB ? > + > +* Fixed an old regression of the ``Age:`` header for passes, see bug > + 3221_ > + > +* Added ``VRT_AllocStrandsWS()`` as a utility function to allocate > + STRANDS on a workspace. > + > +* Reduced compile time of ``vcl_init{}`` / ``vcl_fini{}`` with gcc, > + added ``v_dont_optimize`` attribute macro > + > +* Fixed a case where ``send_timeout`` would have no effect when > + streaming from a backend fetch, see bug 3189_ > + > + *NOTE* Users upgrading varnish should re-check ``send_timeout`` with > + respect to long pass and streaming fetches and watch out for > + increased session close rates. > + > .. _3009: https://github.com/varnishcache/varnish-cache/issues/3009 > .. _3088: https://github.com/varnishcache/varnish-cache/issues/3088 > +.. _3019: https://github.com/varnishcache/varnish-cache/issues/3019 > +.. _3131: https://github.com/varnishcache/varnish-cache/issues/3131 > +.. _3135: https://github.com/varnishcache/varnish-cache/issues/3135 > +.. _3094: https://github.com/varnishcache/varnish-cache/issues/3094 > +.. _3145: https://github.com/varnishcache/varnish-cache/issues/3145 > +.. _3038: https://github.com/varnishcache/varnish-cache/issues/3038 > +.. _3160: https://github.com/varnishcache/varnish-cache/issues/3160 > +.. _3089: https://github.com/varnishcache/varnish-cache/issues/3089 > +.. _2418: https://github.com/varnishcache/varnish-cache/issues/2418 > +.. _1853: https://github.com/varnishcache/varnish-cache/issues/1853 > +.. _3194: https://github.com/varnishcache/varnish-cache/issues/3194 > +.. _varnish_modules: https://github.com/varnish/varnish-modules > +.. _3221: https://github.com/varnishcache/varnish-cache/issues/3221 > +.. _3229: https://github.com/varnishcache/varnish-cache/issues/3229 > +.. _3189: https://github.com/varnishcache/varnish-cache/issues/3189 > > ================================ > Varnish Cache 6.3.0 (2019-09-15) > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Fri Mar 6 08:28:05 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 6 Mar 2020 09:28:05 +0100 Subject: [master] dc8a987da VCL doesn't have bit-shift operators In-Reply-To: <20200306071107.32691BE6EF@lists.varnish-cache.org> References: <20200306071107.32691BE6EF@lists.varnish-cache.org> Message-ID: Thanks, I only looked at the tokenizer code On 06/03/2020 08:11, Dridi Boukelmoune wrote: > > commit dc8a987da37e8fd90306c0871dff8297b502349d > Author: Dridi Boukelmoune > Date: Fri Mar 6 08:10:34 2020 +0100 > > VCL doesn't have bit-shift operators > > diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst > index 0f9fe4a93..a532c96fd 100644 > --- a/doc/sphinx/reference/vcl.rst > +++ b/doc/sphinx/reference/vcl.rst > @@ -57,9 +57,6 @@ The following operators are available in VCL: > ``&&`` / ``||`` > Logical and/or. > > - ``<<``, ``>>`` > - left/right bit-shift > - > > Conditionals > ------------ > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Fri Mar 6 08:31:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 6 Mar 2020 08:31:06 +0000 (UTC) Subject: [master] a7a0be7e3 minor changelog polish Message-ID: <20200306083106.A7752C046D@lists.varnish-cache.org> commit a7a0be7e3879501102e74616f43b317eaa56f000 Author: Nils Goroll Date: Fri Mar 6 09:30:48 2020 +0100 minor changelog polish diff --git a/doc/changes.rst b/doc/changes.rst index 01a957933..8547d1323 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -82,8 +82,8 @@ NEXT (2020-03-15) * The ``if-range`` header is now handled, allowing clients to conditionally request a range based on a date or an ETag. -* (Re)introduced struct reqtop to hold information on the ESI top - request and PRIV_TOP, fixed regression 3019_ +* Introduced struct reqtop to hold information on the ESI top request + and PRIV_TOP, fixed regression 3019_ * Allow numerical expressions in VCL to be negative / negated From dridi.boukelmoune at gmail.com Fri Mar 6 10:03:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 6 Mar 2020 10:03:07 +0000 (UTC) Subject: [master] 647d389c9 Mention dynamic parameter bounds changes in 6.4 Message-ID: <20200306100307.1572B78CE@lists.varnish-cache.org> commit 647d389c9a4803a58008d0e87066d60f741be8d8 Author: Dridi Boukelmoune Date: Fri Mar 6 11:00:06 2020 +0100 Mention dynamic parameter bounds changes in 6.4 diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index fbda9d6b9..d0221e616 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -23,6 +23,30 @@ varnishd Parameters ~~~~~~~~~~ +Some parameters have dependencies and those are better documented now. For +example :ref:`ref_param_thread_pool_min` can't be increased above +:ref:`ref_param_thread_pool_max`, which is now indicated as such in the +manual. + +On a running Varnish instance the ``param.show`` command will display the +actual minimum or maximum, but an attempt to ``param.set`` a parameter above +or below its dynamic maximum or minimum will mention the failure's cause in +the error message:: + + varnish> param.show thread_pool_reserve + 200 + thread_pool_reserve + Value is: 0 [threads] (default) + Maximum is: 95 + + [...] + + varnish> param.set thread_pool_reserve 100 + 106 + Must be no more than 95 (95% of thread_pool_min) + + (attempting to set param 'thread_pool_reserve' to '100') + **XXX changes in -p parameters** Other changes in varnishd From nils.goroll at uplex.de Sat Mar 7 10:55:09 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 7 Mar 2020 10:55:09 +0000 (UTC) Subject: [master] ceada65ea make it a bit more obvious which parameters are defined outside params.h Message-ID: <20200307105509.4E494B114C@lists.varnish-cache.org> commit ceada65ea7bae8b7d0f3e66f1b4614d772929bd9 Author: Nils Goroll Date: Sat Mar 7 11:53:46 2020 +0100 make it a bit more obvious which parameters are defined outside params.h diff --git a/include/tbl/params.h b/include/tbl/params.h index 16ab3d163..b75bc9c68 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1199,7 +1199,7 @@ PARAM( "HTTP2 maximum size of an uncompressed header list." ) -#if 0 +#if 0 /* NOT ACTUALLY DEFINED HERE */ /* actual location mgt_param_bits.c*/ /* see tbl/debug_bits.h */ PARAM( @@ -1593,7 +1593,7 @@ PARAM( "Use +/- prefix in front of VSL tag name to unmask/mask " "individual VSL messages." ) -#endif +#endif /* NOT ACTUALLY DEFINED HERE */ #undef PARAM From fgsch at lodoss.net Sat Mar 7 19:25:10 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 7 Mar 2020 19:25:10 +0000 (UTC) Subject: [master] 608b5ac05 Stab at fixing macos travis job Message-ID: <20200307192510.D6BF7640DC@lists.varnish-cache.org> commit 608b5ac05ccab5e39ef720bdf33f49e74eb211f5 Author: Federico G. Schwindt Date: Sat Mar 7 19:23:25 2020 +0000 Stab at fixing macos travis job diff --git a/.travis.yml b/.travis.yml index f8a432b8a..b4154d50e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,6 +77,7 @@ jobs: - docutils - nghttp2 - sphinx-doc + update: true before_script: - export PATH="/usr/local/opt/sphinx-doc/bin:$PATH" - ./autogen.sh From fgsch at lodoss.net Sat Mar 7 19:39:06 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 7 Mar 2020 19:39:06 +0000 (UTC) Subject: [master] 40f32e5b2 Fix macos travis job, take 2 Message-ID: <20200307193906.957BB64688@lists.varnish-cache.org> commit 40f32e5b2e1c7af77b7d1833a3fbc8410fd84540 Author: Federico G. Schwindt Date: Sat Mar 7 19:37:58 2020 +0000 Fix macos travis job, take 2 diff --git a/.travis.yml b/.travis.yml index b4154d50e..1e236fa82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,6 +80,7 @@ jobs: update: true before_script: - export PATH="/usr/local/opt/sphinx-doc/bin:$PATH" + - brew link --overwrite python - ./autogen.sh - ./configure script: *script-common From fgsch at lodoss.net Sat Mar 7 20:14:09 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 7 Mar 2020 20:14:09 +0000 (UTC) Subject: [master] 5ed9cb9d0 Update osx image Message-ID: <20200307201409.5E9F06E537@lists.varnish-cache.org> commit 5ed9cb9d0237e61be03e3a30edd17bfac6f7e7e8 Author: Federico G. Schwindt Date: Sat Mar 7 20:11:38 2020 +0000 Update osx image diff --git a/.travis.yml b/.travis.yml index 1e236fa82..5632b3c33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,7 +69,7 @@ jobs: - ./configure --with-unwind --enable-developer-warnings --enable-debugging-symbols --disable-stack-protector --with-persistent-storage --enable-asan --enable-ubsan - stage: test os: osx - osx_image: xcode10.2 + osx_image: xcode11.3 compiler: clang addons: homebrew: @@ -80,7 +80,6 @@ jobs: update: true before_script: - export PATH="/usr/local/opt/sphinx-doc/bin:$PATH" - - brew link --overwrite python - ./autogen.sh - ./configure script: *script-common From dridi.boukelmoune at gmail.com Mon Mar 9 04:53:08 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 9 Mar 2020 04:53:08 +0000 (UTC) Subject: [master] e7cf8dc3c Polish Message-ID: <20200309045308.8E2B995A83@lists.varnish-cache.org> commit e7cf8dc3caa112e7a8686ca8ef1ec5c5742cbf46 Author: Dridi Boukelmoune Date: Mon Mar 9 05:51:43 2020 +0100 Polish diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 3da9ba732..c563c29c6 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -396,9 +396,9 @@ WS_VSB_new(struct vsb *vsb, struct ws *ws) WS_Assert(ws); u = WS_ReserveAll(ws); if (WS_Overflowed(ws) || u < 2) - AN(VSB_new(vsb, bogus, sizeof bogus, 0)); + AN(VSB_new(vsb, bogus, sizeof bogus, VSB_FIXEDLEN)); else - AN(VSB_new(vsb, WS_Front(ws), u, 0)); + AN(VSB_new(vsb, WS_Front(ws), u, VSB_FIXEDLEN)); } char * From nils.goroll at uplex.de Mon Mar 9 14:46:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 9 Mar 2020 14:46:07 +0000 (UTC) Subject: [master] 4e8b826f7 Revert "Simplify std.tolower() and std.toupper() with WS_VSB" Message-ID: <20200309144607.730B1AADD6@lists.varnish-cache.org> commit 4e8b826f750bcb3a2b5b803eb23db948aba66d3f Author: Nils Goroll Date: Mon Mar 9 15:44:48 2020 +0100 Revert "Simplify std.tolower() and std.toupper() with WS_VSB" This reverts commit 0c3ec169282427e629f211dc17c4afd1df2bb9da. as decided by bugwash Ref #3218 diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 1fe02fdcf..51a9b7e16 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -46,7 +46,6 @@ #include "vrnd.h" #include "vtcp.h" #include "vsa.h" -#include "vsb.h" #include "vtim.h" #include "vcl.h" @@ -67,34 +66,56 @@ vmod_set_ip_tos(VRT_CTX, VCL_INT tos) IPPROTO_IP, IP_TOS, &itos, sizeof(itos))); } -VCL_STRING v_matchproto_(td_std_toupper) -vmod_toupper(VRT_CTX, VCL_STRANDS s) +static const char * +vmod_updown(VRT_CTX, int up, VCL_STRANDS s) { + unsigned u; + char *b, *e; const char *p; - struct vsb vsb[1]; int i; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - WS_VSB_new(vsb, ctx->ws); - for (i = 0; i < s->n; i++) - for (p = s->p[i]; p != NULL && *p != '\0'; p++) - VSB_putc(vsb, (char)toupper(*p)); - return (WS_VSB_finish(vsb, ctx->ws, NULL)); + u = WS_ReserveAll(ctx->ws); + e = b = ctx->ws->f; + e += u; + for (i = 0; i < s->n && b < e; i++) { + p = s->p[i]; + while (p != NULL && *p != '\0' && b < e) { + if (up) + *b++ = (char)toupper(*p++); + else + *b++ = (char)tolower(*p++); + } + } + if (b < e) + *b = '\0'; + b++; + if (b > e) { + WS_MarkOverflow(ctx->ws); + WS_Release(ctx->ws, 0); + return (NULL); + } else { + e = b; + b = ctx->ws->f; + WS_Release(ctx->ws, e - b); + return (b); + } +} + +VCL_STRING v_matchproto_(td_std_toupper) +vmod_toupper(VRT_CTX, VCL_STRANDS s) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + return (vmod_updown(ctx, 1, s)); } VCL_STRING v_matchproto_(td_std_tolower) vmod_tolower(VRT_CTX, VCL_STRANDS s) { - const char *p; - struct vsb vsb[1]; - int i; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - WS_VSB_new(vsb, ctx->ws); - for (i = 0; i < s->n; i++) - for (p = s->p[i]; p != NULL && *p != '\0'; p++) - VSB_putc(vsb, (char)tolower(*p)); - return (WS_VSB_finish(vsb, ctx->ws, NULL)); + return (vmod_updown(ctx, 0, s)); } VCL_REAL v_matchproto_(td_std_random) From nils.goroll at uplex.de Mon Mar 9 17:10:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 9 Mar 2020 17:10:08 +0000 (UTC) Subject: [master] 3f5de2a97 fix doc leftovers from removed cookie.filter_except() function Message-ID: <20200309171008.3790CAE4CF@lists.varnish-cache.org> commit 3f5de2a972048d7b618c36930885e43aafd0811b Author: Nils Goroll Date: Mon Mar 9 18:09:24 2020 +0100 fix doc leftovers from removed cookie.filter_except() function diff --git a/lib/libvmod_cookie/vmod.vcc b/lib/libvmod_cookie/vmod.vcc index f4263aebd..5c4bf494a 100644 --- a/lib/libvmod_cookie/vmod.vcc +++ b/lib/libvmod_cookie/vmod.vcc @@ -6,7 +6,7 @@ DESCRIPTION Handle HTTP cookies more easily in Varnish VCL. Parses a cookie header into an internal data store, where per-cookie -get/set/delete functions are available. A filter_except() method removes all +get/set/delete functions are available. A keep() method removes all but a set comma-separated list of cookies. A filter() method removes a comma- separated list of cookies. @@ -178,7 +178,7 @@ Example:: sub vcl_recv { cookie.parse(req.http.cookie); - cookie.filter_except("SESSIONID,PHPSESSID"); + cookie.keep("SESSIONID,PHPSESSID"); set req.http.cookie = cookie.get_string(); } From nils.goroll at uplex.de Mon Mar 9 17:12:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 9 Mar 2020 17:12:07 +0000 (UTC) Subject: [master] f2afc54c4 changelog cookie.filter_except Message-ID: <20200309171207.6D9F4AE6FF@lists.varnish-cache.org> commit f2afc54c4a242243c79113afcab35ceb7905b76e Author: Nils Goroll Date: Mon Mar 9 18:10:56 2020 +0100 changelog cookie.filter_except diff --git a/doc/changes.rst b/doc/changes.rst index 8547d1323..908804043 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -144,6 +144,9 @@ NEXT (2020-03-15) * Imported ``vmod_cookie`` from `varnish_modules`_ + The previously deprecated function ``cookie.filter_except()`` has + been removed during import. It was replaced by ``cookie.keep()`` + * ``body_status`` and ``req_body_status`` have been collapsed into one type. In particular, the ``REQ_BODY_*`` enums now have been replaced with ``BS_*``. From dridi at varni.sh Mon Mar 9 19:16:54 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 9 Mar 2020 19:16:54 +0000 Subject: [master] 3f5de2a97 fix doc leftovers from removed cookie.filter_except() function In-Reply-To: <20200309171008.3790CAE4CF@lists.varnish-cache.org> References: <20200309171008.3790CAE4CF@lists.varnish-cache.org> Message-ID: On Mon, Mar 9, 2020 at 5:10 PM Nils Goroll wrote: > > > commit 3f5de2a972048d7b618c36930885e43aafd0811b > Author: Nils Goroll > Date: Mon Mar 9 18:09:24 2020 +0100 > > fix doc leftovers from removed cookie.filter_except() function > > diff --git a/lib/libvmod_cookie/vmod.vcc b/lib/libvmod_cookie/vmod.vcc > index f4263aebd..5c4bf494a 100644 > --- a/lib/libvmod_cookie/vmod.vcc > +++ b/lib/libvmod_cookie/vmod.vcc > @@ -6,7 +6,7 @@ DESCRIPTION > Handle HTTP cookies more easily in Varnish VCL. > > Parses a cookie header into an internal data store, where per-cookie > -get/set/delete functions are available. A filter_except() method removes all > +get/set/delete functions are available. A keep() method removes all > but a set comma-separated list of cookies. A filter() method removes a comma- > separated list of cookies. Nils, I didn't notice until now: could you please replace occurrences of "method" where it refers to functions while you are in the neighborhood? > @@ -178,7 +178,7 @@ Example:: > > sub vcl_recv { > cookie.parse(req.http.cookie); > - cookie.filter_except("SESSIONID,PHPSESSID"); > + cookie.keep("SESSIONID,PHPSESSID"); > set req.http.cookie = cookie.get_string(); > } > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Mon Mar 9 19:27:15 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 9 Mar 2020 20:27:15 +0100 Subject: [master] 3f5de2a97 fix doc leftovers from removed cookie.filter_except() function In-Reply-To: References: <20200309171008.3790CAE4CF@lists.varnish-cache.org> Message-ID: On 09/03/2020 20:16, Dridi Boukelmoune wrote: > Nils, I didn't notice until now: could you please replace occurrences > of "method" where it refers to functions while you are in the > neighborhood? good point, thanks -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Mon Mar 9 19:34:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 9 Mar 2020 19:34:08 +0000 (UTC) Subject: [master] e49de3983 vmod_cookie provides functions, not methods Message-ID: <20200309193408.4DACEB1FC3@lists.varnish-cache.org> commit e49de39838b9f1662b2bb5759499d89b42769a53 Author: Nils Goroll Date: Mon Mar 9 20:33:02 2020 +0100 vmod_cookie provides functions, not methods spotted by @Dridi diff --git a/lib/libvmod_cookie/vmod.vcc b/lib/libvmod_cookie/vmod.vcc index 5c4bf494a..321f544b9 100644 --- a/lib/libvmod_cookie/vmod.vcc +++ b/lib/libvmod_cookie/vmod.vcc @@ -6,8 +6,8 @@ DESCRIPTION Handle HTTP cookies more easily in Varnish VCL. Parses a cookie header into an internal data store, where per-cookie -get/set/delete functions are available. A keep() method removes all -but a set comma-separated list of cookies. A filter() method removes a comma- +get/set/delete functions are available. A keep() function removes all +but a set comma-separated list of cookies. A filter() function removes a comma- separated list of cookies. Regular expressions can be used for either selecting cookies, deleting matching From dridi.boukelmoune at gmail.com Mon Mar 9 19:52:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 9 Mar 2020 19:52:07 +0000 (UTC) Subject: [4.0] 8aa540ac5 What happens with LaTeX stays with LaTeX Message-ID: <20200309195207.E9A65B26F6@lists.varnish-cache.org> commit 8aa540ac5bd100b611b052e2f9abdeed96ba991a Author: Dridi Boukelmoune Date: Mon Mar 9 20:49:49 2020 +0100 What happens with LaTeX stays with LaTeX Better diff with the --word-diff option. diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 31eed388f..c05341ebc 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -10,7 +10,7 @@ BUILDDIR = build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(distdir) +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) $(distdir) .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest @@ -66,7 +66,7 @@ qthelp: @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Varnish.qhc" latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + $(SPHINXBUILD) -b latex $(PAPEROPT_$(PAPER)) $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ From nils.goroll at uplex.de Tue Mar 10 09:21:09 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 09:21:09 +0000 (UTC) Subject: [master] 0c645dbf0 give vbp_mtx its own lock class Message-ID: <20200310092109.801F49693D@lists.varnish-cache.org> commit 0c645dbf093f76bf75fed05bc804d9e5cc6b7bd7 Author: Nils Goroll Date: Tue Mar 10 10:17:21 2020 +0100 give vbp_mtx its own lock class We used lck_backend both for vbp_mtx and for the mtx on individual backends. This lead to confusing results of the brinch hansons arrows produced by make witness. diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 23721fb94..8d0bc41f7 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -748,7 +748,7 @@ VBP_Init(void) { pthread_t thr; - Lck_New(&vbp_mtx, lck_backend); + Lck_New(&vbp_mtx, lck_probe); vbp_heap = binheap_new(NULL, vbp_cmp, vbp_update); AN(vbp_heap); AZ(pthread_cond_init(&vbp_cond, NULL)); diff --git a/include/tbl/locks.h b/include/tbl/locks.h index c3852dad8..026c24253 100644 --- a/include/tbl/locks.h +++ b/include/tbl/locks.h @@ -41,6 +41,7 @@ LOCK(lru) LOCK(mempool) LOCK(objhdr) LOCK(pipestat) +LOCK(probe) LOCK(sess) LOCK(tcp_pool) LOCK(vbe) From nils.goroll at uplex.de Tue Mar 10 09:33:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 09:33:06 +0000 (UTC) Subject: [master] 8f35a32d6 Also give the per-pool lock its own class Message-ID: <20200310093306.83B8896FA6@lists.varnish-cache.org> commit 8f35a32d6b4e4d90135708fdfba214e484126ed3 Author: Nils Goroll Date: Tue Mar 10 10:31:39 2020 +0100 Also give the per-pool lock its own class (see previous) diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index c5991c634..bfdddccb0 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -146,7 +146,7 @@ pool_mkpool(unsigned pool_no) AN(pp->a_stat); pp->b_stat = calloc(1, sizeof *pp->b_stat); AN(pp->b_stat); - Lck_New(&pp->mtx, lck_wq); + Lck_New(&pp->mtx, lck_perpool); VTAILQ_INIT(&pp->idle_queue); VTAILQ_INIT(&pp->poolsocks); diff --git a/include/tbl/locks.h b/include/tbl/locks.h index 026c24253..ca0ed0427 100644 --- a/include/tbl/locks.h +++ b/include/tbl/locks.h @@ -40,6 +40,7 @@ LOCK(hcb) LOCK(lru) LOCK(mempool) LOCK(objhdr) +LOCK(perpool) LOCK(pipestat) LOCK(probe) LOCK(sess) From dridi.boukelmoune at gmail.com Tue Mar 10 09:53:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 10 Mar 2020 09:53:06 +0000 (UTC) Subject: [master] 11d0f63d1 First attempt at a Travis CI witness job Message-ID: <20200310095306.CCBF9978C2@lists.varnish-cache.org> commit 11d0f63d146c4dda99e65650d23ea7e274a327e6 Author: Dridi Boukelmoune Date: Tue Mar 10 10:48:47 2020 +0100 First attempt at a Travis CI witness job At this point I have no idea what I'm doing and no motivation to learn further YAML so I'll do something I hate: trigger a CI build and resume what I was working on until it completes. Refs 0c645dbf093f diff --git a/.travis.yml b/.travis.yml index 5632b3c33..bf31b432b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,8 @@ jobs: before_script: - ./autogen.sh - ./configure + - <<: *test-linux + script: make -j8 witness - <<: *test-linux stage: sanitizers addons: From nils.goroll at uplex.de Tue Mar 10 09:57:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 09:57:06 +0000 (UTC) Subject: [master] f42141f2d drop the probe lock while scheduling a probe Message-ID: <20200310095706.8A51C97BC1@lists.varnish-cache.org> commit f42141f2d99a3a6e1c34641a3778058afdf61e56 Author: Nils Goroll Date: Tue Mar 10 10:51:20 2020 +0100 drop the probe lock while scheduling a probe The great witness test introduced by @bsdphk and reworked relatively recently by @Dridi reported a hypothetical lock cycle here. This was not a real issue, because the scheduling code would not re-enter the probe code, yet we might still drop the probe lock while scheduling. diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 8d0bc41f7..52b1871ba 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -465,6 +465,7 @@ vbp_thread(struct worker *wrk, void *priv) { vtim_real now, nxt; struct vbp_target *vt; + int r; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AZ(priv); @@ -486,7 +487,10 @@ vbp_thread(struct worker *wrk, void *priv) vt->running = 1; vt->task.func = vbp_task; vt->task.priv = vt; - if (Pool_Task_Any(&vt->task, TASK_QUEUE_REQ)) + Lck_Unlock(&vbp_mtx); + r = Pool_Task_Any(&vt->task, TASK_QUEUE_REQ); + Lck_Lock(&vbp_mtx); + if (r) vt->running = 0; } binheap_insert(vbp_heap, vt); From dridi at varni.sh Tue Mar 10 10:17:57 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Mar 2020 10:17:57 +0000 Subject: [master] 11d0f63d1 First attempt at a Travis CI witness job In-Reply-To: <20200310095306.CCBF9978C2@lists.varnish-cache.org> References: <20200310095306.CCBF9978C2@lists.varnish-cache.org> Message-ID: On Tue, Mar 10, 2020 at 9:53 AM Dridi Boukelmoune wrote: > > > commit 11d0f63d146c4dda99e65650d23ea7e274a327e6 > Author: Dridi Boukelmoune > Date: Tue Mar 10 10:48:47 2020 +0100 > > First attempt at a Travis CI witness job > > At this point I have no idea what I'm doing and no motivation to learn > further YAML so I'll do something I hate: trigger a CI build and resume > what I was working on until it completes. It worked! Fail: https://travis-ci.org/varnishcache/varnish-cache/jobs/660548137#L2282 Pass: https://travis-ci.org/varnishcache/varnish-cache/jobs/660549234 Dridi From nils.goroll at uplex.de Tue Mar 10 11:39:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 11:39:06 +0000 (UTC) Subject: [master] 2fd1c81b1 vmod_blob flexelinting Message-ID: <20200310113906.78948A05B3@lists.varnish-cache.org> commit 2fd1c81b16bb9557ec8ef38b97eebcd1b89c944f Author: Nils Goroll Date: Tue Mar 10 12:36:46 2020 +0100 vmod_blob flexelinting diff --git a/lib/libvmod_blob/base64.c b/lib/libvmod_blob/base64.c index 82f897dbe..06b4c1df0 100644 --- a/lib/libvmod_blob/base64.c +++ b/lib/libvmod_blob/base64.c @@ -253,7 +253,8 @@ base64_encode(const enum encoding enc, const enum case_e kase, } } } - assert(p >= buf && p - buf <= buflen); + assert(p >= buf); + assert(p <= buf + buflen); return (p - buf); } diff --git a/lib/libvmod_blob/hex.c b/lib/libvmod_blob/hex.c index 1444faeeb..8c2bf1d28 100644 --- a/lib/libvmod_blob/hex.c +++ b/lib/libvmod_blob/hex.c @@ -135,7 +135,7 @@ hex_decode(const enum encoding dec, blob_dest_t buf, if (len == 0) return (0); - if (n != -1 && len > n) + if (n > 0 && len > n) len = n; if (((len+1) >> 1) > buflen) { diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 14c1d0160..b99bab762 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -422,6 +422,7 @@ vmod_transcode(VRT_CTX, VCL_ENUM decs, VCL_ENUM encs, VCL_ENUM case_s, struct vrt_blob b; VCL_STRING r; size_t l; + ssize_t len; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC); @@ -447,14 +448,16 @@ vmod_transcode(VRT_CTX, VCL_ENUM decs, VCL_ENUM encs, VCL_ENUM case_s, if (length <= 0) length = -1; errno = 0; - b.len = func[dec].decode(dec, buf, l, length, strings); - b.blob = buf; + len = func[dec].decode(dec, buf, l, length, strings); - if (b.len == -1) { + if (len < 0) { err_decode(ctx, strings->p[0]); return (NULL); } + b.len = len; + b.blob = buf; + /* * If the encoding and decoding are the same, and the decoding was * legal, just return the concatenated string. @@ -529,6 +532,11 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) assert(b->len > 0); + if (off < 0 || n < 0) { + ERR(ctx, "size and offset cannot be negative in blob.sub()"); + return (NULL); + } + if (off + n > b->len) { VERR(ctx, "size %jd from offset %jd requires more bytes than " "blob length %zd in blob.sub()", From nils.goroll at uplex.de Tue Mar 10 12:43:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 12:43:06 +0000 (UTC) Subject: [master] 2d8909407 vmod_blob flexelinting round 2 Message-ID: <20200310124306.88FBBA20D8@lists.varnish-cache.org> commit 2d8909407cad5e0c93d931024fa3e736062bdf5a Author: Nils Goroll Date: Tue Mar 10 13:06:36 2020 +0100 vmod_blob flexelinting round 2 diff --git a/lib/libvmod_blob/hex.c b/lib/libvmod_blob/hex.c index 8c2bf1d28..b95e45e90 100644 --- a/lib/libvmod_blob/hex.c +++ b/lib/libvmod_blob/hex.c @@ -135,7 +135,7 @@ hex_decode(const enum encoding dec, blob_dest_t buf, if (len == 0) return (0); - if (n > 0 && len > n) + if (n >= 0 && len > (size_t)n) len = n; if (((len+1) >> 1) > buflen) { diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index b99bab762..0c46106bc 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -532,12 +532,13 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) assert(b->len > 0); - if (off < 0 || n < 0) { - ERR(ctx, "size and offset cannot be negative in blob.sub()"); + if (off < 0 || n < 0 || off > SIZE_MAX || n > SIZE_MAX) { + ERR(ctx, "size or offset negative or out of range in blob.sub()"); return (NULL); } - if (off + n > b->len) { + if ((size_t)off > b->len || (size_t)n > b->len || + (size_t)off + (size_t)n > b->len) { VERR(ctx, "size %jd from offset %jd requires more bytes than " "blob length %zd in blob.sub()", (intmax_t)n, (intmax_t)off, b->len); From nils.goroll at uplex.de Tue Mar 10 12:43:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 12:43:06 +0000 (UTC) Subject: [master] f051f2699 flexelinting Message-ID: <20200310124306.B0AC2A20DD@lists.varnish-cache.org> commit f051f269900f55471f8128c2129ad3dd71440e89 Author: Nils Goroll Date: Tue Mar 10 13:08:59 2020 +0100 flexelinting diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 0a685a4f3..9cecd54a3 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -280,7 +280,7 @@ vbe_dir_gethdrs(VRT_CTX, VCL_BACKEND d) http_PrintfHeader(bo->bereq, "Host: %s", bp->hosthdr); do { - pfd = vbe_dir_getfd(wrk, bp, bo, extrachance == 0); + pfd = vbe_dir_getfd(wrk, bp, bo, extrachance == 0 ? 1 : 0); if (pfd == NULL) return (-1); AN(bo->htc); diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 52b1871ba..39bc68196 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -724,13 +724,17 @@ static int v_matchproto_(binheap_cmp_t) vbp_cmp(void *priv, const void *a, const void *b) { const struct vbp_target *aa, *bb; + int ar, br; AZ(priv); CAST_OBJ_NOTNULL(aa, a, VBP_TARGET_MAGIC); CAST_OBJ_NOTNULL(bb, b, VBP_TARGET_MAGIC); - if ((aa->running == 0) != (bb->running == 0)) - return (aa->running == 0); + ar = aa->running == 0; + br = bb->running == 0; + + if (ar != br) + return (ar); return (aa->due < bb->due); } diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index cc18eccb8..480d5adfc 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -374,7 +374,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) for (; b != NULL; b = VTAILQ_NEXT(b, list)) { if (bd != NULL) ban_lurker_test_ban(wrk, vsl, b, &obans, bd, - count > cutoff); + count > cutoff ? 1 : 0); if (b->flags & BANS_FLAG_COMPLETED) continue; if (b->flags & BANS_FLAG_REQ && diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index dd51a468e..5319d6a76 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -730,6 +730,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) return (F_STP_FAIL); } } + AZ(stale_boc); AZ(bo->stale_oc->flags & OC_F_FAILED); AZ(vbf_beresp2obj(bo)); @@ -974,8 +975,11 @@ vbf_fetch_thread(struct worker *wrk, void *priv) VCL_TaskLeave(bo->privs); http_Teardown(bo->bereq); http_Teardown(bo->beresp); + // XXX after 6.4 release: + // bereq_body should have 0 or 1 references remaining, + // see VRB_Free() for the other end if (bo->bereq_body != NULL) - HSH_DerefObjCore(bo->wrk, &bo->bereq_body, 0); + (void) HSH_DerefObjCore(bo->wrk, &bo->bereq_body, 0); if (bo->fetch_objcore->boc->state == BOS_FINISHED) { AZ(bo->fetch_objcore->flags & OC_F_FAILED); diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index c1f8bd344..9becb176b 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -210,13 +210,18 @@ vrg_ifrange(struct req *req) } /* assume date, strong check [RFC7232 2.2.2 p7] */ - if (!(ims = VTIM_parse(p))) // rfc7233,l,502,512 + ims = VTIM_parse(p); + if (!ims) // rfc7233,l,502,512 return (0); /* the response needs a Date */ // rfc7232 fc7232,l,439,440 - if (!http_GetHdr(req->resp, H_Date, &p) || !(d = VTIM_parse(p))) + if (!http_GetHdr(req->resp, H_Date, &p)) return (0); + d = VTIM_parse(p); + if (!d) + return (0); + /* grab the Last Modified value */ if (!http_GetHdr(req->resp, H_Last_Modified, &p)) diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index b4ad19b90..1d34efe54 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -340,7 +340,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) struct pool_task *tp; struct pool_task tpx, tps; vtim_real tmo; - int i, reserve; + unsigned i, reserve; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); wrk->pool = pp; diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c index 52422e3bc..0fc157ef8 100644 --- a/bin/varnishd/storage/storage_file.c +++ b/bin/varnishd/storage/storage_file.c @@ -177,9 +177,8 @@ smf_init(struct stevedore *parent, int ac, char * const *av) static void insfree(struct smf_sc *sc, struct smf *sp) { - size_t b; + off_t b, ns; struct smf *sp2; - size_t ns; AZ(sp->alloc); assert(sp->flist == NULL); @@ -231,10 +230,10 @@ remfree(const struct smf_sc *sc, struct smf *sp) */ static struct smf * -alloc_smf(struct smf_sc *sc, size_t bytes) +alloc_smf(struct smf_sc *sc, off_t bytes) { struct smf *sp, *sp2; - size_t b; + off_t b; AZ(bytes % sc->pagesize); b = bytes / sc->pagesize; @@ -398,8 +397,6 @@ smf_open_chunk(struct smf_sc *sc, off_t sz, off_t off, off_t *fail, off_t *sum) *fail = sz; h = sz / 2; - if (h > SSIZE_MAX) - h = SSIZE_MAX; h -= (h % sc->pagesize); smf_open_chunk(sc, h, off, fail, sum); @@ -436,13 +433,16 @@ smf_open(struct stevedore *st) /*--------------------------------------------------------------------*/ static struct storage * v_matchproto_(sml_alloc_f) -smf_alloc(const struct stevedore *st, size_t size) +smf_alloc(const struct stevedore *st, size_t sz) { struct smf *smf; struct smf_sc *sc; + off_t size; CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC); - assert(size > 0); + assert(sz > 0); + // XXX missing OFF_T_MAX + size = (off_t)sz; size += (sc->pagesize - 1UL); size &= ~(sc->pagesize - 1UL); Lck_Lock(&sc->mtx); From nils.goroll at uplex.de Tue Mar 10 12:58:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 12:58:07 +0000 (UTC) Subject: [master] 92a047ce3 Stabilize test Message-ID: <20200310125807.28132A2997@lists.varnish-cache.org> commit 92a047ce32a540fdca7e3df05d94ceef5779807d Author: Nils Goroll Date: Tue Mar 10 13:46:41 2020 +0100 Stabilize test Have seen ExpKill interfere with the Debug messages in vtest diff --git a/bin/varnishtest/tests/c00071.vtc b/bin/varnishtest/tests/c00071.vtc index 7ad563095..7b033fcec 100644 --- a/bin/varnishtest/tests/c00071.vtc +++ b/bin/varnishtest/tests/c00071.vtc @@ -54,7 +54,7 @@ logexpect l1 -v v1 -g vxid -q "vxid == 1006" { expect 6 = Error {^workspace_client overflow} } -start -logexpect l2 -v v1 -g raw { +logexpect l2 -v v1 -g raw -i Debug { expect * 0 Debug {^WS_Snapshot.* = overflowed} expect 12 0 Debug {^WS_Reset.*, overflowed} } -start From nils.goroll at uplex.de Tue Mar 10 12:58:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 12:58:07 +0000 (UTC) Subject: [master] 519aeba7b vmod_blob flexelinting round 3 Message-ID: <20200310125807.3C2D7A299A@lists.varnish-cache.org> commit 519aeba7b9510c1cf27d47fef7165e5fd480ce53 Author: Nils Goroll Date: Tue Mar 10 13:56:23 2020 +0100 vmod_blob flexelinting round 3 it seems I wrongly assumed that SIZE_MAX would be defined as a positive integer, in fact it looks like (size_t)-1 was used. Ref: 2d8909407cad5e0c93d931024fa3e736062bdf5a diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 0c46106bc..ec4e15aef 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -532,8 +532,9 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) assert(b->len > 0); - if (off < 0 || n < 0 || off > SIZE_MAX || n > SIZE_MAX) { - ERR(ctx, "size or offset negative or out of range in blob.sub()"); + // XXX check for > SIZE_MAX ? + if (off < 0 || n < 0) { + ERR(ctx, "size or offset negative in blob.sub()"); return (NULL); } From nils.goroll at uplex.de Tue Mar 10 13:05:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 13:05:07 +0000 (UTC) Subject: [master] b365be2d2 try to avoid a make race Message-ID: <20200310130507.19DA1A2F9F@lists.varnish-cache.org> commit b365be2d281944d5c79be92ff73b3dc02c5db6be Author: Nils Goroll Date: Tue Mar 10 14:02:44 2020 +0100 try to avoid a make race vcc_compile.h is generated and included by other compilation units, so we need to make sure that this happens in sequence. Seen on vtest: make[5]: Entering directory '/tmp/vtest.32_sun12.4/varnish-cache/lib/libvcc' CC libvcc_a-vcc_acl.o CC libvcc_a-vcc_action.o CC libvcc_a-vcc_backend.o CC libvcc_a-vcc_backend_util.o CC libvcc_a-vcc_compile.o CC libvcc_a-vcc_expr.o mkdir -p ../../include/tbl /opt/local/bin/python3.4 ../../lib/libvcc/generate.py \ ../.. ../.. CC libvcc_a-vcc_parse.o CC libvcc_a-vcc_storage.o CC libvcc_a-vcc_symb.o "vcc_compile.h", line 194: undefined symbol: VCL_RET_MAX "vcc_compile.h", line 194: can not declare variably modified type at file scope cc: acomp failed for vcc_parse.c Makefile:668: recipe for target 'libvcc_a-vcc_parse.o' failed diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am index be4d4c66d..09ed89b36 100644 --- a/lib/libvcc/Makefile.am +++ b/lib/libvcc/Makefile.am @@ -54,7 +54,7 @@ GEN_H = \ vcc_fixed_token.c \ vcc_token_defs.h -$(GEN_H): vcc_obj.c +$(GEN_H): vcc_obj.c vcc_compile.h GENERATED_H = vcc_obj.c $(GEN_H) From dridi.boukelmoune at gmail.com Tue Mar 10 13:12:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 10 Mar 2020 13:12:06 +0000 (UTC) Subject: [master] 9cd38f907 Witness lock order in VMOD test suites too Message-ID: <20200310131206.E638CA44CE@lists.varnish-cache.org> commit 9cd38f907e590f1c7ab54975df519e896574db25 Author: Dridi Boukelmoune Date: Tue Mar 10 14:08:32 2020 +0100 Witness lock order in VMOD test suites too diff --git a/Makefile.am b/Makefile.am index c70a16293..a48e06819 100644 --- a/Makefile.am +++ b/Makefile.am @@ -65,8 +65,9 @@ gcov_digest: ${PYTHON} tools/gcov_digest.py -o _gcov witness.dot: all - $(MAKE) -C bin/varnishtest check AM_VTC_LOG_FLAGS=-pdebug=+witness - $(AM_V_GEN) $(srcdir)/tools/witness.sh witness.dot bin/varnishtest + $(MAKE) check AM_VTC_LOG_FLAGS=-pdebug=+witness + $(AM_V_GEN) $(srcdir)/tools/witness.sh witness.dot bin/varnishtest/ \ + lib/libvmod_*/ .dot.svg: if ! HAVE_DOT From dridi.boukelmoune at gmail.com Tue Mar 10 15:11:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 10 Mar 2020 15:11:06 +0000 (UTC) Subject: [master] 554f8ba0b More changelog TLC Message-ID: <20200310151106.B285FA8497@lists.varnish-cache.org> commit 554f8ba0b030ccb615c096b96c9ba9ba9b742002 Author: Dridi Boukelmoune Date: Tue Mar 10 16:09:07 2020 +0100 More changelog TLC diff --git a/doc/changes.rst b/doc/changes.rst index 908804043..8260a65bc 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -56,6 +56,10 @@ NEXT (2020-03-15) they are delimited by "double quotes". This was applied to ``SessError`` and ``Backend_health``. +* It is now possible for VMOD authors to customize the connection pooling + of a dynamic backend. A hash is now computed to determine uniqueness and + a backend declaration can contribute arbitrary data to influence the pool. + .. 3109 (vdp gunzip) is probably irrelevant for users because it only happens with gzip objects inserted bypassing the built-in vfps @@ -110,6 +114,9 @@ NEXT (2020-03-15) * Fixed status for truncated CLI responses, bug 3038_ +* New or improved Coccinelle semantic patches that may be useful for + VMOD or utilities authors. + * Output VCC warnings also for VCLs loaded via the ``varnishd -f`` option, see bug 3160_ @@ -118,11 +125,20 @@ NEXT (2020-03-15) * Added a ``Notice`` VSL tag (used for ``varnishlog`` logging) +* Always refer to ``sub`` as subroutine in the documentation and error + messages to avoid confusion with other terms. + .. mention #3176? (backend cooling straightened out) -* Fixed session close reason reporting and accounting, added - ``rx_close_idle`` counter for separate accounting when - ``timeout_idle`` is reached. +* New ``pid`` command in the Varnish CLI, to get the master and optionally + cache process PIDs, for example from ``varnishadm``. + +* Fixed a race that could result in a partial response being served in its + entirety when it is also compressed with gzip. + +* Fixed session close reason reporting and accounting, added ``rx_close_idle`` + counter for separate accounting when ``timeout_idle`` is reached. Also, + ``send_timeout`` is no longer reported as "remote closed". * Fixed handling of request bodies for backend retries @@ -165,27 +181,30 @@ NEXT (2020-03-15) * Fixed a case where ``send_timeout`` would have no effect when streaming from a backend fetch, see bug 3189_ +* Added ``VSB_tofile()`` to ``libvarnishapi``, see 3238_ + *NOTE* Users upgrading varnish should re-check ``send_timeout`` with respect to long pass and streaming fetches and watch out for increased session close rates. +.. _1853: https://github.com/varnishcache/varnish-cache/issues/1853 +.. _2418: https://github.com/varnishcache/varnish-cache/issues/2418 .. _3009: https://github.com/varnishcache/varnish-cache/issues/3009 -.. _3088: https://github.com/varnishcache/varnish-cache/issues/3088 .. _3019: https://github.com/varnishcache/varnish-cache/issues/3019 +.. _3038: https://github.com/varnishcache/varnish-cache/issues/3038 +.. _3088: https://github.com/varnishcache/varnish-cache/issues/3088 +.. _3089: https://github.com/varnishcache/varnish-cache/issues/3089 +.. _3094: https://github.com/varnishcache/varnish-cache/issues/3094 .. _3131: https://github.com/varnishcache/varnish-cache/issues/3131 .. _3135: https://github.com/varnishcache/varnish-cache/issues/3135 -.. _3094: https://github.com/varnishcache/varnish-cache/issues/3094 .. _3145: https://github.com/varnishcache/varnish-cache/issues/3145 -.. _3038: https://github.com/varnishcache/varnish-cache/issues/3038 .. _3160: https://github.com/varnishcache/varnish-cache/issues/3160 -.. _3089: https://github.com/varnishcache/varnish-cache/issues/3089 -.. _2418: https://github.com/varnishcache/varnish-cache/issues/2418 -.. _1853: https://github.com/varnishcache/varnish-cache/issues/1853 +.. _3189: https://github.com/varnishcache/varnish-cache/issues/3189 .. _3194: https://github.com/varnishcache/varnish-cache/issues/3194 -.. _varnish_modules: https://github.com/varnish/varnish-modules .. _3221: https://github.com/varnishcache/varnish-cache/issues/3221 .. _3229: https://github.com/varnishcache/varnish-cache/issues/3229 -.. _3189: https://github.com/varnishcache/varnish-cache/issues/3189 +.. _3238: https://github.com/varnishcache/varnish-cache/issues/3238 +.. _varnish_modules: https://github.com/varnish/varnish-modules ================================ Varnish Cache 6.3.0 (2019-09-15) From dridi.boukelmoune at gmail.com Tue Mar 10 15:11:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 10 Mar 2020 15:11:06 +0000 (UTC) Subject: [master] 0e84a5a58 Sync Red Hat installation docs with reality Message-ID: <20200310151106.C583CA849E@lists.varnish-cache.org> commit 0e84a5a58d6ce1c433a9d0c33e1d62091645ee28 Author: Dridi Boukelmoune Date: Tue Mar 10 16:09:59 2020 +0100 Sync Red Hat installation docs with reality diff --git a/doc/sphinx/installation/install_redhat.rst b/doc/sphinx/installation/install_redhat.rst index 6d29ae203..e2ce63be7 100644 --- a/doc/sphinx/installation/install_redhat.rst +++ b/doc/sphinx/installation/install_redhat.rst @@ -12,11 +12,16 @@ We therefore recommend that you install the latest version directly from our rep Varnish Cache is packaged in RPMs for easy installation and upgrade on Red Hat systems. The Varnish Cache project maintains official packages for the current -Enterprise Linux versions. Varnish Cache 4.1 and 5.x are supported on EL6 and EL7. +Enterprise Linux versions. Varnish Cache 6.x series are supported on el7 and el8. -We try to keep the latest version available as prebuilt RPMs (el5 and el6) +We try to keep the latest version available as prebuilt RPMs (el7 and el8) on `packagecloud.io/varnishcache `_. +Starting with el8 a DNF module will inhibit Varnish packages, and the solution +is to disable the module before installing:: + + dnf module disable varnish + Official packages of 6 ---------------------- @@ -36,17 +41,9 @@ With the release of 6.0.2, users have to switch to switch repositories to get the latest version. Read more about this on `Release 6.0.2 `_. - -Official packages of 4.1 ------------------------- - -To use Varnish Cache 4.1 packages from the official varnish-cache.org repos, -follow the instructions available at: - -* https://packagecloud.io/varnishcache/varnish41/install#manual-rpm - External packaging ------------------ + Varnish Cache is also distributed in third party package repositories. .. _`Fedora EPEL`: https://fedoraproject.org/wiki/EPEL From dridi.boukelmoune at gmail.com Tue Mar 10 15:17:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 10 Mar 2020 15:17:06 +0000 (UTC) Subject: [master] acce95fe1 Mention witness.sh in the changelog Message-ID: <20200310151706.727ABA896F@lists.varnish-cache.org> commit acce95fe1bb06fdd0ecaa3a586de0187e230f3a9 Author: Dridi Boukelmoune Date: Tue Mar 10 16:15:19 2020 +0100 Mention witness.sh in the changelog It can definitely be used in third party VMOD projects. diff --git a/doc/changes.rst b/doc/changes.rst index 8260a65bc..df6a3dbf7 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -64,7 +64,9 @@ NEXT (2020-03-15) happens with gzip objects inserted bypassing the built-in vfps * The option ``varnishtest -W`` is gone, the same can be achieved with - ``varnishtest -p debug=+witness``. + ``varnishtest -p debug=+witness``. A ``witness.sh`` script is available + in the source tree to generate a graphviz dot file and detect potential + lock cycles from the test logs. * The ``Process`` timestamp for ``vcl_synth {}`` was wrongly issued before the VCL subroutine, now it gets emitted after VCL returns for From dridi.boukelmoune at gmail.com Tue Mar 10 16:45:08 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 10 Mar 2020 16:45:08 +0000 (UTC) Subject: [master] 54c0bb85d Manage libvarnish.a with libtool Message-ID: <20200310164508.54174AA8DE@lists.varnish-cache.org> commit 54c0bb85d8122b72259a926dc9e7aff130a37e22 Author: Dridi Boukelmoune Date: Tue Mar 10 16:42:13 2020 +0100 Manage libvarnish.a with libtool It remains a private static library. Neither libvarnishapi nor test cases from libvarnish use individual source files besides the ones containing test drivers. It can lead to very cryptic messages where a symbol is defined twice from different sources, where the sources are the same but the object code is technically different (a copy built from _SOURCES and a copy in libvarnish.a). While at it, add sanitizers flags to all libvarnish test cases. diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am index d0ca15408..0d0c8e709 100644 --- a/bin/varnishadm/Makefile.am +++ b/bin/varnishadm/Makefile.am @@ -12,7 +12,7 @@ varnishadm_CFLAGS = @LIBEDIT_CFLAGS@ \ @SAN_CFLAGS@ varnishadm_LDADD = \ - $(top_builddir)/lib/libvarnish/libvarnish.a \ + $(top_builddir)/lib/libvarnish/libvarnish.la \ $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ ${PTHREAD_LIBS} ${RT_LIBS} ${NET_LIBS} @LIBEDIT_LIBS@ ${LIBM} \ @SAN_LDFLAGS@ diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 2a66a4e44..e124ed5d8 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -170,7 +170,7 @@ varnishd_LDFLAGS = -export-dynamic varnishd_LDADD = \ $(top_builddir)/lib/libvcc/libvcc.a \ - $(top_builddir)/lib/libvarnish/libvarnish.a \ + $(top_builddir)/lib/libvarnish/libvarnish.la \ $(top_builddir)/lib/libvgz/libvgz.a \ @SAN_LDFLAGS@ \ @JEMALLOC_LDADD@ \ @@ -187,21 +187,21 @@ vhp_gen_hufdec_SOURCES = hpack/vhp_gen_hufdec.c vhp_gen_hufdec_CFLAGS = @SAN_CFLAGS@ \ -include config.h vhp_gen_hufdec_LDADD = \ - $(top_builddir)/lib/libvarnish/libvarnish.a + $(top_builddir)/lib/libvarnish/libvarnish.la noinst_PROGRAMS += vhp_table_test vhp_table_test_SOURCES = hpack/vhp_table.c vhp_table_test_CFLAGS = @SAN_CFLAGS@ \ -DTABLE_TEST_DRIVER -include config.h vhp_table_test_LDADD = \ - $(top_builddir)/lib/libvarnish/libvarnish.a + $(top_builddir)/lib/libvarnish/libvarnish.la noinst_PROGRAMS += vhp_decode_test vhp_decode_test_SOURCES = hpack/vhp_decode.c hpack/vhp_table.c vhp_decode_test_CFLAGS = @SAN_CFLAGS@ \ -DDECODE_TEST_DRIVER -include config.h vhp_decode_test_LDADD = \ - $(top_builddir)/lib/libvarnish/libvarnish.a + $(top_builddir)/lib/libvarnish/libvarnish.la noinst_PROGRAMS += esi_parse_fuzzer esi_parse_fuzzer_SOURCES = \ @@ -210,7 +210,7 @@ esi_parse_fuzzer_SOURCES = \ esi_parse_fuzzer_CFLAGS = \ @SAN_CFLAGS@ -DNOT_IN_A_VMOD -DTEST_DRIVER -include config.h esi_parse_fuzzer_LDADD = \ - $(top_builddir)/lib/libvarnish/libvarnish.a \ + $(top_builddir)/lib/libvarnish/libvarnish.la \ $(top_builddir)/lib/libvgz/libvgz.a TESTS = vhp_table_test vhp_decode_test diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index 4ebff33a8..34f11f77e 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -52,7 +52,7 @@ varnishtest_SOURCES = \ vtc_varnish.c varnishtest_LDADD = \ - $(top_builddir)/lib/libvarnish/libvarnish.a \ + $(top_builddir)/lib/libvarnish/libvarnish.la \ $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ $(top_builddir)/lib/libvgz/libvgz.a \ @SAN_LDFLAGS@ \ diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index da5821c7e..298d63400 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -5,15 +5,16 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include \ @PCRE_CFLAGS@ -AM_LDFLAGS = $(AM_LT_LDFLAGS) +AM_CFLAGS = $(AM_LT_CFLAGS) @SAN_CFLAGS@ +AM_LDFLAGS = $(AM_LT_LDFLAGS) @SAN_LDFLAGS@ -noinst_LIBRARIES = libvarnish.a +noinst_LTLIBRARIES = libvarnish.la -libvarnish_a_CFLAGS = \ +libvarnish_la_CFLAGS = \ -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \ - @SAN_CFLAGS@ + $(AM_CFLAGS) -libvarnish_a_SOURCES = \ +libvarnish_la_SOURCES = \ binary_heap.c \ vas.c \ vav.c \ @@ -47,17 +48,18 @@ TESTS = vjsn_test vnum_c_test binheap vsb_test noinst_PROGRAMS = ${TESTS} -binheap_SOURCES = binary_heap.c vas.c vrnd.c -binheap_CFLAGS = -DTEST_DRIVER +binheap_SOURCES = binary_heap.c +binheap_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER +binheap_LDADD = $(AM_LDFLAGS) libvarnish.la -vnum_c_test_SOURCES = vnum.c vas.c -vnum_c_test_CFLAGS = -DNUM_C_TEST -include config.h @SAN_CFLAGS@ -vnum_c_test_LDADD = ${LIBM} libvarnish.a @SAN_LDFLAGS@ +vnum_c_test_SOURCES = vnum.c +vnum_c_test_CFLAGS = $(AM_CFLAGS) -DNUM_C_TEST +vnum_c_test_LDADD = $(AM_LDFLAGS) libvarnish.la ${LIBM} vjsn_test_SOURCES = vjsn.c -vjsn_test_CFLAGS = -DVJSN_TEST @SAN_CFLAGS@ -vjsn_test_LDADD = libvarnish.a @SAN_LDFLAGS@ +vjsn_test_CFLAGS = $(AM_CFLAGS) -DVJSN_TEST +vjsn_test_LDADD = $(AM_LDFLAGS) libvarnish.la -vsb_test_SOURCES = vsb_test.c vsb.c vas.c -vsb_test_CFLAGS = -DVSB_TEST -vsb_test_LDADD = libvarnish.a +vsb_test_SOURCES = vsb_test.c +vsb_test_CFLAGS = $(AM_CFLAGS) -DVSB_TEST +vsb_test_LDADD = $(AM_LDFLAGS) libvarnish.la diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 6f109374c..750128eba 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -12,27 +12,10 @@ lib_LTLIBRARIES = libvarnishapi.la libvarnishapi_la_LDFLAGS = $(AM_LDFLAGS) -version-info 2:0:0 libvarnishapi_la_SOURCES = \ + ../../include/vcs_version.h \ vsl_api.h \ vxp.h \ vxp_tokens.h \ - ../libvarnish/vas.c \ - ../libvarnish/vav.c \ - ../../include/vcs_version.h \ - ../libvarnish/version.c \ - ../libvarnish/vcli_proto.c \ - ../libvarnish/vct.c \ - ../libvarnish/vfil.c \ - ../libvarnish/vfl.c \ - ../libvarnish/vin.c \ - ../libvarnish/vjsn.c \ - ../libvarnish/vlu.c \ - ../libvarnish/vmb.c \ - ../libvarnish/vpf.c \ - ../libvarnish/vre.c \ - ../libvarnish/vsb.c \ - ../libvarnish/vtim.c \ - ../libvarnish/vnum.c \ - ../libvarnish/vsha256.c \ vsc.c \ vsig.c \ vsl.c \ @@ -57,6 +40,7 @@ libvarnishapi_la_CFLAGS = \ @SAN_CFLAGS@ libvarnishapi_la_LIBADD = \ + $(top_builddir)/lib/libvarnish/libvarnish.la \ @SAN_LDFLAGS@ @PCRE_LIBS@ ${RT_LIBS} ${LIBM} if HAVE_LD_VERSION_SCRIPT From nils.goroll at uplex.de Tue Mar 10 16:55:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Mar 2020 16:55:06 +0000 (UTC) Subject: [master] 4e1133e8b try to fix the make race for real Message-ID: <20200310165506.C6EAAAAD80@lists.varnish-cache.org> commit 4e1133e8b031eaf88016a1dceed561e03da957ba Author: Nils Goroll Date: Tue Mar 10 17:53:31 2020 +0100 try to fix the make race for real b365be2d281944d5c79be92ff73b3dc02c5db6be was not correct in that I put vcc_compile.h in the wrong place, it should have been added to GEN_H, not as a dependency for GEN_H I also realized that we should wait for the last file generate.py generates. diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am index 09ed89b36..ce550df7a 100644 --- a/lib/libvcc/Makefile.am +++ b/lib/libvcc/Makefile.am @@ -50,11 +50,15 @@ vcc_obj.c: \ @PYTHON@ $(top_srcdir)/lib/libvcc/generate.py \ $(top_srcdir) $(top_builddir) +## this list is not complete, but it contains important files +## used as includes and the last file built by generate.py GEN_H = \ vcc_fixed_token.c \ - vcc_token_defs.h + vcc_token_defs.h \ + vcc_compile.h \ + tbl/vrt_stv_var.h -$(GEN_H): vcc_obj.c vcc_compile.h +$(GEN_H): vcc_obj.c GENERATED_H = vcc_obj.c $(GEN_H) From dridi.boukelmoune at gmail.com Tue Mar 10 17:36:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 10 Mar 2020 17:36:06 +0000 (UTC) Subject: [master] 65f048ce8 Add missing link in libvarnishapi Message-ID: <20200310173606.55F17ABCE1@lists.varnish-cache.org> commit 65f048ce879dcf47497a552b011140f34757a933 Author: Dridi Boukelmoune Date: Tue Mar 10 18:32:30 2020 +0100 Add missing link in libvarnishapi Spotted by vtest on SunOS. diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 750128eba..0cbff3c4d 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -41,7 +41,7 @@ libvarnishapi_la_CFLAGS = \ libvarnishapi_la_LIBADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ - @SAN_LDFLAGS@ @PCRE_LIBS@ ${RT_LIBS} ${LIBM} + @SAN_LDFLAGS@ @PCRE_LIBS@ ${NET_LIBS} ${RT_LIBS} ${LIBM} if HAVE_LD_VERSION_SCRIPT libvarnishapi_la_LDFLAGS += -Wl,--version-script=$(srcdir)/libvarnishapi.map From nils.goroll at uplex.de Wed Mar 11 07:17:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Mar 2020 07:17:06 +0000 (UTC) Subject: [master] 198caa60a flexelinting Message-ID: <20200311071706.98A1D6E65F@lists.varnish-cache.org> commit 198caa60a70234e3a825642674689c435805cefd Author: Nils Goroll Date: Wed Mar 11 07:52:31 2020 +0100 flexelinting fix signed/unsigned comparisons diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 4f19de3f0..1e5905093 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -142,17 +142,18 @@ update(void) const unsigned n = w * hist_range; unsigned bm[n], bh[n]; unsigned max; - unsigned i, j, scale; + unsigned scale; + int i, j; unsigned k, l; erase(); /* Draw horizontal axis */ - for (i = 0; i < n; ++i) - (void)mvaddch(LINES - 2, i, '-'); - for (k = 0, l = hist_low; k < hist_range; ++k, ++l) { - (void)mvaddch(LINES - 2, w * k, '+'); - mvprintw(LINES - 1, w * k, "|1e%d", l); + for (k = 0; k < n; ++k) + (void)mvaddch(LINES - 2, k, '-'); + for (i = 0, j = hist_low; i < hist_range; ++i, ++j) { + (void)mvaddch(LINES - 2, w * i, '+'); + mvprintw(LINES - 1, w * i, "|1e%d", j); } if (end_of_file) @@ -191,11 +192,11 @@ update(void) mvprintw((LINES - 2) - j, 0, "%u_", j * scale); /* show them */ - for (i = 0; i < n; ++i) { - for (j = 0; j < bm[i] / scale; ++j) - (void)mvaddch((LINES - 3) - j, i, '#'); - for (; j < (bm[i] + bh[i]) / scale; ++j) - (void)mvaddch((LINES - 3) - j, i, '|'); + for (k = 0; k < n; ++k) { + for (l = 0; l < bm[k] / scale; ++l) + (void)mvaddch((LINES - 3) - l, k, '#'); + for (; l < (bm[k] + bh[k]) / scale; ++l) + (void)mvaddch((LINES - 3) - l, k, '|'); } refresh(); @@ -306,7 +307,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], i = hist_high * HIST_RES - 1; i -= hist_low * HIST_RES; assert(i >= 0); - assert(i < hist_buckets); + assert((unsigned)i < hist_buckets); AZ(pthread_mutex_lock(&mtx)); diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 657d4a5fc..084cf4e70 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -710,7 +710,7 @@ parse_format(const char *format) if (!*q) VUT_Error(vut, 1, "Unmatched bracket at: %s", p - 2); - assert(q - p < sizeof buf - 1); + assert((unsigned)(q - p) < sizeof buf - 1); strncpy(buf, p, q - p); buf[q - p] = '\0'; q++; @@ -755,7 +755,7 @@ isprefix(const char *prefix, size_t len, const char *b, const char *e, const char **next) { assert(len > 0); - if (e - b < len || strncasecmp(b, prefix, len)) + if (e < b + len || strncasecmp(b, prefix, len)) return (0); b += len; if (next) { diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index de64c7519..8d8d572e8 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -77,7 +77,7 @@ struct top { double count; }; -static int period = 60; /* seconds */ +static unsigned period = 60; /* seconds */ static int end_of_file = 0; static unsigned ntop; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; @@ -184,7 +184,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], } static void -update(int p) +update(unsigned p) { struct top *tp, *tp2; int l, len; From nils.goroll at uplex.de Wed Mar 11 07:42:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Mar 2020 07:42:07 +0000 (UTC) Subject: [master] 0fd4edc54 stabilize test Message-ID: <20200311074207.301861001B2@lists.varnish-cache.org> commit 0fd4edc54ad4a094dfce8fe1594e13d70d2f78a6 Author: Nils Goroll Date: Wed Mar 11 08:38:21 2020 +0100 stabilize test The refcount of the vcl can also have gone down to zero as in this vtc output: ** top === process p1 -write "vcl.li\t\r" **** p1 Writing 8 bytes ** top === process p1 -expect-text 0 1 "active auto warm 1... **** dT 2.143 **** p1 output|varnish> vcl.list **** dT 2.173 **** p1 output|varnish> **** p1 output| **** p1 output|200 **** p1 output|active auto warm 0 vcl1 **** dT 2.212 **** v1 vsl| 0 CLI - Rd vcl.list **** v1 vsl| 0 CLI - Wr 200 40 active auto warm 0 vcl1 As the process facility does not support regexp-expects (yet?), we just check for the beginning of the line diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc index 52d222bd4..439fbfafc 100644 --- a/bin/varnishtest/tests/u00011.vtc +++ b/bin/varnishtest/tests/u00011.vtc @@ -24,7 +24,7 @@ process p1 -expect-text 0 1 "PONG" process p1 -write "vcl.li\t\r" -process p1 -expect-text 0 1 "active auto warm 1 vcl1" +process p1 -expect-text 0 1 "active auto warm" process p1 -write "vcl.s\t\th\t vcl1\r" From nils.goroll at uplex.de Wed Mar 11 07:55:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Mar 2020 07:55:06 +0000 (UTC) Subject: [master] 03970d3f4 We know that the next release is going to be 6.4 by now Message-ID: <20200311075506.636CE10078B@lists.varnish-cache.org> commit 03970d3f4eac0863e5cf53a4558fff5548bde75f Author: Nils Goroll Date: Wed Mar 11 08:53:42 2020 +0100 We know that the next release is going to be 6.4 by now diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-6.4.rst similarity index 100% rename from doc/sphinx/whats-new/changes-trunk.rst rename to doc/sphinx/whats-new/changes-6.4.rst diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-6.4.rst similarity index 100% rename from doc/sphinx/whats-new/upgrading-trunk.rst rename to doc/sphinx/whats-new/upgrading-6.4.rst From nils.goroll at uplex.de Wed Mar 11 07:55:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Mar 2020 07:55:06 +0000 (UTC) Subject: [master] ac602e018 Start skeleton release notes for the next version. Message-ID: <20200311075506.7A4B010078E@lists.varnish-cache.org> commit ac602e0187a5e78821ea093f9c9ef3e6234f3fe3 Author: Geoff Simmons Date: Tue Sep 25 16:31:17 2018 +0200 Start skeleton release notes for the next version. Restructured so that: * 'Upgrading' is limited to work that has to be done to upgrade from a current deployment to the new version. * 'Changes' is a comprehensive, user-level description of changes and new features. Conflicts: doc/sphinx/whats-new/index.rst diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst new file mode 100644 index 000000000..fbda9d6b9 --- /dev/null +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -0,0 +1,73 @@ +**Note: This is a working document for a future release, with running +updates for changes in the development branch. For changes in the +released versions of Varnish, see:** :ref:`whats-new-index` + +.. _whatsnew_changes_CURRENT: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +Changes in Varnish **$NEXT_RELEASE** +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +For information about updating your current Varnish deployment to the +new version, see :ref:`whatsnew_upgrading_CURRENT`. + +A more detailed and technical account of changes in Varnish, with +links to issues that have been fixed and pull requests that have been +merged, may be found in the `change log`_. + +.. _change log: https://github.com/varnishcache/varnish-cache/blob/master/doc/changes.rst + +varnishd +======== + +Parameters +~~~~~~~~~~ + +**XXX changes in -p parameters** + +Other changes in varnishd +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Changes to VCL +============== + +VCL variables +~~~~~~~~~~~~~ + +**XXX new, deprecated or removed variables, or changed semantics** + +Other changes to VCL +~~~~~~~~~~~~~~~~~~~~ + +VMODs +===== + +**XXX changes in the bundled VMODs** + +varnishlog +========== + +**XXX changes concerning varnishlog(1) and/or vsl(7)** + +varnishadm +========== + +**XXX changes concerning varnishadm(1) and/or varnish-cli(7)** + +varnishstat +=========== + +**XXX changes concerning varnishstat(1) and/or varnish-counters(7)** + +varnishtest +=========== + +**XXX changes concerning varnishtest(1) and/or vtc(7)** + +Changes for developers and VMOD authors +======================================= + +**XXX changes concerning VRT, the public APIs, source code organization, +builds etc.** + +*eof* diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst new file mode 100644 index 000000000..6143fde99 --- /dev/null +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -0,0 +1,33 @@ +**Note: This is a working document for a future release, with running +updates for changes in the development branch. For changes in the +released versions of Varnish, see:** :ref:`whats-new-index` + +.. _whatsnew_upgrading_CURRENT: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish **$NEXT_RELEASE** +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +**XXX: how to upgrade from previous deployments to this +version. Limited to work that has to be done for an upgrade, new +features are listed in "Changes". Explicitly mention what does *not* +have to be changed, especially in VCL. May include, but is not limited +to:** + +* Elements of VCL that have been removed or are deprecated, or whose + semantics have changed. + +* -p parameters that have been removed or are deprecated, or whose + semantics have changed. + +* Changes in the CLI. + +* Changes in the output or interpretation of stats or the log, including + changes affecting varnishncsa/-hist/-top. + +* Changes that may be necessary in VTCs or in the use of varnishtest. + +* Changes in public APIs that may require changes in VMODs or VAPI/VUT + clients. + +*eof* From nils.goroll at uplex.de Wed Mar 11 08:00:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Mar 2020 08:00:08 +0000 (UTC) Subject: [master] a7a9e513f more places for numbers and dates Message-ID: <20200311080008.4E437100B89@lists.varnish-cache.org> commit a7a9e513f7f4c241668861b8a1d0f87a1fc5b5e3 Author: Nils Goroll Date: Wed Mar 11 08:57:34 2020 +0100 more places for numbers and dates sorry, the previous commit probably broke building the docs diff --git a/doc/changes.rst b/doc/changes.rst index df6a3dbf7..478e3083c 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -27,7 +27,11 @@ individual releases. These documents are updated as part of the release process. ================================ -NEXT (2020-03-15) +NEXT (2020-09-15) +================================ + +================================ +Varnish Cache 6.4.0 (2019-09-16) ================================ * The ``MAIN.sess_drop`` counter is gone. diff --git a/doc/sphinx/whats-new/changes-6.4.rst b/doc/sphinx/whats-new/changes-6.4.rst index d0221e616..dd10e7e9c 100644 --- a/doc/sphinx/whats-new/changes-6.4.rst +++ b/doc/sphinx/whats-new/changes-6.4.rst @@ -1,12 +1,8 @@ -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` +.. _whatsnew_changes_6.4: -.. _whatsnew_changes_CURRENT: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -Changes in Varnish **$NEXT_RELEASE** -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%% +Changes in Varnish 6.4.0 +%%%%%%%%%%%%%%%%%%%%%%%% For information about updating your current Varnish deployment to the new version, see :ref:`whatsnew_upgrading_CURRENT`. diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst index ad1c6bbcd..29dc31d84 100644 --- a/doc/sphinx/whats-new/index.rst +++ b/doc/sphinx/whats-new/index.rst @@ -21,6 +21,15 @@ released versions of Varnish, see the chapters listed below.** changes-trunk upgrading-trunk +Varnish 6.4 +----------- + +.. toctree:: + :maxdepth: 2 + + changes-6.4 + upgrading-6.4 + Varnish 6.3 ----------- diff --git a/doc/sphinx/whats-new/upgrading-6.4.rst b/doc/sphinx/whats-new/upgrading-6.4.rst index 6143fde99..a65f41bef 100644 --- a/doc/sphinx/whats-new/upgrading-6.4.rst +++ b/doc/sphinx/whats-new/upgrading-6.4.rst @@ -1,12 +1,8 @@ -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` +.. _whatsnew_upgrading_6.4: -.. _whatsnew_upgrading_CURRENT: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -Upgrading to Varnish **$NEXT_RELEASE** -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish 6.4.0 +%%%%%%%%%%%%%%%%%%%%%%%%%% **XXX: how to upgrade from previous deployments to this version. Limited to work that has to be done for an upgrade, new From nils.goroll at uplex.de Wed Mar 11 09:02:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Mar 2020 09:02:07 +0000 (UTC) Subject: [master] 83b0190f0 first round of release docs Message-ID: <20200311090207.C4F44102038@lists.varnish-cache.org> commit 83b0190f091c39de625ae8de921a3648505aed3d Author: Nils Goroll Date: Wed Mar 11 09:57:34 2020 +0100 first round of release docs I went through changes.rst and carried all aspects over to the release docs which I considered relevant. Please feel free to polish, I will not be able to get back to this today. diff --git a/doc/changes.rst b/doc/changes.rst index 478e3083c..bcd4659cb 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -92,8 +92,8 @@ Varnish Cache 6.4.0 (2019-09-16) * The ``if-range`` header is now handled, allowing clients to conditionally request a range based on a date or an ETag. -* Introduced struct reqtop to hold information on the ESI top request - and PRIV_TOP, fixed regression 3019_ +* Introduced ``struct reqtop`` to hold information on the ESI top + request and ``PRIV_TOP``, fixed regression 3019_ * Allow numerical expressions in VCL to be negative / negated @@ -187,12 +187,12 @@ Varnish Cache 6.4.0 (2019-09-16) * Fixed a case where ``send_timeout`` would have no effect when streaming from a backend fetch, see bug 3189_ -* Added ``VSB_tofile()`` to ``libvarnishapi``, see 3238_ - *NOTE* Users upgrading varnish should re-check ``send_timeout`` with respect to long pass and streaming fetches and watch out for increased session close rates. +* Added ``VSB_tofile()`` to ``libvarnishapi``, see 3238_ + .. _1853: https://github.com/varnishcache/varnish-cache/issues/1853 .. _2418: https://github.com/varnishcache/varnish-cache/issues/2418 .. _3009: https://github.com/varnishcache/varnish-cache/issues/3009 diff --git a/doc/sphinx/whats-new/changes-6.4.rst b/doc/sphinx/whats-new/changes-6.4.rst index dd10e7e9c..642d52503 100644 --- a/doc/sphinx/whats-new/changes-6.4.rst +++ b/doc/sphinx/whats-new/changes-6.4.rst @@ -5,7 +5,7 @@ Changes in Varnish 6.4.0 %%%%%%%%%%%%%%%%%%%%%%%% For information about updating your current Varnish deployment to the -new version, see :ref:`whatsnew_upgrading_CURRENT`. +new version, see :ref:`whatsnew_upgrading_6.4`. A more detailed and technical account of changes in Varnish, with links to issues that have been fixed and pull requests that have been @@ -16,8 +16,13 @@ merged, may be found in the `change log`_. varnishd ======== -Parameters -~~~~~~~~~~ +bugs +~~~~ + +Numerous bugs have been fixed. + +Generic Parameter Handling +~~~~~~~~~~~~~~~~~~~~~~~~~~ Some parameters have dependencies and those are better documented now. For example :ref:`ref_param_thread_pool_min` can't be increased above @@ -43,51 +48,133 @@ the error message:: (attempting to set param 'thread_pool_reserve' to '100') -**XXX changes in -p parameters** +Parameters +~~~~~~~~~~ + +* Raised the minimum for the ``vcl_cooldown`` parameter to 1 second. + +Changes in behavior +~~~~~~~~~~~~~~~~~~~ + +* The ``if-range`` header is now handled, allowing clients to conditionally + request a range based on a date or an ETag. -Other changes in varnishd -~~~~~~~~~~~~~~~~~~~~~~~~~ +* Output VCC warnings also for VCLs loaded via the ``varnishd -f`` + option Changes to VCL ============== +* backend ``none`` was added for "no backend". + + It can be used whenever a backend is needed for syntactical + reasons. The ``none`` backend will fail any attempt to use it. + +* ``std.rollback(bereq)`` is now safe to use + +* Deliberately closing backend requests through ``return(abandon)``, + ``return(fail)`` or ``return(error)`` is no longer accounted as a + fetch failure + +* Numerical expressions can now be negative or negated as in ``set + resp.http.o = -std.integer("-200");`` + +* Overloaded the ``+=`` operator to also append to headers as in ``set + reqp.http.header += "add this"`` + VCL variables ~~~~~~~~~~~~~ -**XXX new, deprecated or removed variables, or changed semantics** - -Other changes to VCL -~~~~~~~~~~~~~~~~~~~~ +* Add more vcl control over timeouts with ``sess.timeout_linger``, + ``sess.send_timeout`` and ``sess.idle_send_timeout`` VMODs ===== -**XXX changes in the bundled VMODs** +* Imported ``vmod_cookie`` from `varnish_modules`_ + + The previously deprecated function ``cookie.filter_except()`` has + been removed during import. It was replaced by ``cookie.keep()`` varnishlog ========== -**XXX changes concerning varnishlog(1) and/or vsl(7)** +* A ``Notice`` VSL tag has been added varnishadm ========== -**XXX changes concerning varnishadm(1) and/or varnish-cli(7)** +* New ``pid`` command in the Varnish CLI, to get the master and optionally + cache process PIDs, for example from ``varnishadm``. varnishstat =========== -**XXX changes concerning varnishstat(1) and/or varnish-counters(7)** +* Add vi-stype CTRL-f / CTRL-b for page down/up to interactive + varnishstat -varnishtest -=========== +* The ``MAIN.sess_drop`` counter is gone. -**XXX changes concerning varnishtest(1) and/or vtc(7)** +* Added ``rx_close_idle`` counter for separate accounting when + ``timeout_idle`` is reached. Also, ``send_timeout`` is no longer + reported as "remote closed". Changes for developers and VMOD authors ======================================= -**XXX changes concerning VRT, the public APIs, source code organization, -builds etc.** +general +------- + +* New configure switch: --with-unwind. Alpine linux appears to offer a + ``libexecinfo`` implementation that crashes when called by Varnish, this + offers the alternative of using ``libunwind`` instead. + +* The option ``varnishtest -W`` is gone, the same can be achieved with + ``varnishtest -p debug=+witness``. A ``witness.sh`` script is available + in the source tree to generate a graphviz dot file and detect potential + lock cycles from the test logs. + +* Introduced ``struct reqtop`` to hold information on the ESI top request + and ``PRIV_TOP`` + +* New or improved Coccinelle semantic patches that may be useful for + VMOD or utilities authors. + +* Added ``VSLs()`` and ``VSLbs()`` functions for logging ``STRANDS`` to + VSL + +* Added ``WS_VSB_new()`` / ``WS_VSB_finish()`` for VSBs on workspaces + +* added ``v_dont_optimize`` attribute macro to instruct compilers + (only gcc as of this release) to not optimize a function + +* Added ``VSB_tofile()`` to ``libvarnishapi`` + +VMODs +----- + +* It is now possible for VMOD authors to customize the connection pooling + of a dynamic backend. A hash is now computed to determine uniqueness and + a backend declaration can contribute arbitrary data to influence the pool. + +* ``VRB_Iterate()`` signature has changed + +* ``VRT_fail()`` now also works from director code + +* ``body_status`` and ``req_body_status`` have been collapsed into one + type. In particular, the ``REQ_BODY_*`` enums now have been replaced + with ``BS_*``. + +* Added ``VRT_AllocStrandsWS()`` as a utility function to allocate + STRANDS on a workspace. + +log tools +--------- + +* Log records can safely have empty fields or fields containing blanks + if they are delimited by "double quotes". This was applied to + ``SessError`` and ``Backend_health``. + + Authors of log handling tools might need to review this change. *eof* diff --git a/doc/sphinx/whats-new/upgrading-6.4.rst b/doc/sphinx/whats-new/upgrading-6.4.rst index a65f41bef..1962681e2 100644 --- a/doc/sphinx/whats-new/upgrading-6.4.rst +++ b/doc/sphinx/whats-new/upgrading-6.4.rst @@ -4,26 +4,53 @@ Upgrading to Varnish 6.4.0 %%%%%%%%%%%%%%%%%%%%%%%%%% -**XXX: how to upgrade from previous deployments to this -version. Limited to work that has to be done for an upgrade, new -features are listed in "Changes". Explicitly mention what does *not* -have to be changed, especially in VCL. May include, but is not limited -to:** +Upgrading to Varnish 6.4 from 6.3 should not require any changes +to VCL. -* Elements of VCL that have been removed or are deprecated, or whose - semantics have changed. +This document contains information about other relevant aspects which +should be considered when upgrading. -* -p parameters that have been removed or are deprecated, or whose - semantics have changed. +varnishd +-------- -* Changes in the CLI. +* The hash algorithm of the ``hash`` director was changed, so backend + selection will change once only when upgrading. -* Changes in the output or interpretation of stats or the log, including - changes affecting varnishncsa/-hist/-top. + Users of the ``hash`` director are advised to consider using the + ``shard`` director instead, which, amongst other advantages, offers + more stable backend selection through consistent hashing. -* Changes that may be necessary in VTCs or in the use of varnishtest. +* We fixed a case where ``send_timeout`` had no effect when streaming + from a backend fetch. -* Changes in public APIs that may require changes in VMODs or VAPI/VUT - clients. + Users with long running backend fetches should watch out of + increases connection close rates and consider increasing + ``send_timeout`` appropriately. + + The timeout can also be set per connection from vcl as + ``sess.send_timeout``. + +Statistics +---------- + +* The ``MAIN.sess_drop`` counter is gone. It should be removed from + any statistics gathering tools, if present + +* Added ``rx_close_idle`` counter for separate accounting when + ``timeout_idle`` is reached. Also, ``send_timeout`` is no longer + reported as "remote closed". + +vsl/logs +-------- + +* The ``Process`` timestamp for ``vcl_synth {}`` was wrongly issued + before the VCL subroutine, now it gets emitted after VCL returns for + consistency with ``vcl_deliver {}``. + + Users of this timestamp should be aware that it now includes + ``vcl_synth {}`` processing time and appears at a different + position. + +* A ``Notice`` VSL tag has been added *eof* From nils.goroll at uplex.de Wed Mar 11 09:24:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Mar 2020 09:24:07 +0000 (UTC) Subject: [master] 8bed975da Revert "Start skeleton release notes for the next version." Message-ID: <20200311092407.4FC491028CC@lists.varnish-cache.org> commit 8bed975dac40c9d9a34d045a77f2a233462e3097 Author: Nils Goroll Date: Wed Mar 11 10:19:40 2020 +0100 Revert "Start skeleton release notes for the next version." This reverts commit ac602e0187a5e78821ea093f9c9ef3e6234f3fe3. @Dridi reminded me that we do not want "trunk" release notes in the 6.4 branch to come. diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst deleted file mode 100644 index fbda9d6b9..000000000 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ /dev/null @@ -1,73 +0,0 @@ -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` - -.. _whatsnew_changes_CURRENT: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -Changes in Varnish **$NEXT_RELEASE** -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -For information about updating your current Varnish deployment to the -new version, see :ref:`whatsnew_upgrading_CURRENT`. - -A more detailed and technical account of changes in Varnish, with -links to issues that have been fixed and pull requests that have been -merged, may be found in the `change log`_. - -.. _change log: https://github.com/varnishcache/varnish-cache/blob/master/doc/changes.rst - -varnishd -======== - -Parameters -~~~~~~~~~~ - -**XXX changes in -p parameters** - -Other changes in varnishd -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Changes to VCL -============== - -VCL variables -~~~~~~~~~~~~~ - -**XXX new, deprecated or removed variables, or changed semantics** - -Other changes to VCL -~~~~~~~~~~~~~~~~~~~~ - -VMODs -===== - -**XXX changes in the bundled VMODs** - -varnishlog -========== - -**XXX changes concerning varnishlog(1) and/or vsl(7)** - -varnishadm -========== - -**XXX changes concerning varnishadm(1) and/or varnish-cli(7)** - -varnishstat -=========== - -**XXX changes concerning varnishstat(1) and/or varnish-counters(7)** - -varnishtest -=========== - -**XXX changes concerning varnishtest(1) and/or vtc(7)** - -Changes for developers and VMOD authors -======================================= - -**XXX changes concerning VRT, the public APIs, source code organization, -builds etc.** - -*eof* diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst deleted file mode 100644 index 6143fde99..000000000 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ /dev/null @@ -1,33 +0,0 @@ -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` - -.. _whatsnew_upgrading_CURRENT: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -Upgrading to Varnish **$NEXT_RELEASE** -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -**XXX: how to upgrade from previous deployments to this -version. Limited to work that has to be done for an upgrade, new -features are listed in "Changes". Explicitly mention what does *not* -have to be changed, especially in VCL. May include, but is not limited -to:** - -* Elements of VCL that have been removed or are deprecated, or whose - semantics have changed. - -* -p parameters that have been removed or are deprecated, or whose - semantics have changed. - -* Changes in the CLI. - -* Changes in the output or interpretation of stats or the log, including - changes affecting varnishncsa/-hist/-top. - -* Changes that may be necessary in VTCs or in the use of varnishtest. - -* Changes in public APIs that may require changes in VMODs or VAPI/VUT - clients. - -*eof* From nils.goroll at uplex.de Wed Mar 11 09:24:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Mar 2020 09:24:07 +0000 (UTC) Subject: [master] 0d4495f03 remove reference to trunk docs Message-ID: <20200311092407.6A97B1028CF@lists.varnish-cache.org> commit 0d4495f038a92e818aac94aab07025c150907b26 Author: Nils Goroll Date: Wed Mar 11 10:20:24 2020 +0100 remove reference to trunk docs diff --git a/doc/changes.rst b/doc/changes.rst index bcd4659cb..ebeb7de4a 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -26,10 +26,6 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via individual releases. These documents are updated as part of the release process. -================================ -NEXT (2020-09-15) -================================ - ================================ Varnish Cache 6.4.0 (2019-09-16) ================================ diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst index 29dc31d84..652248b20 100644 --- a/doc/sphinx/whats-new/index.rst +++ b/doc/sphinx/whats-new/index.rst @@ -8,19 +8,6 @@ This section describes the changes and improvements between different versions of Varnish, and what upgrading between the different versions entail. -Varnish **$NEXT_RELEASE** -------------------------- - -**Note: These are working documents for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see the chapters listed below.** - -.. toctree:: - :maxdepth: 2 - - changes-trunk - upgrading-trunk - Varnish 6.4 ----------- From nils.goroll at uplex.de Wed Mar 11 09:24:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 11 Mar 2020 09:24:07 +0000 (UTC) Subject: [master] 7862b1387 fix rst syntax Message-ID: <20200311092407.8EACB1028D3@lists.varnish-cache.org> commit 7862b1387a1d07a9fdfe1e91041049a60968c979 Author: Nils Goroll Date: Wed Mar 11 10:23:00 2020 +0100 fix rst syntax diff --git a/doc/sphinx/whats-new/changes-6.4.rst b/doc/sphinx/whats-new/changes-6.4.rst index 642d52503..11d9ee23e 100644 --- a/doc/sphinx/whats-new/changes-6.4.rst +++ b/doc/sphinx/whats-new/changes-6.4.rst @@ -123,7 +123,7 @@ Changes for developers and VMOD authors ======================================= general -------- +~~~~~~~ * New configure switch: --with-unwind. Alpine linux appears to offer a ``libexecinfo`` implementation that crashes when called by Varnish, this @@ -151,7 +151,7 @@ general * Added ``VSB_tofile()`` to ``libvarnishapi`` VMODs ------ +~~~~~ * It is now possible for VMOD authors to customize the connection pooling of a dynamic backend. A hash is now computed to determine uniqueness and @@ -169,7 +169,7 @@ VMODs STRANDS on a workspace. log tools ---------- +~~~~~~~~~ * Log records can safely have empty fields or fields containing blanks if they are delimited by "double quotes". This was applied to @@ -178,3 +178,5 @@ log tools Authors of log handling tools might need to review this change. *eof* + +.. _varnish_modules: https://github.com/varnish/varnish-modules From dridi.boukelmoune at gmail.com Wed Mar 11 09:28:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 11 Mar 2020 09:28:07 +0000 (UTC) Subject: [master] 20c8aafb2 Swap the VSB_tofile() arguments Message-ID: <20200311092807.35C80102E04@lists.varnish-cache.org> commit 20c8aafb281ed5693771d65dc4e33e827173946c Author: Dridi Boukelmoune Date: Wed Mar 11 10:14:01 2020 +0100 Swap the VSB_tofile() arguments Automated with Coccinelle, so the semantic patch could be reused in the vtest project. Closes #3238 diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 383d90c36..a60efd918 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -208,7 +208,7 @@ vsmw_append_record(struct vsmw *vsmw, struct vsmwseg *seg, char act) VSB_clear(vsmw->vsb); vsmw_fmt_index(vsmw, seg, act); AZ(VSB_finish(vsmw->vsb)); - XXXAZ(VSB_tofile(fd, vsmw->vsb)); // XXX handle ENOSPC? #2764 + XXXAZ(VSB_tofile(vsmw->vsb, fd)); // XXX handle ENOSPC? #2764 closefd(&fd); } @@ -255,7 +255,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg) VTAILQ_FOREACH(s2, &vsmw->segs, list) vsmw_fmt_index(vsmw, s2, '+'); AZ(VSB_finish(vsmw->vsb)); - XXXAZ(VSB_tofile(fd, vsmw->vsb)); // XXX handle ENOSPC? #2764 + XXXAZ(VSB_tofile(vsmw->vsb, fd)); // XXX handle ENOSPC? #2764 closefd(&fd); AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx)); REPLACE(t, NULL); diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index 8cf182bac..b046d20c7 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -147,7 +147,7 @@ mcf_askchild(struct cli *cli, const char * const *av, void *priv) } VSB_putc(cli_buf, '\n'); AZ(VSB_finish(cli_buf)); - if (VSB_tofile(cli_o, cli_buf)) { + if (VSB_tofile(cli_buf, cli_o)) { VCLI_SetResult(cli, CLIS_COMMS); VCLI_Out(cli, "CLI communication error"); MCH_Cli_Fail(); @@ -196,7 +196,7 @@ mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...) AZ(VSB_finish(cli_buf)); i = VSB_len(cli_buf); assert(i > 0 && VSB_data(cli_buf)[i - 1] == '\n'); - if (VSB_tofile(cli_o, cli_buf)) { + if (VSB_tofile(cli_buf, cli_o)) { *status = CLIS_COMMS; if (resp != NULL) *resp = strdup("CLI communication error"); diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 80fb8b827..c650f81b5 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -229,7 +229,7 @@ cmd_haproxy_cli_send(CMD_ARGS) } vtc_dump(hc->vl, 4, "CLI send", VSB_data(vsb), -1); - if (VSB_tofile(hc->sock, vsb)) + if (VSB_tofile(vsb, hc->sock)) vtc_fatal(hc->vl, "CLI fd %d send error %s", hc->sock, strerror(errno)); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 85441f25f..b122aa343 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -217,7 +217,7 @@ http_write(const struct http *hp, int lvl, const char *pfx) AZ(VSB_finish(hp->vsb)); vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb)); - if (VSB_tofile(hp->fd, hp->vsb)) + if (VSB_tofile(hp->vsb, hp->fd)) vtc_log(hp->vl, hp->fatal, "Write failed: %s", strerror(errno)); } @@ -1381,7 +1381,7 @@ cmd_http_sendhex(CMD_ARGS) vsb = vtc_hex_to_bin(hp->vl, av[1]); assert(VSB_len(vsb) >= 0); vtc_hexdump(hp->vl, 4, "sendhex", VSB_data(vsb), VSB_len(vsb)); - if (VSB_tofile(hp->fd, vsb)) + if (VSB_tofile(vsb, hp->fd)) vtc_log(hp->vl, hp->fatal, "Write failed: %s", strerror(errno)); VSB_destroy(&vsb); diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 99c68cc0c..24f7bab0b 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -803,7 +803,7 @@ process_write_hex(const struct process *p, const char *text) vsb = vtc_hex_to_bin(p->vl, text); assert(VSB_len(vsb) >= 0); vtc_hexdump(p->vl, 4, "sendhex", VSB_data(vsb), VSB_len(vsb)); - AZ(VSB_tofile(p->fd_term, vsb)); + AZ(VSB_tofile(vsb, p->fd_term)); VSB_destroy(&vsb); } diff --git a/bin/varnishtest/vtc_proxy.c b/bin/varnishtest/vtc_proxy.c index 7a3fd7ac7..1bee39657 100644 --- a/bin/varnishtest/vtc_proxy.c +++ b/bin/varnishtest/vtc_proxy.c @@ -128,7 +128,7 @@ vtc_send_proxy(int fd, int version, const struct suckaddr *sac, WRONG("Wrong proxy version"); AZ(VSB_finish(vsb)); - i = VSB_tofile(fd, vsb); + i = VSB_tofile(vsb, fd); VSB_delete(vsb); return (i); } diff --git a/include/vsb.h b/include/vsb.h index a8fe4f6e5..b4305e42a 100644 --- a/include/vsb.h +++ b/include/vsb.h @@ -89,7 +89,7 @@ void VSB_quote_pfx(struct vsb *, const char*, const void *, int len, int how); void VSB_quote(struct vsb *, const void *, int len, int how); void VSB_indent(struct vsb *, int); -int VSB_tofile(int fd, const struct vsb *); +int VSB_tofile(const struct vsb *, int fd); #ifdef __cplusplus }; #endif diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c index e0b48c50e..4c4570c7b 100644 --- a/lib/libvarnish/vsb.c +++ b/lib/libvarnish/vsb.c @@ -658,7 +658,7 @@ VSB_indent(struct vsb *s, int i) } int -VSB_tofile(int fd, const struct vsb *s) +VSB_tofile(const struct vsb *s, int fd) { int sz; diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 1be668bfd..87c6614e9 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -110,7 +110,7 @@ TLWriteVSB(struct vcc *tl, const char *fn, const struct vsb *vsb, what, fn, strerror(errno)); return (-1); } - i = VSB_tofile(fo, vsb); + i = VSB_tofile(vsb, fo); if (i) { VSB_printf(tl->sb, "Could not write %s to %s: %s\n", diff --git a/tools/coccinelle/vsb_tofile.cocci b/tools/coccinelle/vsb_tofile.cocci new file mode 100644 index 000000000..b8ca8254d --- /dev/null +++ b/tools/coccinelle/vsb_tofile.cocci @@ -0,0 +1,46 @@ +/* + * This patch fixes the order of VSB_tofile arguments. + */ + +@@ +idexpression struct vsb *vsb; +idexpression int fd; +@@ + +- VSB_tofile(fd, vsb) ++ VSB_tofile(vsb, fd) + +@@ +idexpression struct vsb[] vsb; +idexpression int fd; +@@ + +- VSB_tofile(fd, vsb) ++ VSB_tofile(vsb, fd) + +@@ +idexpression struct vsb *vsb; +expression fd; +@@ + +- VSB_tofile(fd, vsb) ++ VSB_tofile(vsb, fd) + +/* Opportunistic fallback */ + +@@ +idexpression int fd; +expression vsb; +@@ + +- VSB_tofile(fd, vsb) ++ VSB_tofile(vsb, fd) + +/* Opportunistic last resort */ + +@@ +expression fd, other; +@@ + +- VSB_tofile(fd, other->vsb) ++ VSB_tofile(other->vsb, fd) From dridi.boukelmoune at gmail.com Wed Mar 11 11:48:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 11 Mar 2020 11:48:07 +0000 (UTC) Subject: [master] c97ed2dde Missing manual pages dependencies Message-ID: <20200311114807.3227110695F@lists.varnish-cache.org> commit c97ed2dde09df5b5222fb294d534f754d0567498 Author: Dridi Boukelmoune Date: Wed Mar 11 12:46:25 2020 +0100 Missing manual pages dependencies Refs 0cf29671a4f2 diff --git a/man/Makefile.am b/man/Makefile.am index c341ddc87..53646cde6 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -30,10 +30,12 @@ RST2ANY_FLAGS = --halt=2 BUILD_MAN = $(AM_V_GEN) $(RST2MAN) $(RST2ANY_FLAGS) -varnish-cli.7: $(top_builddir)/doc/sphinx/reference/varnish-cli.rst +varnish-cli.7: $(top_builddir)/doc/sphinx/reference/varnish-cli.rst \ + $(top_srcdir)/doc/sphinx/include/cli.rst $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@ -varnish-counters.7: $(top_builddir)/doc/sphinx/reference/varnish-counters.rst +varnish-counters.7: $(top_builddir)/doc/sphinx/reference/varnish-counters.rst \ + $(top_srcdir)/doc/sphinx/include/counters.rst $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnish-counters.rst $@ vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \ @@ -70,7 +72,8 @@ varnishlog.1: \ varnishstat.1: $(top_builddir)/doc/sphinx/reference/varnishstat.rst \ $(top_builddir)/doc/sphinx/include/varnishstat_options.rst \ - $(top_builddir)/doc/sphinx/include/varnishstat_synopsis.rst + $(top_builddir)/doc/sphinx/include/varnishstat_synopsis.rst \ + $(top_builddir)/doc/sphinx/include/varnishstat_bindings.rst $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@ varnishtest.1: $(top_builddir)/doc/sphinx/reference/varnishtest.rst From dridi.boukelmoune at gmail.com Thu Mar 12 14:12:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 12 Mar 2020 14:12:06 +0000 (UTC) Subject: [master] b39b8b2b1 More platform dependent defaults Message-ID: <20200312141206.CE98E104A88@lists.varnish-cache.org> commit b39b8b2b1c1cd7499d3eda6d63d89d21c81d8c90 Author: Dridi Boukelmoune Date: Thu Mar 12 15:08:44 2020 +0100 More platform dependent defaults diff --git a/include/tbl/params.h b/include/tbl/params.h index b75bc9c68..6773d9ae5 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -856,12 +856,15 @@ PARAM( /* type */ timeout, /* min */ "1", /* max */ "100", - /* def */ "", + /* def */ NULL, /* units */ "seconds", /* descr */ "The number of seconds between TCP keep-alive probes. " "Ignored for Unix domain sockets.", - /* flags */ XYZZY + /* flags */ XYZZY, + /* dyn_min_reason */ NULL, + /* dyn_max_reason */ NULL, + /* dyn_def_reason */ "platform dependent" ) PARAM( @@ -869,13 +872,16 @@ PARAM( /* type */ uint, /* min */ "1", /* max */ "100", - /* def */ "", + /* def */ NULL, /* units */ "probes", /* descr */ "The maximum number of TCP keep-alive probes to send before giving " "up and killing the connection if no response is obtained from the " "other end. Ignored for Unix domain sockets.", - /* flags */ XYZZY + /* flags */ XYZZY, + /* dyn_min_reason */ NULL, + /* dyn_max_reason */ NULL, + /* dyn_def_reason */ "platform dependent" ) PARAM( @@ -883,13 +889,16 @@ PARAM( /* type */ timeout, /* min */ "1", /* max */ "7200", - /* def */ "", + /* def */ NULL, /* units */ "seconds", /* descr */ "The number of seconds a connection needs to be idle before TCP " "begins sending out keep-alive probes. " "Ignored for Unix domain sockets.", - /* flags */ XYZZY + /* flags */ XYZZY, + /* dyn_min_reason */ NULL, + /* dyn_max_reason */ NULL, + /* dyn_def_reason */ "platform dependent" ) #undef XYZZY From dridi.boukelmoune at gmail.com Fri Mar 13 08:15:10 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 13 Mar 2020 08:15:10 +0000 (UTC) Subject: [master] b932d06e9 Use our DNS canary in the test suite Message-ID: <20200313081511.105919175@lists.varnish-cache.org> commit b932d06e924ec8a622195133fc4516020f7b9b2d Author: Dridi Boukelmoune Date: Fri Mar 13 09:12:47 2020 +0100 Use our DNS canary in the test suite Closes #3237 diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index 7a0252320..6f09d10fb 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -85,7 +85,7 @@ varnish v1 -errvcl {Regexp compilation error:} { varnish v1 -errvcl {resolves to too many addresses} { backend b { .host = "127.0.0.1"; } sub vcl_recv { - if (remote.ip == "fastly.com") {} + if (remote.ip == "dns-canary-multi.varnish-cache.org") {} } } diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index dd0cd2e6f..db604ecc6 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -354,21 +354,21 @@ dns_works(void) char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; - sa = VSS_ResolveOne(NULL, "varnish.org", NULL, + sa = VSS_ResolveOne(NULL, "dns-canary.varnish-cache.org", NULL, AF_INET, SOCK_STREAM, 0); if (sa == NULL) return (0); VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf); free(sa); - if (strcmp(abuf, "176.58.90.154")) + if (strcmp(abuf, "192.0.2.255")) return (0); - sa = VSS_ResolveOne(NULL, "varnish.org", NULL, + sa = VSS_ResolveOne(NULL, "dns-canary.varnish-cache.org", NULL, AF_INET6, SOCK_STREAM, 0); if (sa == NULL) - return (0); + return (1); /* the canary is ipv4 only */ free(sa); - return (1); + return (0); } /* SECTION: feature feature From nils.goroll at uplex.de Fri Mar 13 08:30:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 13 Mar 2020 08:30:07 +0000 (UTC) Subject: [master] 965a1f4e5 release docs polish Message-ID: <20200313083007.974B860508@lists.varnish-cache.org> commit 965a1f4e53f1d05014cd0daacdf919cd4f306c6e Author: Nils Goroll Date: Fri Mar 13 09:29:10 2020 +0100 release docs polish diff --git a/doc/sphinx/whats-new/upgrading-6.4.rst b/doc/sphinx/whats-new/upgrading-6.4.rst index 1962681e2..53ead0a4a 100644 --- a/doc/sphinx/whats-new/upgrading-6.4.rst +++ b/doc/sphinx/whats-new/upgrading-6.4.rst @@ -18,16 +18,27 @@ varnishd Users of the ``hash`` director are advised to consider using the ``shard`` director instead, which, amongst other advantages, offers - more stable backend selection through consistent hashing. + more stable backend selection through consistent hashing. See + :ref:`vmod_directors(3)` for details. -* We fixed a case where ``send_timeout`` had no effect when streaming - from a backend fetch. +* We fixed a case where ``send_timeout`` had no effect on HTTP/1 + connections when streaming from a backend fetch, in other words, a + connection might not have got closed despite the ``send_timeout`` + having been reached. HTTP/2 was not affected. - Users with long running backend fetches should watch out of - increases connection close rates and consider increasing - ``send_timeout`` appropriately. + When ``send_timeout`` is reached on HTTP/1, the ``MAIN.sc_tx_error`` + is increased and a ``Debug`` message ``Hit total send timeout, wrote + = x/y; not retrying`` is logged. - The timeout can also be set per connection from vcl as + .. actually H2 is really quite different and we have a plan to + harmonize timeout handling across the board + + Users with long running backend fetches and HTTP/1 clients should + watch out for an increase of the ``MAIN.sc_tx_error`` compared to + before the deployment and consider increasing ``send_timeout`` + appropriately. + + The timeout can also be set per connection from VCL as ``sess.send_timeout``. Statistics @@ -36,20 +47,24 @@ Statistics * The ``MAIN.sess_drop`` counter is gone. It should be removed from any statistics gathering tools, if present -* Added ``rx_close_idle`` counter for separate accounting when - ``timeout_idle`` is reached. Also, ``send_timeout`` is no longer - reported as "remote closed". +* ``timeout_idle`` being reached on HTTP/1 used to be accounted to the + ``MAIN.rx_timeout`` statistic. We have now added the + ``MAIN.rx_close_idle`` counter for this case specifically. + +* ``send_timeout`` being reached on HTTP/1 used to be accounted to + ``MAIN.sc_rem_close``. Such timeout events are now accounted towards + ``MAIN.sc_tx_error``. vsl/logs -------- * The ``Process`` timestamp for ``vcl_synth {}`` was wrongly issued - before the VCL subroutine, now it gets emitted after VCL returns for - consistency with ``vcl_deliver {}``. + before the VCL subroutine was called, now it gets emitted after VCL + returns for consistency with ``vcl_deliver {}``. Users of this timestamp should be aware that it now includes ``vcl_synth {}`` processing time and appears at a different - position. + position in the log. * A ``Notice`` VSL tag has been added From dridi at varni.sh Fri Mar 13 09:00:53 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Fri, 13 Mar 2020 09:00:53 +0000 Subject: [master] 965a1f4e5 release docs polish In-Reply-To: <20200313083007.974B860508@lists.varnish-cache.org> References: <20200313083007.974B860508@lists.varnish-cache.org> Message-ID: On Fri, Mar 13, 2020 at 8:30 AM Nils Goroll wrote: > > > commit 965a1f4e53f1d05014cd0daacdf919cd4f306c6e > Author: Nils Goroll > Date: Fri Mar 13 09:29:10 2020 +0100 > > release docs polish > > diff --git a/doc/sphinx/whats-new/upgrading-6.4.rst b/doc/sphinx/whats-new/upgrading-6.4.rst > index 1962681e2..53ead0a4a 100644 > --- a/doc/sphinx/whats-new/upgrading-6.4.rst > +++ b/doc/sphinx/whats-new/upgrading-6.4.rst > @@ -18,16 +18,27 @@ varnishd > > Users of the ``hash`` director are advised to consider using the > ``shard`` director instead, which, amongst other advantages, offers > - more stable backend selection through consistent hashing. > + more stable backend selection through consistent hashing. See > + :ref:`vmod_directors(3)` for details. > > -* We fixed a case where ``send_timeout`` had no effect when streaming > - from a backend fetch. > +* We fixed a case where ``send_timeout`` had no effect on HTTP/1 > + connections when streaming from a backend fetch, in other words, a > + connection might not have got closed despite the ``send_timeout`` > + having been reached. HTTP/2 was not affected. > > - Users with long running backend fetches should watch out of > - increases connection close rates and consider increasing > - ``send_timeout`` appropriately. > + When ``send_timeout`` is reached on HTTP/1, the ``MAIN.sc_tx_error`` > + is increased and a ``Debug`` message ``Hit total send timeout, wrote > + = x/y; not retrying`` is logged. I don't think it's a good idea to suggest looking for Debug records, they are part of the default mask. With c186423b5edde99d7fc487cd4b785a7ee73dfdd0 we improved things a bit, and they can look for "SessClose TX_ERROR" instead, or as you suggest below the sc_tx_error counter. > - The timeout can also be set per connection from vcl as > + .. actually H2 is really quite different and we have a plan to > + harmonize timeout handling across the board > + > + Users with long running backend fetches and HTTP/1 clients should > + watch out for an increase of the ``MAIN.sc_tx_error`` compared to > + before the deployment and consider increasing ``send_timeout`` > + appropriately. > + > + The timeout can also be set per connection from VCL as > ``sess.send_timeout``. > > Statistics > @@ -36,20 +47,24 @@ Statistics > * The ``MAIN.sess_drop`` counter is gone. It should be removed from > any statistics gathering tools, if present > > -* Added ``rx_close_idle`` counter for separate accounting when > - ``timeout_idle`` is reached. Also, ``send_timeout`` is no longer > - reported as "remote closed". > +* ``timeout_idle`` being reached on HTTP/1 used to be accounted to the > + ``MAIN.rx_timeout`` statistic. We have now added the > + ``MAIN.rx_close_idle`` counter for this case specifically. > + > +* ``send_timeout`` being reached on HTTP/1 used to be accounted to > + ``MAIN.sc_rem_close``. Such timeout events are now accounted towards > + ``MAIN.sc_tx_error``. > > vsl/logs > -------- > > * The ``Process`` timestamp for ``vcl_synth {}`` was wrongly issued > - before the VCL subroutine, now it gets emitted after VCL returns for > - consistency with ``vcl_deliver {}``. > + before the VCL subroutine was called, now it gets emitted after VCL > + returns for consistency with ``vcl_deliver {}``. > > Users of this timestamp should be aware that it now includes > ``vcl_synth {}`` processing time and appears at a different > - position. > + position in the log. > > * A ``Notice`` VSL tag has been added > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Fri Mar 13 09:14:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 13 Mar 2020 09:14:06 +0000 (UTC) Subject: [master] 0a8d3c5be add vmod_cookie to the documentation Message-ID: <20200313091406.DAF8F618BF@lists.varnish-cache.org> commit 0a8d3c5be1a09a5f7032cf923500c4d2ef5d6ed2 Author: Nils Goroll Date: Fri Mar 13 10:12:43 2020 +0100 add vmod_cookie to the documentation diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 41305b4a2..ea8d04076 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -214,6 +214,8 @@ include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC) @mv ${@}_ ${@} BUILT_SOURCES += include/vtc-syntax.rst +# XXX copy/paste rules need some TLB + include/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@ BUILT_SOURCES += include/vmod_std.generated.rst @@ -234,6 +236,10 @@ include/vmod_blob.generated.rst: $(top_builddir)/lib/libvmod_blob/vmod_blob.rst cp $(top_builddir)/lib/libvmod_blob/vmod_blob.rst $@ BUILT_SOURCES += include/vmod_blob.generated.rst +include/vmod_cookie.generated.rst: $(top_builddir)/lib/libvmod_cookie/vmod_cookie.rst + cp $(top_builddir)/lib/libvmod_cookie/vmod_cookie.rst $@ +BUILT_SOURCES += include/vmod_cookie.generated.rst + include/vmod_unix.generated.rst: $(top_builddir)/lib/libvmod_unix/vmod_unix.rst cp $(top_builddir)/lib/libvmod_unix/vmod_unix.rst $@ BUILT_SOURCES += include/vmod_unix.generated.rst diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst index 832fc39af..934cbc02f 100644 --- a/doc/sphinx/reference/index.rst +++ b/doc/sphinx/reference/index.rst @@ -19,14 +19,15 @@ The Varnish Reference Manual varnishtest.rst varnishtop.rst vsm.rst - vmod.rst - vmod_std.rst + vmod_blob.rst + vmod_cookie.rst vmod_directors.rst + vmod_proxy.rst vmod_purge.rst - vmod_blob.rst + vmod_std.rst vmod_unix.rst - vmod_proxy.rst vmod_vtc.rst + vmod.rst directors.rst varnish-counters.rst vsl.rst @@ -34,6 +35,8 @@ The Varnish Reference Manual vtc.rst vtla.rst +.. vmod_debug left out deliberately ? + .. todo:: The programs: . varnishd manual page diff --git a/doc/sphinx/reference/vmod_cookie.rst b/doc/sphinx/reference/vmod_cookie.rst new file mode 100644 index 000000000..f38ff037c --- /dev/null +++ b/doc/sphinx/reference/vmod_cookie.rst @@ -0,0 +1,2 @@ + +.. include:: ../include/vmod_cookie.generated.rst From nils.goroll at uplex.de Fri Mar 13 09:14:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 13 Mar 2020 09:14:06 +0000 (UTC) Subject: [master] b011633e9 release docs: better links and some more polishing Message-ID: <20200313091406.F385E618C5@lists.varnish-cache.org> commit b011633e98d0c16fc0c661fdef8d974ab95b8a25 Author: Nils Goroll Date: Fri Mar 13 10:13:07 2020 +0100 release docs: better links and some more polishing diff --git a/doc/sphinx/whats-new/changes-6.4.rst b/doc/sphinx/whats-new/changes-6.4.rst index 11d9ee23e..afc45a4bf 100644 --- a/doc/sphinx/whats-new/changes-6.4.rst +++ b/doc/sphinx/whats-new/changes-6.4.rst @@ -51,7 +51,8 @@ the error message:: Parameters ~~~~~~~~~~ -* Raised the minimum for the ``vcl_cooldown`` parameter to 1 second. +* Raised the minimum for the :ref:`ref_param_vcl_cooldown` parameter + to 1 second. Changes in behavior ~~~~~~~~~~~~~~~~~~~ @@ -70,7 +71,8 @@ Changes to VCL It can be used whenever a backend is needed for syntactical reasons. The ``none`` backend will fail any attempt to use it. -* ``std.rollback(bereq)`` is now safe to use +* ``std.rollback(bereq)`` is now safe to use, see :ref:`vmod_std(3)` + for details. * Deliberately closing backend requests through ``return(abandon)``, ``return(fail)`` or ``return(error)`` is no longer accounted as a @@ -85,13 +87,14 @@ Changes to VCL VCL variables ~~~~~~~~~~~~~ -* Add more vcl control over timeouts with ``sess.timeout_linger``, - ``sess.send_timeout`` and ``sess.idle_send_timeout`` +* Add more vcl control over timeouts with the ``sess.timeout_linger``, + ``sess.send_timeout`` and ``sess.idle_send_timeout`` variables + corresponding the parameters by the same names. VMODs ===== -* Imported ``vmod_cookie`` from `varnish_modules`_ +* Imported :ref:`vmod_cookie(3)` from `varnish_modules`_ The previously deprecated function ``cookie.filter_except()`` has been removed during import. It was replaced by ``cookie.keep()`` @@ -116,8 +119,11 @@ varnishstat * The ``MAIN.sess_drop`` counter is gone. * Added ``rx_close_idle`` counter for separate accounting when - ``timeout_idle`` is reached. Also, ``send_timeout`` is no longer - reported as "remote closed". + ``sess.timeout_idle`` / :ref:`ref_param_timeout_idle` is reached. + +* ``sess.send_timeout`` / :ref:`ref_param_send_timeout` being reached + is no longer reported as ``MAIN.sc_rem_close``, but as + ``MAIN.sc_tx_error``. Changes for developers and VMOD authors ======================================= diff --git a/doc/sphinx/whats-new/upgrading-6.4.rst b/doc/sphinx/whats-new/upgrading-6.4.rst index 53ead0a4a..557f37697 100644 --- a/doc/sphinx/whats-new/upgrading-6.4.rst +++ b/doc/sphinx/whats-new/upgrading-6.4.rst @@ -21,12 +21,12 @@ varnishd more stable backend selection through consistent hashing. See :ref:`vmod_directors(3)` for details. -* We fixed a case where ``send_timeout`` had no effect on HTTP/1 +* We fixed a case where :ref:`ref_param_send_timeout` had no effect on HTTP/1 connections when streaming from a backend fetch, in other words, a - connection might not have got closed despite the ``send_timeout`` + connection might not have got closed despite the :ref:`ref_param_send_timeout` having been reached. HTTP/2 was not affected. - When ``send_timeout`` is reached on HTTP/1, the ``MAIN.sc_tx_error`` + When :ref:`ref_param_send_timeout` is reached on HTTP/1, the ``MAIN.sc_tx_error`` is increased and a ``Debug`` message ``Hit total send timeout, wrote = x/y; not retrying`` is logged. @@ -35,8 +35,8 @@ varnishd Users with long running backend fetches and HTTP/1 clients should watch out for an increase of the ``MAIN.sc_tx_error`` compared to - before the deployment and consider increasing ``send_timeout`` - appropriately. + before the deployment and consider increasing + :ref:`ref_param_send_timeout` appropriately. The timeout can also be set per connection from VCL as ``sess.send_timeout``. @@ -47,13 +47,16 @@ Statistics * The ``MAIN.sess_drop`` counter is gone. It should be removed from any statistics gathering tools, if present -* ``timeout_idle`` being reached on HTTP/1 used to be accounted to the - ``MAIN.rx_timeout`` statistic. We have now added the - ``MAIN.rx_close_idle`` counter for this case specifically. +* ``sess.timeout_idle`` / :ref:`ref_param_timeout_idle` being reached + on HTTP/1 used to be accounted to the ``MAIN.rx_timeout`` + statistic. We have now added the ``MAIN.rx_close_idle`` counter for + this case specifically. -* ``send_timeout`` being reached on HTTP/1 used to be accounted to - ``MAIN.sc_rem_close``. Such timeout events are now accounted towards - ``MAIN.sc_tx_error``. +* ``sess.send_timeout`` / :ref:`ref_param_send_timeout` being reached + on HTTP/1 used to be accounted to ``MAIN.sc_rem_close``. Such + timeout events are now accounted towards ``MAIN.sc_tx_error``. + +see :ref:`varnish-counters(7)` for details vsl/logs -------- From nils.goroll at uplex.de Fri Mar 13 09:17:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 13 Mar 2020 09:17:06 +0000 (UTC) Subject: [master] f4c1ad417 fix comment Message-ID: <20200313091706.55A4661CD1@lists.varnish-cache.org> commit f4c1ad417412e83e0678544bf6ee5fed7ec7c89a Author: Nils Goroll Date: Fri Mar 13 10:15:45 2020 +0100 fix comment no, they do not need a Translation Lookaside Buffer diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index ea8d04076..82ba04760 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -214,7 +214,7 @@ include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC) @mv ${@}_ ${@} BUILT_SOURCES += include/vtc-syntax.rst -# XXX copy/paste rules need some TLB +# XXX copy/paste rules need some TLC include/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@ From nils.goroll at uplex.de Fri Mar 13 10:10:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 13 Mar 2020 10:10:06 +0000 (UTC) Subject: [master] 31d22d44a release docs: Do not mention Debug logs Message-ID: <20200313101006.D555463227@lists.varnish-cache.org> commit 31d22d44a624e133eaba0f69b254e8f752450f41 Author: Nils Goroll Date: Fri Mar 13 11:08:53 2020 +0100 release docs: Do not mention Debug logs As @Dridi rightfully commented via email, the default vsl_mask does not contain Debug diff --git a/doc/sphinx/whats-new/upgrading-6.4.rst b/doc/sphinx/whats-new/upgrading-6.4.rst index 557f37697..b70da8d23 100644 --- a/doc/sphinx/whats-new/upgrading-6.4.rst +++ b/doc/sphinx/whats-new/upgrading-6.4.rst @@ -26,21 +26,18 @@ varnishd connection might not have got closed despite the :ref:`ref_param_send_timeout` having been reached. HTTP/2 was not affected. - When :ref:`ref_param_send_timeout` is reached on HTTP/1, the ``MAIN.sc_tx_error`` - is increased and a ``Debug`` message ``Hit total send timeout, wrote - = x/y; not retrying`` is logged. - - .. actually H2 is really quite different and we have a plan to - harmonize timeout handling across the board - - Users with long running backend fetches and HTTP/1 clients should - watch out for an increase of the ``MAIN.sc_tx_error`` compared to - before the deployment and consider increasing + When :ref:`ref_param_send_timeout` is reached on HTTP/1, the + ``MAIN.sc_tx_error`` is increased. Users with long running backend + fetches and HTTP/1 clients should watch out for an increase of that + counter compared to before the deployment and consider increasing :ref:`ref_param_send_timeout` appropriately. The timeout can also be set per connection from VCL as ``sess.send_timeout``. + .. actually H2 is really quite different and we have a plan to + harmonize timeout handling across the board + Statistics ---------- From nils.goroll at uplex.de Fri Mar 13 10:51:05 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 13 Mar 2020 10:51:05 +0000 (UTC) Subject: [master] 0ec0747ad stabilize test Message-ID: <20200313105106.06E6664279@lists.varnish-cache.org> commit 0ec0747ad943d2b9c5710ad6915ad623fa5bb66f Author: Nils Goroll Date: Fri Mar 13 11:49:15 2020 +0100 stabilize test have seen this failing on Travis with: **** v1 vsl| 1004 VCL_call b BACKEND_FETCH **** v1 vsl| 1004 VCL_return b abandon **** v1 vsl| 1004 BereqAcct b 0 0 0 0 0 0 Tested ok locally with varnishtest -n 1000 -j 60. diff --git a/bin/varnishtest/tests/d00036.vtc b/bin/varnishtest/tests/d00036.vtc index b75278b44..d21646bf8 100644 --- a/bin/varnishtest/tests/d00036.vtc +++ b/bin/varnishtest/tests/d00036.vtc @@ -1,6 +1,6 @@ varnishtest "Test dynamic UDS backend hot swap after it was picked by a bereq" -barrier b1 cond 2 +barrier b1 sock 2 server s1 -listen "${tmpdir}/s1.sock" { } -start @@ -30,8 +30,8 @@ varnish v1 -vcl { sub vcl_backend_fetch { set bereq.backend = s1.backend(); - # hot swap should happen while we sleep - vtc.sleep(2s); + # hot swap has happened + vtc.barrier_sync("${b1_sock}"); if (std.healthy(bereq.backend)) { return(abandon); } else { @@ -44,17 +44,16 @@ varnish v1 -expect MAIN.n_backend == 1 client c1 { txreq - barrier b1 sync rxresp expect resp.status == 200 } client c2 { - barrier b1 sync delay 0.1 txreq -req "SWAP" -hdr "X-Path: ${s2_sock}" rxresp expect resp.status == 200 + barrier b1 sync } client c1 -start From dridi.boukelmoune at gmail.com Fri Mar 13 15:22:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 13 Mar 2020 15:22:06 +0000 (UTC) Subject: [master] ad8f450f4 Allow failures for Travis CI arm64 jobs Message-ID: <20200313152206.B4D20104B84@lists.varnish-cache.org> commit ad8f450f4097ce79459a716afc03075f4caf9561 Author: Dridi Boukelmoune Date: Fri Mar 13 13:42:14 2020 +0100 Allow failures for Travis CI arm64 jobs We are finding more flaky tests thanks to them, so for now we prefer to avoid false positive over slowing down the build. diff --git a/.travis.yml b/.travis.yml index bf31b432b..ad0b036f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ jobs: allow_failures: - os: osx - stage: sanitizers + - arch: arm64 fast_finish: true include: - &test-linux From dridi.boukelmoune at gmail.com Fri Mar 13 15:26:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 13 Mar 2020 15:26:07 +0000 (UTC) Subject: [master] 8e2e13f19 Be more specific than EXTRA_DIST Message-ID: <20200313152607.67EC8104E13@lists.varnish-cache.org> commit 8e2e13f19a6a868a7784b52248ee37ffd1701564 Author: Dridi Boukelmoune Date: Fri Mar 13 16:25:12 2020 +0100 Be more specific than EXTRA_DIST diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am index ce550df7a..9333284fd 100644 --- a/lib/libvcc/Makefile.am +++ b/lib/libvcc/Makefile.am @@ -32,7 +32,7 @@ libvcc_a_SOURCES = \ vcc_vmod.c \ vcc_xref.c -EXTRA_DIST = \ +dist_noinst_SCRIPTS = \ generate.py dist_pkgdata_SCRIPTS = \ From dridi.boukelmoune at gmail.com Fri Mar 13 16:16:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 13 Mar 2020 16:16:06 +0000 (UTC) Subject: [master] 28fe20266 More 6.4 release notes Message-ID: <20200313161606.89048105EB8@lists.varnish-cache.org> commit 28fe2026690e7f4da1e75ddbc42c70bbac54ab0c Author: Dridi Boukelmoune Date: Fri Mar 13 16:57:33 2020 +0100 More 6.4 release notes Not sure I like that everything is a bullet point, but well, someone else did the dirty work for this release so I'll shut up and roll with it. diff --git a/doc/sphinx/whats-new/changes-6.4.rst b/doc/sphinx/whats-new/changes-6.4.rst index afc45a4bf..4248847df 100644 --- a/doc/sphinx/whats-new/changes-6.4.rst +++ b/doc/sphinx/whats-new/changes-6.4.rst @@ -42,12 +42,22 @@ the error message:: [...] + varnish> param.show thread_pool_min + 200 + thread_pool_min + Value is: 100 [threads] (default) + Maximum is: 5000 + + [...] + varnish> param.set thread_pool_reserve 100 106 Must be no more than 95 (95% of thread_pool_min) (attempting to set param 'thread_pool_reserve' to '100') +Expect further improvements in future releases. + Parameters ~~~~~~~~~~ @@ -66,23 +76,34 @@ Changes in behavior Changes to VCL ============== -* backend ``none`` was added for "no backend". +* New syntax for "no backend":: + + backend dummy none; + + sub vcl_recv { + set req.backend_hint = dummy; + } It can be used whenever a backend is needed for syntactical reasons. The ``none`` backend will fail any attempt to use it. + The other purpose is to avoid the declaration of a dummy backend + when one is not needed: for example an active VCL only passing + requests to other VCLs with the ``return (vcl(...))`` syntax or + setups relying on dynamic backends from a VMOD. * ``std.rollback(bereq)`` is now safe to use, see :ref:`vmod_std(3)` for details. * Deliberately closing backend requests through ``return(abandon)``, ``return(fail)`` or ``return(error)`` is no longer accounted as a - fetch failure + fetch failure. -* Numerical expressions can now be negative or negated as in ``set - resp.http.o = -std.integer("-200");`` +* Numerical expressions can now be negative or negated as in + ``set resp.http.ok = -std.integer("-200");``. -* Overloaded the ``+=`` operator to also append to headers as in ``set - reqp.http.header += "add this"`` +* The ``+=`` operator is now available for headers and response bodies:: + + set resp.http.header += "string"; VCL variables ~~~~~~~~~~~~~ @@ -102,7 +123,11 @@ VMODs varnishlog ========== -* A ``Notice`` VSL tag has been added +* A ``Notice`` VSL tag has been added. + +* Log records can safely have empty fields or fields containing blanks + if they are delimited by "double quotes". This was applied to + ``SessError`` and ``Backend_health``. varnishadm ========== @@ -113,8 +138,8 @@ varnishadm varnishstat =========== -* Add vi-stype CTRL-f / CTRL-b for page down/up to interactive - varnishstat +* Add vi-style CTRL-f / CTRL-b for page down/up to interactive + ``varnishstat``. * The ``MAIN.sess_drop`` counter is gone. @@ -128,10 +153,10 @@ varnishstat Changes for developers and VMOD authors ======================================= -general +General ~~~~~~~ -* New configure switch: --with-unwind. Alpine linux appears to offer a +* New configure switch: ``--with-unwind``. Alpine linux appears to offer a ``libexecinfo`` implementation that crashes when called by Varnish, this offers the alternative of using ``libunwind`` instead. @@ -141,20 +166,20 @@ general lock cycles from the test logs. * Introduced ``struct reqtop`` to hold information on the ESI top request - and ``PRIV_TOP`` + and ``PRIV_TOP``. * New or improved Coccinelle semantic patches that may be useful for VMOD or utilities authors. * Added ``VSLs()`` and ``VSLbs()`` functions for logging ``STRANDS`` to - VSL + VSL. -* Added ``WS_VSB_new()`` / ``WS_VSB_finish()`` for VSBs on workspaces +* Added ``WS_VSB_new()`` / ``WS_VSB_finish()`` for VSBs on workspaces. * added ``v_dont_optimize`` attribute macro to instruct compilers - (only gcc as of this release) to not optimize a function + (only gcc as of this release) to not optimize a function. -* Added ``VSB_tofile()`` to ``libvarnishapi`` +* Added ``VSB_tofile()`` to ``libvarnishapi``. VMODs ~~~~~ @@ -163,9 +188,9 @@ VMODs of a dynamic backend. A hash is now computed to determine uniqueness and a backend declaration can contribute arbitrary data to influence the pool. -* ``VRB_Iterate()`` signature has changed +* ``VRB_Iterate()`` signature has changed. -* ``VRT_fail()`` now also works from director code +* ``VRT_fail()`` now also works from director code. * ``body_status`` and ``req_body_status`` have been collapsed into one type. In particular, the ``REQ_BODY_*`` enums now have been replaced @@ -174,15 +199,6 @@ VMODs * Added ``VRT_AllocStrandsWS()`` as a utility function to allocate STRANDS on a workspace. -log tools -~~~~~~~~~ - -* Log records can safely have empty fields or fields containing blanks - if they are delimited by "double quotes". This was applied to - ``SessError`` and ``Backend_health``. - - Authors of log handling tools might need to review this change. - *eof* .. _varnish_modules: https://github.com/varnish/varnish-modules diff --git a/doc/sphinx/whats-new/upgrading-6.4.rst b/doc/sphinx/whats-new/upgrading-6.4.rst index b70da8d23..2510044a2 100644 --- a/doc/sphinx/whats-new/upgrading-6.4.rst +++ b/doc/sphinx/whats-new/upgrading-6.4.rst @@ -53,7 +53,7 @@ Statistics on HTTP/1 used to be accounted to ``MAIN.sc_rem_close``. Such timeout events are now accounted towards ``MAIN.sc_tx_error``. -see :ref:`varnish-counters(7)` for details +See :ref:`varnish-counters(7)` for details. vsl/logs -------- @@ -66,6 +66,6 @@ vsl/logs ``vcl_synth {}`` processing time and appears at a different position in the log. -* A ``Notice`` VSL tag has been added +* A ``Notice`` VSL tag has been added. *eof* From dridi.boukelmoune at gmail.com Sun Mar 15 18:09:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sun, 15 Mar 2020 18:09:07 +0000 (UTC) Subject: [master] 230315e0b More semi-dynamic parameter defaults Message-ID: <20200315180907.40A2F9F8C@lists.varnish-cache.org> commit 230315e0b0f6a57859424aafd12d7193c00df16e Author: Dridi Boukelmoune Date: Sun Mar 15 19:06:52 2020 +0100 More semi-dynamic parameter defaults diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index ea4da117a..f0fabec51 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -55,7 +55,8 @@ struct parspec mgt_parspec[] = { "dlopen(3) loadable object. Any occurrence of %s in " "the string will be replaced with the source file name, " "and %o will be replaced with the output file name.", - MUST_RELOAD }, + MUST_RELOAD, + NULL, NULL, "defined when Varnish is built" }, { "vcl_path", tweak_string, &mgt_vcl_path, NULL, NULL, VARNISH_VCL_DIR, NULL, diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 63b3ede40..a574cd17a 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -89,7 +89,7 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par, struct parspec WRK_parspec[] = { { "thread_pools", tweak_uint, &mgt_param.wthread_pools, - "1", NULL, "2", + "1", NULL, /* maximum defined in mgt_param.c */ "2", "pools", "Number of worker thread pools.\n" "\n" @@ -104,7 +104,8 @@ struct parspec WRK_parspec[] = { "\n" "Can be increased on the fly, but decreases require a " "restart to take effect.", - EXPERIMENTAL | DELAYED_EFFECT }, + EXPERIMENTAL | DELAYED_EFFECT, + NULL, "defined when Varnish is built" }, { "thread_pool_max", tweak_thread_pool_max, &mgt_param.wthread_max, NULL, NULL, "5000", "threads", From fgsch at lodoss.net Sun Mar 15 23:25:10 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 15 Mar 2020 23:25:10 +0000 (UTC) Subject: [master] d47b8d4e4 Let the san build get a bit further Message-ID: <20200315232510.C8ABE103195@lists.varnish-cache.org> commit d47b8d4e4d419ebd70369bfac9bdb1c4412744b9 Author: Federico G. Schwindt Date: Sun Mar 15 23:24:00 2020 +0000 Let the san build get a bit further diff --git a/tools/lsan.suppr b/tools/lsan.suppr index 4e9892357..cc58dd207 100644 --- a/tools/lsan.suppr +++ b/tools/lsan.suppr @@ -10,6 +10,7 @@ leak:VSL_Setup leak:WRK_BgThread # leak:binheap_new +leak:binheap_insert # ev leak:mct_callback # From fgsch at lodoss.net Mon Mar 16 11:29:07 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 16 Mar 2020 11:29:07 +0000 (UTC) Subject: [master] 416e3bf6e Ignore leaks in the binheap test for now Message-ID: <20200316112907.5D001113E9D@lists.varnish-cache.org> commit 416e3bf6ea15ff9cca48abc20cf057189bbf0f94 Author: Federico G. Schwindt Date: Mon Mar 16 11:27:32 2020 +0000 Ignore leaks in the binheap test for now diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c index 0079efe1d..f77b63d9c 100644 --- a/lib/libvarnish/binary_heap.c +++ b/lib/libvarnish/binary_heap.c @@ -46,6 +46,10 @@ #include "vas.h" #include "binary_heap.h" +#if !defined(__has_feature) +#define __has_feature(x) 0 +#endif + /* Parameters --------------------------------------------------------*/ /* @@ -526,6 +530,14 @@ vrnd_lock(void) { } +#if defined(__SANITIZER) || __has_feature(address_sanitizer) +int __lsan_is_turned_off(void); +int __lsan_is_turned_off(void) +{ + return (1); +} + +#endif int main(void) { diff --git a/tools/lsan.suppr b/tools/lsan.suppr index cc58dd207..4e9892357 100644 --- a/tools/lsan.suppr +++ b/tools/lsan.suppr @@ -10,7 +10,6 @@ leak:VSL_Setup leak:WRK_BgThread # leak:binheap_new -leak:binheap_insert # ev leak:mct_callback # From nils.goroll at uplex.de Mon Mar 16 11:52:45 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 16 Mar 2020 12:52:45 +0100 Subject: [master] 416e3bf6e Ignore leaks in the binheap test for now In-Reply-To: <20200316112907.5D001113E9D@lists.varnish-cache.org> References: <20200316112907.5D001113E9D@lists.varnish-cache.org> Message-ID: Hi, how is that better than suppressing in lsan.suppr? On 16/03/2020 12:29, Federico G. Schwindt wrote: > > commit 416e3bf6ea15ff9cca48abc20cf057189bbf0f94 > Author: Federico G. Schwindt > Date: Mon Mar 16 11:27:32 2020 +0000 > > Ignore leaks in the binheap test for now > > diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c > index 0079efe1d..f77b63d9c 100644 > --- a/lib/libvarnish/binary_heap.c > +++ b/lib/libvarnish/binary_heap.c > @@ -46,6 +46,10 @@ > #include "vas.h" > #include "binary_heap.h" > > +#if !defined(__has_feature) > +#define __has_feature(x) 0 > +#endif > + > /* Parameters --------------------------------------------------------*/ > > /* > @@ -526,6 +530,14 @@ vrnd_lock(void) > { > } > > +#if defined(__SANITIZER) || __has_feature(address_sanitizer) > +int __lsan_is_turned_off(void); > +int __lsan_is_turned_off(void) > +{ > + return (1); > +} > + > +#endif > int > main(void) > { > diff --git a/tools/lsan.suppr b/tools/lsan.suppr > index cc58dd207..4e9892357 100644 > --- a/tools/lsan.suppr > +++ b/tools/lsan.suppr > @@ -10,7 +10,6 @@ leak:VSL_Setup > leak:WRK_BgThread > # > leak:binheap_new > -leak:binheap_insert > # ev > leak:mct_callback > # > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From hermunn at varnish-software.com Mon Mar 16 12:46:03 2020 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Mon, 16 Mar 2020 12:46:03 +0000 (UTC) Subject: [master] 13f137934 Prepare for 6.4.0 Message-ID: <20200316124603.5D1D0115AE0@lists.varnish-cache.org> commit 13f137934ec1cf14af66baf7896311115ee35598 Author: P?l Hermunn Johansen Date: Mon Mar 16 12:53:35 2020 +0100 Prepare for 6.4.0 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 3829b6292..aee7140c6 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -141,7 +141,7 @@ Resp_Setup_Deliver(struct req *req) http_PrintfHeader(h, "Age: %.0f", floor(fmax(0., req->t_prev - oc->t_origin))); - http_SetHeader(h, "Via: 1.1 varnish (Varnish/6.3)"); + http_SetHeader(h, "Via: 1.1 varnish (Varnish/6.4)"); if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, oc, OF_GZIPED) && diff --git a/configure.ac b/configure.ac index 87fe4b11f..dd43dc82f 100644 --- a/configure.ac +++ b/configure.ac @@ -1,8 +1,8 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS -Copyright (c) 2006-2019 Varnish Software]) +Copyright (c) 2006-2020 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [trunk], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [6.4.0], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index ebeb7de4a..dc1ceac2a 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -50,8 +50,6 @@ Varnish Cache 6.4.0 (2019-09-16) director, which, amongst other advantages, offers more stable backend selection through consistent hashing. -.. do we want to mention VSV00004 ? - * Log records can safely have empty fields or fields containing blanks if they are delimited by "double quotes". This was applied to ``SessError`` and ``Backend_health``. @@ -60,9 +58,6 @@ Varnish Cache 6.4.0 (2019-09-16) of a dynamic backend. A hash is now computed to determine uniqueness and a backend declaration can contribute arbitrary data to influence the pool. -.. 3109 (vdp gunzip) is probably irrelevant for users because it only - happens with gzip objects inserted bypassing the built-in vfps - * The option ``varnishtest -W`` is gone, the same can be achieved with ``varnishtest -p debug=+witness``. A ``witness.sh`` script is available in the source tree to generate a graphviz dot file and detect potential @@ -130,8 +125,6 @@ Varnish Cache 6.4.0 (2019-09-16) * Always refer to ``sub`` as subroutine in the documentation and error messages to avoid confusion with other terms. -.. mention #3176? (backend cooling straightened out) - * New ``pid`` command in the Varnish CLI, to get the master and optionally cache process PIDs, for example from ``varnishadm``. diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index e86d038fe..e5c33b1e4 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -37,7 +37,7 @@ Longer listings like example command output and VCL look like this:: $ /opt/varnish/sbin/varnishd -V varnishd (varnish-trunk revision 199de9b) Copyright (c) 2006 Verdens Gang AS - Copyright (c) 2006-2019 Varnish Software AS + Copyright (c) 2006-2020 Varnish Software AS .. For maintainers: diff --git a/include/vrt.h b/include/vrt.h index 2baea19a8..6d5dcdc93 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -53,7 +53,7 @@ * Whenever something is deleted or changed in a way which is not * binary/load-time compatible, increment MAJOR version * - * unreleased (planned for 2020-03-15) + * 11.0 (2020-03-16) * Changed type of vsa_suckaddr_len from int to size_t * New prefix_{ptr|len} fields in vrt_backend * VRT_HashStrands32() added @@ -159,7 +159,7 @@ * vrt_acl type added */ -#define VRT_MAJOR_VERSION 10U +#define VRT_MAJOR_VERSION 11U #define VRT_MINOR_VERSION 0U diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index d2e344f7e..f54e78407 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -74,7 +74,7 @@ VCS_String(const char *which) ")" "\n" "Copyright (c) 2006 Verdens Gang AS\n" - "Copyright (c) 2006-2019 Varnish Software AS\n" + "Copyright (c) 2006-2020 Varnish Software AS\n" ); default: WRONG("Wrong argument to VCS_String"); From guillaume at varnish-software.com Mon Mar 16 17:14:06 2020 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 16 Mar 2020 17:14:06 +0000 (UTC) Subject: [6.4] 09c4d1fd6 [cci] add package revision Message-ID: <20200316171406.A50DD11B2B9@lists.varnish-cache.org> commit 09c4d1fd6480d8d57d9703b62483ce36a944227c Author: Guillaume Quintard Date: Mon Mar 16 10:12:50 2020 -0700 [cci] add package revision diff --git a/.circleci/config.yml b/.circleci/config.yml index 44fee78dc..62b2155ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,7 +277,7 @@ jobs: WEEKLY= fi VERSION=$(./configure --version | awk 'NR == 1 {print $NF}')$WEEKLY~<< parameters.release >> - sed -i -e "s|@VERSION@|$VERSION|" "debian/changelog" + sed -i -e "s|@VERSION@|$VERSION-1|" "debian/changelog" - run: name: Install Build-Depends packages command: | From guillaume at varnish-software.com Mon Mar 16 17:22:02 2020 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 16 Mar 2020 17:22:02 +0000 (UTC) Subject: [master] 09c4d1fd6 [cci] add package revision Message-ID: <20200316172202.AF5CF11B676@lists.varnish-cache.org> commit 09c4d1fd6480d8d57d9703b62483ce36a944227c Author: Guillaume Quintard Date: Mon Mar 16 10:12:50 2020 -0700 [cci] add package revision diff --git a/.circleci/config.yml b/.circleci/config.yml index 44fee78dc..62b2155ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,7 +277,7 @@ jobs: WEEKLY= fi VERSION=$(./configure --version | awk 'NR == 1 {print $NF}')$WEEKLY~<< parameters.release >> - sed -i -e "s|@VERSION@|$VERSION|" "debian/changelog" + sed -i -e "s|@VERSION@|$VERSION-1|" "debian/changelog" - run: name: Install Build-Depends packages command: | From guillaume at varnish-software.com Mon Mar 16 17:54:06 2020 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 16 Mar 2020 17:54:06 +0000 (UTC) Subject: [6.4] ff719b22b [cci] fix rpm revision too Message-ID: <20200316175406.B6C9911C205@lists.varnish-cache.org> commit ff719b22bbeed635a1a848b4b58d2c23c4790209 Author: Guillaume Quintard Date: Mon Mar 16 10:53:10 2020 -0700 [cci] fix rpm revision too diff --git a/.circleci/config.yml b/.circleci/config.yml index cba211128..7d4462e6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -367,7 +367,7 @@ jobs: --define "_srcrpmdir $CUR_DIR/${RESULT_DIR}" \ --define "_rpmdir $CUR_DIR/${RESULT_DIR}" \ --define "versiontag ${RPMVERSION}" \ - --define "releasetag 0.0" \ + --define "releasetag 1" \ --define "srcname $DIST_DIR" \ --define "nocheck 1" \ "$@" From dridi.boukelmoune at gmail.com Tue Mar 17 09:20:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 17 Mar 2020 09:20:07 +0000 (UTC) Subject: [master] 890170c8b Homework for later Message-ID: <20200317092007.7F11F10E76C@lists.varnish-cache.org> commit 890170c8b06221c7e911b0987e67f98ce198ad61 Author: Dridi Boukelmoune Date: Tue Mar 17 10:18:34 2020 +0100 Homework for later Refs #3251 diff --git a/include/tbl/params.h b/include/tbl/params.h index 6773d9ae5..bfbeb5ac9 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -543,6 +543,7 @@ PARAM( /* units */ "bool", /* descr */ "Enable support for HTTP Range headers." + /* XXX: what about the effect on beresp.filters? */ ) PARAM( From fgsch at lodoss.net Tue Mar 17 13:37:06 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 17 Mar 2020 13:37:06 +0000 (UTC) Subject: [master] eed885100 Add a hint for the UI Message-ID: <20200317133706.AD957113D4B@lists.varnish-cache.org> commit eed885100678c4a8810c2cab2bcf1d0aa9b10d88 Author: Federico G. Schwindt Date: Tue Mar 17 13:35:40 2020 +0000 Add a hint for the UI diff --git a/.travis.yml b/.travis.yml index ad0b036f1..7713f3331 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,7 @@ jobs: - ./autogen.sh - ./configure - <<: *test-linux + env: WITNESS=1 script: make -j8 witness - <<: *test-linux stage: sanitizers From fgsch at lodoss.net Tue Mar 17 14:09:07 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 17 Mar 2020 14:09:07 +0000 (UTC) Subject: [master] 4d8c3586a Limit notifications to the main repo Message-ID: <20200317140907.4F27A114930@lists.varnish-cache.org> commit 4d8c3586ae288964f4e97e75d1edd2add6ccfedc Author: Federico G. Schwindt Date: Tue Mar 17 14:07:40 2020 +0000 Limit notifications to the main repo diff --git a/.travis.yml b/.travis.yml index 7713f3331..258ac0c4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -122,7 +122,8 @@ stages: notifications: irc: - if: branch = master + if: branch = master AND + repo = varnishcache/varnish-cache channels: - "irc.linpro.no#varnish-hacking" on_success: change From fgsch at lodoss.net Tue Mar 17 14:26:06 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 17 Mar 2020 14:26:06 +0000 (UTC) Subject: [master] 12df4f60a Use bionic where possible Message-ID: <20200317142606.A07A11150D0@lists.varnish-cache.org> commit 12df4f60af2ce3cebd26b0075d98908a21109785 Author: Federico G. Schwindt Date: Tue Mar 17 14:24:55 2020 +0000 Use bionic where possible diff --git a/.travis.yml b/.travis.yml index 258ac0c4b..bcce43f23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ jobs: - &test-linux stage: test os: linux - dist: xenial + dist: bionic arch: amd64 compiler: clang addons: @@ -50,6 +50,7 @@ jobs: script: make -j8 witness - <<: *test-linux stage: sanitizers + dist: xenial addons: apt: sources: From dridi.boukelmoune at gmail.com Fri Mar 20 15:21:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 20 Mar 2020 15:21:07 +0000 (UTC) Subject: [master] 60f5a0c09 Mention drop_pools for brave users Message-ID: <20200320152107.A601010FE2D@lists.varnish-cache.org> commit 60f5a0c09dcb1dba0ad92ca136f513fa4021462d Author: Dridi Boukelmoune Date: Fri Mar 20 16:18:23 2020 +0100 Mention drop_pools for brave users diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index a574cd17a..a2e73245f 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -103,7 +103,8 @@ struct parspec WRK_parspec[] = { "pool for each CPU is most likely detrimental to performance.\n" "\n" "Can be increased on the fly, but decreases require a " - "restart to take effect.", + "restart to take effect, unless the drop_pools experimental " + "debug flag is set.", EXPERIMENTAL | DELAYED_EFFECT, NULL, "defined when Varnish is built" }, { "thread_pool_max", tweak_thread_pool_max, &mgt_param.wthread_max, From nils.goroll at uplex.de Sun Mar 22 15:44:09 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 22 Mar 2020 15:44:09 +0000 (UTC) Subject: [master] 4f9d8bed6 remove code duplication for http header filtering Message-ID: <20200322154409.6AF7E104F87@lists.varnish-cache.org> commit 4f9d8bed6b24bf9ee900c754f37615fdba1c44db Author: Nils Goroll Date: Sun Mar 22 16:32:33 2020 +0100 remove code duplication for http header filtering no semantic changes, just consolidating three places in one (thus pushed during freeze) diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 91069f103..cf949b765 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -884,6 +884,23 @@ http_PutResponse(struct http *to, const char *proto, uint16_t status, http_SetStatus(to, status, reason); } +/*-------------------------------------------------------------------- + * check if header is filterd by the dynamic marker or the static + * definitions in http_headers.h + */ + +static inline int +http_isfiltered(const struct http *fm, unsigned u, unsigned how) +{ + if (fm->hdf[u] & HDF_FILTER) + return (1); +#define HTTPH(a, b, c) \ + if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \ + return (1); +#include "tbl/http_headers.h" + return (0); +} + /*-------------------------------------------------------------------- * Estimate how much workspace we need to Filter this header according * to 'how'. @@ -900,12 +917,8 @@ http_EstimateWS(const struct http *fm, unsigned how) if (u == HTTP_HDR_METHOD || u == HTTP_HDR_URL) continue; Tcheck(fm->hd[u]); - if (fm->hdf[u] & HDF_FILTER) + if (http_isfiltered(fm, u, how)) continue; -#define HTTPH(a, b, c) \ - if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \ - continue; -#include "tbl/http_headers.h" l += Tlen(fm->hd[u]) + 1L; } return (PRNDUP(l + 1L)); @@ -941,12 +954,8 @@ HTTP_Encode(const struct http *fm, uint8_t *p0, unsigned l, unsigned how) if (u == HTTP_HDR_METHOD || u == HTTP_HDR_URL) continue; Tcheck(fm->hd[u]); - if (fm->hdf[u] & HDF_FILTER) - continue; -#define HTTPH(a, b, c) \ - if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \ + if (http_isfiltered(fm, u, how)) continue; -#include "tbl/http_headers.h" http_VSLH(fm, u); w = Tlen(fm->hd[u]) + 1L; assert(p + w + 1 <= e); @@ -1133,13 +1142,8 @@ http_filterfields(struct http *to, const struct http *fm, unsigned how) to->status = fm->status; for (u = HTTP_HDR_FIRST; u < fm->nhd; u++) { Tcheck(fm->hd[u]); - if (fm->hdf[u] & HDF_FILTER) + if (http_isfiltered(fm, u, how)) continue; - Tcheck(fm->hd[u]); -#define HTTPH(a, b, c) \ - if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \ - continue; -#include "tbl/http_headers.h" assert (to->nhd < to->shd); to->hd[to->nhd] = fm->hd[u]; to->hdf[to->nhd] = 0; From fgsch at lodoss.net Mon Mar 30 23:13:12 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 30 Mar 2020 23:13:12 +0000 (UTC) Subject: [master] b50bd74ac Use the latest url for downloading coverity Message-ID: <20200330231312.46153113670@lists.varnish-cache.org> commit b50bd74acca599c7ff70f2adb49265858e33acd0 Author: Federico G. Schwindt Date: Tue Mar 31 00:10:51 2020 +0100 Use the latest url for downloading coverity diff --git a/.travis.yml b/.travis.yml index bcce43f23..a68713459 100644 --- a/.travis.yml +++ b/.travis.yml @@ -96,7 +96,7 @@ jobs: - curl --data "token=$COVTOKEN&project=varnish" --insecure - -o coverity_tool.tgz https://scan.coverity.com/download/linux64 + -o coverity_tool.tgz https://scan.coverity.com/download/cxx/linux64 - tar xfz coverity_tool.tgz - export PATH=$PATH:$(echo $(pwd)/cov-analysis-*/bin) script: From nils.goroll at uplex.de Tue Mar 31 17:00:50 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 31 Mar 2020 19:00:50 +0200 Subject: [master] 416e3bf6e Ignore leaks in the binheap test for now In-Reply-To: References: <20200316112907.5D001113E9D@lists.varnish-cache.org> Message-ID: ping fgs? On 16/03/2020 12:52, Nils Goroll wrote: > Hi, > > how is that better than suppressing in lsan.suppr? > > On 16/03/2020 12:29, Federico G. Schwindt wrote: >> >> commit 416e3bf6ea15ff9cca48abc20cf057189bbf0f94 >> Author: Federico G. Schwindt >> Date: Mon Mar 16 11:27:32 2020 +0000 >> >> Ignore leaks in the binheap test for now >> >> diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c >> index 0079efe1d..f77b63d9c 100644 >> --- a/lib/libvarnish/binary_heap.c >> +++ b/lib/libvarnish/binary_heap.c >> @@ -46,6 +46,10 @@ >> #include "vas.h" >> #include "binary_heap.h" >> >> +#if !defined(__has_feature) >> +#define __has_feature(x) 0 >> +#endif >> + >> /* Parameters --------------------------------------------------------*/ >> >> /* >> @@ -526,6 +530,14 @@ vrnd_lock(void) >> { >> } >> >> +#if defined(__SANITIZER) || __has_feature(address_sanitizer) >> +int __lsan_is_turned_off(void); >> +int __lsan_is_turned_off(void) >> +{ >> + return (1); >> +} >> + >> +#endif >> int >> main(void) >> { >> diff --git a/tools/lsan.suppr b/tools/lsan.suppr >> index cc58dd207..4e9892357 100644 >> --- a/tools/lsan.suppr >> +++ b/tools/lsan.suppr >> @@ -10,7 +10,6 @@ leak:VSL_Setup >> leak:WRK_BgThread >> # >> leak:binheap_new >> -leak:binheap_insert >> # ev >> leak:mct_callback >> # >> _______________________________________________ >> varnish-commit mailing list >> varnish-commit at varnish-cache.org >> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit >> > > -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: