[master] a32fb96 First babystep of the new selfdocumenting VSC counters.
Poul-Henning Kamp
phk at FreeBSD.org
Tue May 23 10:34:05 CEST 2017
commit a32fb96c1475b342b96de7083e35419582ef5d57
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date: Tue May 23 08:30:36 2017 +0000
First babystep of the new selfdocumenting VSC counters.
The .VSC file steals .RST's directive syntax, but it is not a RST
file, and it is not processed as such. Maybe later it will become
a .RST file, but that is TBD.
A C-struct for use in varnishd is built, (along with a JSON structure
which describes the counters.)
For now that structure is identical with what comes out of
include/tbl/vsc_f_main, so nothing special happens in the VSM yet.
diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index f548ab6..c03ff70 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -107,12 +107,14 @@ varnishd_SOURCES = \
waiter/cache_waiter_epoll.c \
waiter/cache_waiter_kqueue.c \
waiter/cache_waiter_poll.c \
- waiter/cache_waiter_ports.c
+ waiter/cache_waiter_ports.c \
+ VSC_main.c
nodist_varnishd_SOURCES = \
builtin_vcl.c
noinst_HEADERS = \
+ VSC_main.h \
cache/cache_ban.h \
cache/cache_esi.h \
cache/cache_obj.h \
@@ -215,3 +217,12 @@ DISTCLEANFILES = builtin_vcl.c
BUILT_SOURCES = vhp_hufdec.h
DISTCLEANFILES += vhp_hufdec.h
+#######################################################################
+VSC_main.c VSC_main.h: $(srcdir)/main.vsc $(top_builddir)/lib/libvcc/vsctool.py
+ $(PYTHON) $(top_builddir)/lib/libvcc/vsctool.py $(srcdir)/main.vsc
+
+$(varnishd_OBJECTS): VSC_main.h
+
+EXTRA_DIST += main.vsc
+DISTCLEANFILES += VSC_main.c VSC_main.h
+
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b772822..a5fcef8 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -977,7 +977,6 @@ void SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src);
const char *SES_Get_String_Attr(const struct sess *sp, enum sess_attr a);
/* cache_shmlog.c */
-extern struct VSC_C_main *VSC_C_main;
void *VSM_Alloc(unsigned size, const char *class, const char *type,
const char *ident);
void VSM_Free(void *ptr);
diff --git a/bin/varnishd/cache/cache_priv.h b/bin/varnishd/cache/cache_priv.h
index 0ff51eb..215ebc9 100644
--- a/bin/varnishd/cache/cache_priv.h
+++ b/bin/varnishd/cache/cache_priv.h
@@ -29,6 +29,8 @@
* Stuff that should *never* be exposed to a VMOD
*/
+#include "VSC_main.h"
+
/* Prototypes etc ----------------------------------------------------*/
/* cache_acceptor.c */
@@ -103,6 +105,7 @@ void SES_NewPool(struct pool *, unsigned pool_no);
void SES_DestroyPool(struct pool *);
/* cache_shmlog.c */
+extern struct VSC_main *VSC_C_main;
void VSM_Init(void);
void VSL_Setup(struct vsl_log *vsl, void *ptr, size_t len);
void VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why,
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 66fac59..113d0fc 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -51,8 +51,7 @@ static uint32_t *vsl_ptr;
static unsigned vsl_segment_n;
static ssize_t vsl_segsize;
-struct VSC_C_main *VSC_C_main;
-
+struct VSC_main *VSC_C_main;
static void
vsl_sanity(const struct vsl_log *vsl)
@@ -471,6 +470,28 @@ VSL_End(struct vsl_log *vsl)
/*--------------------------------------------------------------------*/
+void *
+VSC_Alloc(const char *nm, size_t sj, const unsigned char *zj, size_t szj,
+ const char *fmt, va_list va)
+{
+ (void)nm;
+ (void)zj;
+ (void)szj;
+ (void)sj;
+ (void)fmt;
+ (void)va;
+ return (0);
+}
+
+void
+VSC_Destroy(const char *nm, void *p)
+{
+ (void)nm;
+ (void)p;
+}
+
+/*--------------------------------------------------------------------*/
+
static void *
vsm_cleaner(void *priv)
{
@@ -478,7 +499,7 @@ vsm_cleaner(void *priv)
THR_SetName("vsm_cleaner");
while (1) {
AZ(pthread_mutex_lock(&vsm_mtx));
- VSM_common_cleaner(heritage.vsm, VSC_C_main);
+ VSM_common_cleaner(heritage.vsm, (void*)VSC_C_main);
AZ(pthread_mutex_unlock(&vsm_mtx));
VTIM_sleep(1.1);
}
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 0c302c2..107f5b8 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -33,6 +33,7 @@
#endif
#define COMMON_COMMON_H
+#include <stdarg.h>
#include <stdint.h>
#include <sys/types.h>
@@ -90,5 +91,9 @@ void VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from);
void VSM_common_cleaner(struct vsm_sc *sc, struct VSC_C_main *stats);
void VSM_common_ageupdate(const struct vsm_sc *sc);
+void *VSC_Alloc(const char *, size_t, const unsigned char *, size_t,
+ const char *, va_list);
+void VSC_Destroy(const char *, void *);
+
/* mgt_cli.c */
extern struct VCLS *mgt_cls;
diff --git a/bin/varnishd/main.vsc b/bin/varnishd/main.vsc
new file mode 100644
index 0000000..f901fca
--- /dev/null
+++ b/bin/varnishd/main.vsc
@@ -0,0 +1,756 @@
+..
+ This is *NOT* a RST file but the syntax has been chosen so
+ that it may become an RST file at some later date.
+
+ XXX: For now this file must be kept in 100% agreement with
+ XXX: include/tbl/vsc_f_main.h
+
+.. varnish_vsc_begin:: main
+ :oneliner: Main counters
+
+.. varnish_vsc:: uptime
+ :oneliner: Child process uptime
+
+How long the child process has been running.
+
+.. varnish_vsc:: sess_conn
+ :oneliner: Sessions accepted
+
+Count of sessions successfully accepted
+
+.. varnish_vsc:: sess_drop
+ :oneliner: Sessions dropped
+
+Count of sessions silently dropped due to lack of worker thread.
+
+.. varnish_vsc:: sess_fail
+ :oneliner: Session accept failures
+
+Count of failures to accept TCP connection. Either the client
+changed its mind, or the kernel ran out of some resource like file
+descriptors.
+
+.. varnish_vsc:: client_req_400
+ :oneliner: Client requests received, subject to 400 errors
+
+400 means we couldn't make sense of the request, it was malformed
+in some drastic way.
+
+.. varnish_vsc:: client_req_417
+ :oneliner: Client requests received, subject to 417 errors
+
+417 means that something went wrong with an Expect: header.
+
+.. varnish_vsc:: client_req
+ :oneliner: Good client requests received
+
+The count of parseable client requests seen.
+
+.. varnish_vsc:: cache_hit
+ :oneliner: Cache hits
+
+Count of cache hits. A cache hit indicates that an object has been
+delivered to a client without fetching it from a backend server.
+
+.. varnish_vsc:: cache_hitpass
+ :oneliner: Cache hits for pass.
+
+Count of hits for pass. A cache hit for pass indicates that Varnish
+is going to pass the request to the backend and this decision has
+been cached in it self. This counts how many times the cached
+decision is being used.
+
+.. varnish_vsc:: cache_hitmiss
+ :oneliner: Cache hits for miss.
+
+Count of hits for miss. A cache hit for miss indicates that Varnish
+is going to proceed as for a cache miss without request coalescing,
+and this decision has been cached. This counts how many times the
+cached decision is being used.
+
+.. varnish_vsc:: cache_miss
+ :oneliner: Cache misses
+
+Count of misses. A cache miss indicates the object was fetched from
+the backend before delivering it to the client.
+
+.. varnish_vsc:: backend_conn
+ :oneliner: Backend conn. success
+
+How many backend connections have successfully been established.
+
+.. varnish_vsc:: backend_unhealthy
+ :oneliner: Backend conn. not attempted
+
+
+.. varnish_vsc:: backend_busy
+ :oneliner: Backend conn. too many
+
+
+.. varnish_vsc:: backend_fail
+ :oneliner: Backend conn. failures
+
+
+.. varnish_vsc:: backend_reuse
+ :oneliner: Backend conn. reuses
+
+Count of backend connection reuses. This counter is increased
+whenever we reuse a recycled connection.
+
+.. varnish_vsc:: backend_recycle
+ :oneliner: Backend conn. recycles
+
+Count of backend connection recycles. This counter is increased
+whenever we have a keep-alive connection that is put back into the
+pool of connections. It has not yet been used, but it might be,
+unless the backend closes it.
+
+.. varnish_vsc:: backend_retry
+ :oneliner: Backend conn. retry
+
+
+.. varnish_vsc:: fetch_head
+ :oneliner: Fetch no body (HEAD)
+
+beresp with no body because the request is HEAD.
+
+.. varnish_vsc:: fetch_length
+ :oneliner: Fetch with Length
+
+beresp.body with Content-Length.
+
+.. varnish_vsc:: fetch_chunked
+ :oneliner: Fetch chunked
+
+beresp.body with Chunked.
+
+.. varnish_vsc:: fetch_eof
+ :oneliner: Fetch EOF
+
+beresp.body with EOF.
+
+.. varnish_vsc:: fetch_bad
+ :oneliner: Fetch bad T-E
+
+beresp.body length/fetch could not be determined.
+
+.. varnish_vsc:: fetch_none
+ :oneliner: Fetch no body
+
+beresp.body empty
+
+.. varnish_vsc:: fetch_1xx
+ :oneliner: Fetch no body (1xx)
+
+beresp with no body because of 1XX response.
+
+.. varnish_vsc:: fetch_204
+ :oneliner: Fetch no body (204)
+
+beresp with no body because of 204 response.
+
+.. varnish_vsc:: fetch_304
+ :oneliner: Fetch no body (304)
+
+beresp with no body because of 304 response.
+
+.. varnish_vsc:: fetch_failed
+ :oneliner: Fetch failed (all causes)
+
+beresp fetch failed.
+
+.. varnish_vsc:: fetch_no_thread
+ :oneliner: Fetch failed (no thread)
+
+beresp fetch failed, no thread available.
+
+.. varnish_vsc:: pools
+ :type: gauge
+ :oneliner: Number of thread pools
+
+Number of thread pools. See also parameter thread_pools. NB: Presently
+pools cannot be removed once created.
+
+.. varnish_vsc:: threads
+ :type: gauge
+ :oneliner: Total number of threads
+
+Number of threads in all pools. See also parameters thread_pools,
+thread_pool_min and thread_pool_max.
+
+.. varnish_vsc:: threads_limited
+ :oneliner: Threads hit max
+
+Number of times more threads were needed, but limit was reached in
+a thread pool. See also parameter thread_pool_max.
+
+.. varnish_vsc:: threads_created
+ :oneliner: Threads created
+
+Total number of threads created in all pools.
+
+.. varnish_vsc:: threads_destroyed
+ :oneliner: Threads destroyed
+
+Total number of threads destroyed in all pools.
+
+.. varnish_vsc:: threads_failed
+ :oneliner: Thread creation failed
+
+Number of times creating a thread failed. See VSL::Debug for
+diagnostics. See also parameter thread_fail_delay.
+
+.. varnish_vsc:: thread_queue_len
+ :type: gauge
+ :oneliner: Length of session queue
+
+Length of session queue waiting for threads. NB: Only updates once
+per second. See also parameter thread_queue_limit.
+
+.. varnish_vsc:: busy_sleep
+ :oneliner: Number of requests sent to sleep on busy objhdr
+
+Number of requests sent to sleep without a worker thread because
+they found a busy object.
+
+.. varnish_vsc:: busy_wakeup
+ :oneliner: Number of requests woken after sleep on busy objhdr
+
+Number of requests taken off the busy object sleep list and rescheduled.
+
+.. varnish_vsc:: busy_killed
+ :oneliner: Number of requests killed after sleep on busy objhdr
+
+Number of requests killed from the busy object sleep list due to
+lack of resources.
+
+.. varnish_vsc:: sess_queued
+ :oneliner: Sessions queued for thread
+
+Number of times session was queued waiting for a thread. See also
+parameter thread_queue_limit.
+
+.. varnish_vsc:: sess_dropped
+ :oneliner: Sessions dropped for thread
+
+Number of times session was dropped because the queue were too long
+already. See also parameter thread_queue_limit.
+
+.. varnish_vsc:: n_object
+ :type: gauge
+ :oneliner: object structs made
+
+Approximate number of HTTP objects (headers + body, if present) in
+the cache.
+
+.. varnish_vsc:: n_vampireobject
+ :type: gauge
+ :level: diag
+ :oneliner: unresurrected objects
+
+Number of unresurrected objects
+
+.. varnish_vsc:: n_objectcore
+ :type: gauge
+ :oneliner: objectcore structs made
+
+Approximate number of object metadata elements in the cache. Each
+object needs an objectcore, extra objectcores are for hit-for-miss,
+hit-for-pass and busy objects.
+
+.. varnish_vsc:: n_objecthead
+ :type: gauge
+ :oneliner: objecthead structs made
+
+Approximate number of different hash entries in the cache.
+
+.. varnish_vsc:: n_backend
+ :type: gauge
+ :oneliner: Number of backends
+
+Number of backends known to us.
+
+.. varnish_vsc:: n_expired
+ :type: gauge
+ :oneliner: Number of expired objects
+
+Number of objects that expired from cache because of old age.
+
+.. varnish_vsc:: n_lru_nuked
+ :type: gauge
+ :oneliner: Number of LRU nuked objects
+
+How many objects have been forcefully evicted from storage to make
+room for a new object.
+
+.. varnish_vsc:: n_lru_moved
+ :type: gauge
+ :level: diag
+ :oneliner: Number of LRU moved objects
+
+Number of move operations done on the LRU list.
+
+.. varnish_vsc:: losthdr
+ :oneliner: HTTP header overflows
+
+
+.. varnish_vsc:: s_sess
+ :oneliner: Total sessions seen
+
+
+.. varnish_vsc:: s_req
+ :oneliner: Total requests seen
+
+
+.. varnish_vsc:: s_pipe
+ :oneliner: Total pipe sessions seen
+
+
+.. varnish_vsc:: s_pass
+ :oneliner: Total pass-ed requests seen
+
+
+.. varnish_vsc:: s_fetch
+ :oneliner: Total backend fetches initiated
+
+
+.. varnish_vsc:: s_synth
+ :oneliner: Total synthethic responses made
+
+
+.. varnish_vsc:: s_req_hdrbytes
+ :oneliner: Request header bytes
+
+Total request header bytes received
+
+.. varnish_vsc:: s_req_bodybytes
+ :oneliner: Request body bytes
+
+Total request body bytes received
+
+.. varnish_vsc:: s_resp_hdrbytes
+ :oneliner: Response header bytes
+
+Total response header bytes transmitted
+
+.. varnish_vsc:: s_resp_bodybytes
+ :oneliner: Response body bytes
+
+Total response body bytes transmitted
+
+.. varnish_vsc:: s_pipe_hdrbytes
+ :oneliner: Pipe request header bytes
+
+Total request bytes received for piped sessions
+
+.. varnish_vsc:: s_pipe_in
+ :oneliner: Piped bytes from client
+
+Total number of bytes forwarded from clients in pipe sessions
+
+.. varnish_vsc:: s_pipe_out
+ :oneliner: Piped bytes to client
+
+Total number of bytes forwarded to clients in pipe sessions
+
+.. varnish_vsc:: sess_closed
+ :oneliner: Session Closed
+
+
+.. varnish_vsc:: sess_closed_err
+ :oneliner: Session Closed with error
+
+Total number of sessions closed with errors. See sc_* diag counters
+for detailed breakdown
+
+.. varnish_vsc:: sess_readahead
+ :oneliner: Session Read Ahead
+
+
+.. varnish_vsc:: sess_herd
+ :level: diag
+ :oneliner: Session herd
+
+Number of times the timeout_linger triggered
+
+.. varnish_vsc:: sc_rem_close
+ :level: diag
+ :oneliner: Session OK REM_CLOSE
+
+Number of session closes with REM_CLOSE (Client Closed)
+
+.. varnish_vsc:: sc_req_close
+ :level: diag
+ :oneliner: Session OK REQ_CLOSE
+
+Number of session closes with REQ_CLOSE (Client requested close)
+
+.. varnish_vsc:: sc_req_http10
+ :level: diag
+ :oneliner: Session Err REQ_HTTP10
+
+Number of session closes with Error REQ_HTTP10 (Proto < HTTP/1.1)
+
+.. varnish_vsc:: sc_rx_bad
+ :level: diag
+ :oneliner: Session Err RX_BAD
+
+Number of session closes with Error RX_BAD (Received bad req/resp)
+
+.. varnish_vsc:: sc_rx_body
+ :level: diag
+ :oneliner: Session Err RX_BODY
+
+Number of session closes with Error RX_BODY (Failure receiving req.body)
+
+.. varnish_vsc:: sc_rx_junk
+ :level: diag
+ :oneliner: Session Err RX_JUNK
+
+Number of session closes with Error RX_JUNK (Received junk data)
+
+.. varnish_vsc:: sc_rx_overflow
+ :level: diag
+ :oneliner: Session Err RX_OVERFLOW
+
+Number of session closes with Error RX_OVERFLOW (Received buffer overflow)
+
+.. varnish_vsc:: sc_rx_timeout
+ :level: diag
+ :oneliner: Session Err RX_TIMEOUT
+
+Number of session closes with Error RX_TIMEOUT (Receive timeout)
+
+.. varnish_vsc:: sc_tx_pipe
+ :level: diag
+ :oneliner: Session OK TX_PIPE
+
+Number of session closes with TX_PIPE (Piped transaction)
+
+.. varnish_vsc:: sc_tx_error
+ :level: diag
+ :oneliner: Session Err TX_ERROR
+
+Number of session closes with Error TX_ERROR (Error transaction)
+
+.. varnish_vsc:: sc_tx_eof
+ :level: diag
+ :oneliner: Session OK TX_EOF
+
+Number of session closes with TX_EOF (EOF transmission)
+
+.. varnish_vsc:: sc_resp_close
+ :level: diag
+ :oneliner: Session OK RESP_CLOSE
+
+Number of session closes with RESP_CLOSE (Backend/VCL requested close)
+
+.. varnish_vsc:: sc_overload
+ :level: diag
+ :oneliner: Session Err OVERLOAD
+
+Number of session closes with Error OVERLOAD (Out of some resource)
+
+.. varnish_vsc:: sc_pipe_overflow
+ :level: diag
+ :oneliner: Session Err PIPE_OVERFLOW
+
+Number of session closes with Error PIPE_OVERFLOW (Session pipe overflow)
+
+.. varnish_vsc:: sc_range_short
+ :level: diag
+ :oneliner: Session Err RANGE_SHORT
+
+Number of session closes with Error RANGE_SHORT (Insufficient data for range)
+
+.. varnish_vsc:: sc_req_http20
+ :level: diag
+ :oneliner: Session Err REQ_HTTP20
+
+Number of session closes with Error REQ_HTTP20 (HTTP2 not accepted)
+
+.. varnish_vsc:: sc_vcl_failure
+ :level: diag
+ :oneliner: Session Err VCL_FAILURE
+
+Number of session closes with Error VCL_FAILURE (VCL failure)
+
+.. varnish_vsc:: shm_records
+ :level: diag
+ :oneliner: SHM records
+
+
+.. varnish_vsc:: shm_writes
+ :level: diag
+ :oneliner: SHM writes
+
+
+.. varnish_vsc:: shm_flushes
+ :level: diag
+ :oneliner: SHM flushes due to overflow
+
+
+.. varnish_vsc:: shm_cont
+ :level: diag
+ :oneliner: SHM MTX contention
+
+
+.. varnish_vsc:: shm_cycles
+ :level: diag
+ :oneliner: SHM cycles through buffer
+
+
+.. varnish_vsc:: backend_req
+ :oneliner: Backend requests made
+
+
+.. varnish_vsc:: n_vcl
+ :oneliner: Number of loaded VCLs in total
+
+
+.. varnish_vsc:: n_vcl_avail
+ :level: diag
+ :oneliner: Number of VCLs available
+
+
+.. varnish_vsc:: n_vcl_discard
+ :level: diag
+ :oneliner: Number of discarded VCLs
+
+
+.. varnish_vsc:: vcl_fail
+ :oneliner: VCL failures
+
+Count of failures which prevented VCL from completing.
+
+.. varnish_vsc:: bans
+ :type: gauge
+ :oneliner: Count of bans
+
+Number of all bans in system, including bans superseded by newer
+bans and bans already checked by the ban-lurker.
+
+.. varnish_vsc:: bans_completed
+ :type: gauge
+ :level: diag
+ :oneliner: Number of bans marked 'completed'
+
+Number of bans which are no longer active, either because they got
+checked by the ban-lurker or superseded by newer identical bans.
+
+.. varnish_vsc:: bans_obj
+ :type: gauge
+ :level: diag
+ :oneliner: Number of bans using obj.*
+
+Number of bans which use obj.* variables. These bans can possibly
+be washed by the ban-lurker.
+
+.. varnish_vsc:: bans_req
+ :type: gauge
+ :level: diag
+ :oneliner: Number of bans using req.*
+
+Number of bans which use req.* variables. These bans can not be
+washed by the ban-lurker.
+
+.. varnish_vsc:: bans_added
+ :level: diag
+ :oneliner: Bans added
+
+Counter of bans added to ban list.
+
+.. varnish_vsc:: bans_deleted
+ :level: diag
+ :oneliner: Bans deleted
+
+Counter of bans deleted from ban list.
+
+.. varnish_vsc:: bans_tested
+ :level: diag
+ :oneliner: Bans tested against objects (lookup)
+
+Count of how many bans and objects have been tested against each
+other during hash lookup.
+
+.. varnish_vsc:: bans_obj_killed
+ :level: diag
+ :oneliner: Objects killed by bans (lookup)
+
+Number of objects killed by bans during object lookup.
+
+.. varnish_vsc:: bans_lurker_tested
+ :level: diag
+ :oneliner: Bans tested against objects (lurker)
+
+Count of how many bans and objects have been tested against each
+other by the ban-lurker.
+
+.. varnish_vsc:: bans_tests_tested
+ :level: diag
+ :oneliner: Ban tests tested against objects (lookup)
+
+Count of how many tests and objects have been tested against each
+other during lookup. 'ban req.url == foo && req.http.host == bar'
+counts as one in 'bans_tested' and as two in 'bans_tests_tested'
+
+.. varnish_vsc:: bans_lurker_tests_tested
+ :level: diag
+ :oneliner: Ban tests tested against objects (lurker)
+
+Count of how many tests and objects have been tested against each
+other by the ban-lurker. 'ban req.url == foo && req.http.host ==
+bar' counts as one in 'bans_tested' and as two in 'bans_tests_tested'
+
+.. varnish_vsc:: bans_lurker_obj_killed
+ :level: diag
+ :oneliner: Objects killed by bans (lurker)
+
+Number of objects killed by the ban-lurker.
+
+.. varnish_vsc:: bans_lurker_obj_killed_cutoff
+ :level: diag
+ :oneliner: Objects killed by bans for cutoff (lurker)
+
+Number of objects killed by the ban-lurker to keep the number of
+bans below ban_cutoff.
+
+.. varnish_vsc:: bans_dups
+ :level: diag
+ :oneliner: Bans superseded by other bans
+
+Count of bans replaced by later identical bans.
+
+.. varnish_vsc:: bans_lurker_contention
+ :level: diag
+ :oneliner: Lurker gave way for lookup
+
+Number of times the ban-lurker had to wait for lookups.
+
+.. varnish_vsc:: bans_persisted_bytes
+ :type: gauge
+ :level: diag
+ :oneliner: Bytes used by the persisted ban lists
+
+Number of bytes used by the persisted ban lists.
+
+.. varnish_vsc:: bans_persisted_fragmentation
+ :type: gauge
+ :level: diag
+ :oneliner: Extra bytes in persisted ban lists due to fragmentation
+
+Number of extra bytes accumulated through dropped and completed
+bans in the persistent ban lists.
+
+.. varnish_vsc:: n_purges
+ :type: gauge
+ :oneliner: Number of purge operations executed
+
+
+.. varnish_vsc:: n_obj_purged
+ :type: gauge
+ :oneliner: Number of purged objects
+
+
+.. varnish_vsc:: exp_mailed
+ :level: diag
+ :oneliner: Number of objects mailed to expiry thread
+
+Number of objects mailed to expiry thread for handling.
+
+.. varnish_vsc:: exp_received
+ :level: diag
+ :oneliner: Number of objects received by expiry thread
+
+Number of objects received by expiry thread for handling.
+
+.. varnish_vsc:: hcb_nolock
+ :level: debug
+ :oneliner: HCB Lookups without lock
+
+
+.. varnish_vsc:: hcb_lock
+ :level: debug
+ :oneliner: HCB Lookups with lock
+
+
+.. varnish_vsc:: hcb_insert
+ :level: debug
+ :oneliner: HCB Inserts
+
+
+.. varnish_vsc:: esi_errors
+ :level: diag
+ :oneliner: ESI parse errors (unlock)
+
+
+.. varnish_vsc:: esi_warnings
+ :level: diag
+ :oneliner: ESI parse warnings (unlock)
+
+
+.. varnish_vsc:: vmods
+ :type: gauge
+ :oneliner: Loaded VMODs
+
+
+.. varnish_vsc:: n_gzip
+ :oneliner: Gzip operations
+
+
+.. varnish_vsc:: n_gunzip
+ :oneliner: Gunzip operations
+
+
+.. varnish_vsc:: n_test_gunzip
+ :oneliner: Test gunzip operations
+
+Those operations occur when Varnish receives a compressed object
+from a backend. They are done to verify the gzip stream while it's
+inserted in storage.
+
+.. varnish_vsc:: vsm_free
+ :type: gauge
+ :level: diag
+ :oneliner: Free VSM space
+
+Number of bytes free in the shared memory used to communicate with
+tools like varnishstat, varnishlog etc.
+
+.. varnish_vsc:: vsm_used
+ :type: gauge
+ :level: diag
+ :oneliner: Used VSM space
+
+Number of bytes used in the shared memory used to communicate with
+tools like varnishstat, varnishlog etc.
+
+.. varnish_vsc:: vsm_cooling
+ :type: gauge
+ :level: debug
+ :oneliner: Cooling VSM space
+
+Number of bytes which will soon (max 1 minute) be freed in the
+shared memory used to communicate with tools like varnishstat,
+varnishlog etc.
+
+.. varnish_vsc:: vsm_overflow
+ :type: gauge
+ :level: diag
+ :oneliner: Overflow VSM space
+
+Number of bytes which does not fit in the shared memory used to
+communicate with tools like varnishstat, varnishlog etc. If this
+counter is not zero, consider increasing the runtime variable
+vsm_space.
+
+.. varnish_vsc:: vsm_overflowed
+ :level: diag
+ :oneliner: Overflowed VSM space
+
+Total number of bytes which did not fit in the shared memory used
+to communicate with tools like varnishstat, varnishlog etc. If this
+counter is not zero, consider increasing the runtime variable
+vsm_space.
+
+.. varnish_vsc_end:: MAIN
diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am
index 795b173..133dd1b 100644
--- a/lib/libvcc/Makefile.am
+++ b/lib/libvcc/Makefile.am
@@ -36,7 +36,8 @@ EXTRA_DIST = \
generate.py
dist_pkgdata_SCRIPTS = \
- vmodtool.py
+ vmodtool.py \
+ vsctool.py
vcc_obj.c vcc_fixed_token.c vcc_token_defs.h: \
$(top_builddir)/include/vcl.h
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
new file mode 100644
index 0000000..afc2ffb
--- /dev/null
+++ b/lib/libvcc/vsctool.py
@@ -0,0 +1,314 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2017 Varnish Software AS
+# All rights reserved.
+#
+# Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+"""
+This program compiles a .vsc file to C language constructs.
+"""
+
+from __future__ import print_function
+
+import getopt
+import json
+import sys
+import gzip
+import StringIO
+import collections
+
+def gzip_str(s):
+ out = StringIO.StringIO()
+ gzip.GzipFile(fileobj=out, mode="w").write(s)
+ return out.getvalue()
+
+def genhdr(fo, name):
+ fo.write('/*\n')
+ fo.write(' * NB: This file is machine generated, DO NOT EDIT!\n')
+ fo.write(' *\n')
+ fo.write(' * Edit %s.vsc run lib/libvcc/vsctool.py instead.\n' % name)
+ fo.write(' */\n')
+ fo.write('\n')
+
+#######################################################################
+
+class vscset(object):
+ def __init__(self, name, m):
+ self.name = name
+ self.struct = "struct VSC_" + name
+ self.mbrs = []
+ self.head = m
+ self.completed = False
+
+ def addmbr(self, m):
+ assert not self.completed
+ self.mbrs.append(m)
+
+ def complete(self):
+ self.completed = True
+
+ def emit_json(self, fo):
+ dd = collections.OrderedDict()
+ dd["version"] = "1"
+ dd["name"] = self.name
+ dd["1line"] = self.head.param["oneliner"].strip()
+ dd["docs"] = self.head.getdoc()
+ dd["elements"] = len(self.mbrs)
+ el = collections.OrderedDict()
+ dd["elem"] = el
+ en = 0
+ for i in self.mbrs:
+ en += 1
+ ed = collections.OrderedDict()
+ el[i.arg] = ed
+ ed["index"] = en
+ ed["name"] = i.arg
+ ed["type"] = i.param["type"]
+ ed["level"] = i.param["level"]
+ ed["1line"] = i.param["oneliner"].strip()
+ ed["docs"] = i.getdoc()
+ s=json.dumps(dd, separators=(",",":"))
+ fo.write("\nstatic const size_t %s_jsonlen = %dL;\n" %
+ (self.name, len(s)))
+ z = gzip_str(s)
+ fo.write("\nstatic const unsigned char");
+ fo.write(" %s_zjson[%d] = {\n" % (self.name, len(z)))
+ bz = bytearray(z)
+ t = "\t"
+ for i in bz:
+ t += "%d," % i
+ if len(t) >= 70:
+ fo.write(t + "\n")
+ t = "\t"
+ if len(t) > 1:
+ fo.write(t[:-1])
+ fo.write("\n};\n")
+ s = json.dumps(dd, indent=2, separators=(',', ': '))
+ fo.write("\n// ")
+ fo.write("\n// ".join(s.split("\n")))
+ fo.write("\n")
+
+
+ def emit_h(self):
+ fon="VSC_" + self.name + ".h"
+ fo = open(fon, "w")
+ genhdr(fo, self.name)
+ fo.write(self.struct + " {\n")
+ for i in self.mbrs:
+ fo.write("\tuint64_t\t%s;\n" % i.arg)
+ fo.write("};\n")
+ fo.write("\n");
+ fo.write(self.struct + " *VSC_" + self.name + "_New")
+ fo.write("(const char *fmt, ...);\n");
+ fo.write("void VSCL_" + self.name + "_Destroy")
+ fo.write("(" + self.struct + "**);\n")
+
+ def emit_c(self):
+ fon="VSC_" + self.name + ".c"
+ fo = open(fon, "w")
+ genhdr(fo, self.name)
+ fo.write('#include "config.h"\n')
+ fo.write('#include <stdarg.h>\n')
+ fo.write('#include <stdio.h>\n')
+ fo.write('#include <stdint.h>\n')
+ fo.write('#include "common/common.h"\n')
+ fo.write('#include "VSC_%s.h"\n' % self.name)
+ self.emit_json(fo)
+ fo.write("\n")
+ fo.write(self.struct + "*\n");
+ fo.write("VSC_" + self.name + "_New")
+ fo.write("(const char *fmt, ...)\n");
+ fo.write("{\n")
+ fo.write("\tva_list ap;\n")
+ fo.write("\t" + self.struct + " *retval;\n")
+ fo.write("\n")
+ fo.write("\tva_start(ap, fmt);\n")
+ fo.write("\tretval = VSC_Alloc")
+ fo.write('("' + self.name + '", ')
+ fo.write(self.name + "_jsonlen, ")
+ fo.write(self.name + "_zjson, ")
+ fo.write("sizeof " + self.name + "_zjson,\n")
+ fo.write("\t fmt, ap);\n")
+ fo.write("\tva_end(ap);\n")
+ fo.write("\treturn(retval);\n")
+ fo.write("}\n")
+ fo.write("\n")
+ fo.write("void\n")
+ fo.write("VSCL_" + self.name + "_Destroy")
+ fo.write("(" + self.struct + "**pp)\n")
+ fo.write("{\n")
+ fo.write("\n")
+ fo.write("\tAN(pp);\n")
+ fo.write('\tVSC_Destroy("%s", *pp);\n' % self.name)
+ fo.write("\t*pp = NULL;\n")
+ fo.write("}\n")
+
+#######################################################################
+
+class directive(object):
+ def __init__(self, s):
+ ll = s.split("\n")
+ i = ll.pop(0).split("::", 2)
+ self.cmd = i[0]
+ self.arg = i[1].strip()
+ assert len(self.arg.split()) == 1
+
+ self.param = {}
+ while len(ll):
+ j = ll[0].split(":",2)
+ if len(j) != 3 or not j[0].isspace():
+ break
+ self.param[j[1]] = j[2].strip()
+ ll.pop(0)
+ self.ldoc = ll
+
+ def getdoc(self):
+ while len(self.ldoc) and self.ldoc[0].strip() == "":
+ self.ldoc.pop(0)
+ while len(self.ldoc) and self.ldoc[-1].strip() == "":
+ self.ldoc.pop(-1)
+ return self.ldoc
+
+ def moredoc(self, s):
+ self.getdoc()
+ self.ldoc += s.split("\n")
+
+ def emit_rst(self, fo):
+ fo.write("\n.. " + self.cmd + ":: " + self.arg + "\n")
+ self.emit_rst_doc(fo)
+
+ def emit_rst_doc(self, fo):
+ fo.write("\n".join(self.ldoc))
+
+ def emit_h(self, fo):
+ return
+
+class rst_vsc_begin(directive):
+ def __init__(self, s):
+ super(rst_vsc_begin, self).__init__(s)
+
+ def vscset(self, ss):
+ ss.append(vscset(self.arg, self))
+
+class rst_vsc(directive):
+ def __init__(self, s):
+ super(rst_vsc, self).__init__(s)
+ if "type" not in self.param:
+ self.param["type"] = "counter"
+ if "level" not in self.param:
+ self.param["level"] = "info"
+
+ def emit_rst(self, fo):
+ fo.write("\n``%s`` - " % self.arg)
+ fo.write("`%s` - " % self.param["type"])
+ fo.write("%s\n" % self.param["level"])
+ self.emit_rst_doc(fo)
+
+ def vscset(self, ss):
+ ss[-1].addmbr(self)
+
+
+class rst_vsc_end(directive):
+ def __init__(self, s):
+ super(rst_vsc_end, self).__init__(s)
+
+ def vscset(self, ss):
+ ss[-1].complete()
+
+class other(object):
+ def __init__(self, s):
+ self.s = s
+
+ def emit_rst(self, fo):
+ fo.write(self.s)
+
+ def emit_h(self, fo):
+ return
+
+ def vscset(self, ss):
+ return
+
+#######################################################################
+
+class vsc_file(object):
+ def __init__(self, fin):
+ self.c = []
+ scs = open(fin).read().split("\n.. ")
+ self.c.append(other(scs[0]))
+ ld = None
+ for i in scs[1:]:
+ j = i.split(None, 1)
+ f = {
+ "varnish_vsc_begin::": rst_vsc_begin,
+ "varnish_vsc::": rst_vsc,
+ "varnish_vsc_end::": rst_vsc_end,
+ }.get(j[0])
+ if f is None:
+ s = "\n.. " + i
+ o = other(s)
+ if ld is not None:
+ ld.moredoc(s)
+ else:
+ o = f(i)
+ ld = o
+ self.c.append(o)
+
+ self.vscset = []
+ for i in self.c:
+ i.vscset(self.vscset)
+
+ print(self.vscset)
+
+ def emit_h(self):
+ for i in self.vscset:
+ i.emit_h()
+
+ def emit_c(self):
+ for i in self.vscset:
+ i.emit_c()
+
+ def emit_rst(self, fon):
+ fo = open(fon, "w")
+ for i in self.c:
+ i.emit_rst(fo)
+
+#######################################################################
+
+if __name__ == "__main__":
+
+ optlist, args = getopt.getopt(sys.argv[1:], "")
+
+ fo = sys.stdout
+
+ for f, v in optlist:
+ assert False
+ if len(args) != 1:
+ print("Need exactly one filename argument")
+ exit(2)
+
+ vf = vsc_file(args[0])
+ vf.emit_rst("_.rst")
+ vf.emit_h()
+ vf.emit_c()
More information about the varnish-commit
mailing list