r2579 - in branches/1.1: . bin/varnishlog bin/varnishreplay include lib/libvarnishapi

des at projects.linpro.no des at projects.linpro.no
Sun Mar 9 16:26:55 CET 2008


Author: des
Date: 2008-03-09 16:26:55 +0100 (Sun, 09 Mar 2008)
New Revision: 2579

Modified:
   branches/1.1/
   branches/1.1/bin/varnishlog/varnishlog.1
   branches/1.1/bin/varnishreplay/varnishreplay.c
   branches/1.1/include/varnishapi.h
   branches/1.1/lib/libvarnishapi/shmlog.c
Log:
Merged revisions 2575-2577 via svnmerge from 
svn+ssh://projects.linpro.no/svn/varnish/trunk/varnish-cache

........
  r2575 | des | 2008-03-09 16:14:04 +0100 (Sun, 09 Mar 2008) | 4 lines
  
  Add -k option which specifies the number of log entries to keep.  Along with
  -s, this allows varnishlog to be used to extract part of a log file, or
  partition a log file into smaller sections.
........
  r2576 | des | 2008-03-09 16:17:49 +0100 (Sun, 09 Mar 2008) | 4 lines
  
  Include complete headers with each replayed request.
  Ignore SIGPIPE, otherwise varnishreplay will stop as soon as varnishd starts
  overflowing.
........
  r2577 | des | 2008-03-09 16:24:37 +0100 (Sun, 09 Mar 2008) | 2 lines
  
  Braino in previous commit.
........



Property changes on: branches/1.1
___________________________________________________________________
Name: svnmerge-integrated
   - /trunk/varnish-cache:1-1722,1727-1729,1733,1738,1743-1777,1779-1798,1800-1815,1817,1819,1823,1830-1838,1846,1853-1855,1857-1859,1862,1865-1868,1871-1880,1883-1884,1886,1888-1889,1896,1898,1902-1905,1907,1909,1912-1916,1920-1928,1935-1939,1941-1949,1955,1957-1958,1967-1968,1970-1974,1976-1977,1984,1986-1989,1991-1998,2026,2031-2033,2057,2077-2080,2086,2088,2091,2097,2106-2107,2116,2133,2154,2173,2181,2206-2207,2211-2212,2215-2245,2256-2262,2270-2271,2275,2285-2286,2288-2291,2295-2301,2304-2327,2337,2357-2359,2361-2364,2366,2374-2386,2404,2414-2415,2421-2422,2426,2432-2434,2444-2445,2447,2453-2461,2467,2492-2505,2520-2524,2545,2563-2565,2569,2572
   + /trunk/varnish-cache:1-1722,1727-1729,1733,1738,1743-1777,1779-1798,1800-1815,1817,1819,1823,1830-1838,1846,1853-1855,1857-1859,1862,1865-1868,1871-1880,1883-1884,1886,1888-1889,1896,1898,1902-1905,1907,1909,1912-1916,1920-1928,1935-1939,1941-1949,1955,1957-1958,1967-1968,1970-1974,1976-1977,1984,1986-1989,1991-1998,2026,2031-2033,2057,2077-2080,2086,2088,2091,2097,2106-2107,2116,2133,2154,2173,2181,2206-2207,2211-2212,2215-2245,2256-2262,2270-2271,2275,2285-2286,2288-2291,2295-2301,2304-2327,2337,2357-2359,2361-2364,2366,2374-2386,2404,2414-2415,2421-2422,2426,2432-2434,2444-2445,2447,2453-2461,2467,2492-2505,2520-2524,2545,2563-2565,2569,2572,2575-2577

Modified: branches/1.1/bin/varnishlog/varnishlog.1
===================================================================
--- branches/1.1/bin/varnishlog/varnishlog.1	2008-03-09 15:26:31 UTC (rev 2578)
+++ branches/1.1/bin/varnishlog/varnishlog.1	2008-03-09 15:26:55 UTC (rev 2579)
@@ -28,7 +28,7 @@
 .\"
 .\" $Id$
 .\"
-.Dd March 8, 2008
+.Dd March 9, 2008
 .Dt VARNISHLOG 1
 .Os
 .Sh NAME
@@ -44,6 +44,7 @@
 .Op Fl d
 .Op Fl I Ar regex
 .Op Fl i Ar tag
+.Op Fl k Ar keep
 .Op Fl n Ar varnish_name
 .Op Fl o
 .Op Fl P Ar file
@@ -108,6 +109,10 @@
 nor
 .Fl i
 is specified, all log entries are included.
+.It Fl k Ar num
+Only show the first
+.Nm num
+log records.
 .It Fl n
 Specifies the name of the
 .Nm varnishd

Modified: branches/1.1/bin/varnishreplay/varnishreplay.c
===================================================================
--- branches/1.1/bin/varnishreplay/varnishreplay.c	2008-03-09 15:26:31 UTC (rev 2578)
+++ branches/1.1/bin/varnishreplay/varnishreplay.c	2008-03-09 15:26:55 UTC (rev 2579)
@@ -30,6 +30,9 @@
 
 #include "config.h"
 
+#include <sys/signal.h>
+#include <sys/uio.h>
+
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -172,7 +175,7 @@
 		while (fd >= newnthreads)
 			newnthreads += newnthreads + 1;
 		newthreads = realloc(newthreads, newnthreads * sizeof *newthreads);
-		assert(newthreads != NULL);
+		XXXAN(newthreads != NULL);
 		memset(newthreads + nthreads, 0,
 		    (newnthreads - nthreads) * sizeof *newthreads);
 		threads = newthreads;
@@ -188,7 +191,7 @@
 			mailbox_destroy(&threads[fd]->mbox);
 			freez(threads[fd]);
 		}
-		thread_log(1, 0, "thread %p started",
+		thread_log(0, 0, "thread %p started",
 		    (void *)threads[fd]->thread_id);
 	}
 	return (threads[fd]);
@@ -209,7 +212,7 @@
 		return;
 	mailbox_close(&threads[fd]->mbox);
 	pthread_join(threads[fd]->thread_id, NULL);
-	thread_log(1, 0, "thread %p stopped",
+	thread_log(0, 0, "thread %p stopped",
 	    (void *)threads[fd]->thread_id);
 	mailbox_destroy(&threads[fd]->mbox);
 	freez(threads[fd]);
@@ -236,6 +239,7 @@
 	return (1);
 }
 
+#if 0
 static int
 isequal(const char *str, const char *reference, const char *end)
 {
@@ -247,6 +251,7 @@
 		return (0);
 	return (1);
 }
+#endif
 
 /*
  * Returns a copy of the entire string with leading and trailing spaces
@@ -403,6 +408,7 @@
 		line_len = read_line(&line, sock);
 		if (line_len < 0)
 			return (-1);
+		thread_log(2, 0, "< %.*s", line_len, line);
 		end = line + line_len;
 		if (line_len == 0) {
 			freez(line);
@@ -473,6 +479,8 @@
 static void *
 replay_thread(void *arg)
 {
+	struct iovec iov[6];
+	char space[1] = " ", crlf[2] = "\r\n";
 	struct thread *thr = arg;
 	struct message *msg;
 	enum shmlogtag tag;
@@ -480,15 +488,21 @@
 	char *ptr;
 	const char *end, *next;
 
-	char *df_H = NULL;			/* %H, Protocol version */
-	char *df_Host = NULL;			/* %{Host}i */
-	char *df_Uq = NULL;			/* %U%q, URL path and query string */
-	char *df_m = NULL;			/* %m, Request method*/
-	char *df_c = NULL;			/* Connection info (keep-alive, close) */
+	char *df_method = NULL;			/* Request method*/
+	char *df_proto = NULL;			/* Protocol version */
+	char *df_url = NULL;			/* URL and query string */
+	char *df_conn = NULL;			/* Connection info (keep-alive, close) */
+	char **df_hdr = NULL;			/* Headers */
+	size_t df_hdrsz = 0;			/* Size of df_hdr */
+	int df_nhdr = 0;			/* Number of headers */
 	int bogus = 0;				/* bogus request */
+	int i;
 
-	int sock, reopen = 1;
+	int sock = -1, reopen = 1;
 
+	df_hdrsz = 16;
+	df_hdr = malloc(df_hdrsz * sizeof *df_hdr);
+
 	while ((msg = mailbox_get(&thr->mbox)) != NULL) {
 		tag = msg->tag;
 		len = msg->len;
@@ -499,115 +513,134 @@
 
 		switch (tag) {
 		case SLT_RxRequest:
-			if (df_m != NULL)
+			if (df_method != NULL)
 				bogus = 1;
 			else
-				df_m = trimline(ptr, end);
+				df_method = trimline(ptr, end);
 			break;
 
 		case SLT_RxURL:
-			if (df_Uq != NULL)
+			if (df_url != NULL)
 				bogus = 1;
 			else
-				df_Uq = trimline(ptr, end);
+				df_url = trimline(ptr, end);
 			break;
 
 		case SLT_RxProtocol:
-			if (df_H != NULL)
+			if (df_proto != NULL)
 				bogus = 1;
 			else
-				df_H = trimline(ptr, end);
+				df_proto = trimline(ptr, end);
 			break;
 
 		case SLT_RxHeader:
-			if (isprefix(ptr, "host:", end, &next))
-				df_Host = trimline(next, end);
+			while (df_hdrsz <= df_nhdr) {
+				df_hdrsz *= 2;
+				df_hdr = realloc(df_hdr, df_hdrsz * sizeof *df_hdr);
+				XXXAN(df_hdr);
+			}
+			df_hdr[df_nhdr++] = trimline(ptr, end);
 			if (isprefix(ptr, "connection:", end, &next))
-				df_c = trimline(next, end);
+				df_conn = trimline(next, end);
 			break;
 
 		default:
 			break;
 		}
 
+		freez(msg->ptr);
+		freez(msg);
+
 		if (tag != SLT_ReqEnd)
 			continue;
 
-		if (!df_m || !df_Uq || !df_H)
+		if (!df_method || !df_url || !df_proto) {
 			bogus = 1;
+		} else if (strcmp(df_method, "GET") != 0 && strcmp(df_method, "HEAD") != 0) {
+			bogus = 1;
+		} else if (strcmp(df_proto, "HTTP/1.0") == 0) {
+			reopen = !(df_conn && strcasecmp(df_conn, "keep-alive") == 0);
+		} else if (strcmp(df_proto, "HTTP/1.1") == 0) {
+			reopen = (df_conn && strcasecmp(df_conn, "close") == 0);
+		} else {
+			bogus = 1;
+		}
 
 		if (bogus) {
 			thread_log(1, 0, "bogus");
 		} else {
-			/* If the method is supported (GET or HEAD), send the request out
-			 * on the socket. If the socket needs reopening, reopen it first.
-			 * When the request is sent, call the function for receiving
-			 * the answer.
-			 */
-			if (!(strcmp(df_m, "GET") && strcmp(df_m, "HEAD"))) {
-				if (reopen)
-					sock = VSS_connect(addr_info);
-				reopen = 0;
+			if (sock == -1) {
+				thread_log(1, 0, "open");
+				sock = VSS_connect(addr_info);
+				assert(sock != -1);
+			}
 
-				thread_log(1, 0, "%s %s %s", df_m, df_Uq, df_H);
+			thread_log(1, 0, "%s %s %s", df_method, df_url, df_proto);
 
-				write(sock, df_m, strlen(df_m));
-				write(sock, " ", 1);
-				write(sock, df_Uq, strlen(df_Uq));
-				write(sock, " ", 1);
-				write(sock, df_H, strlen(df_H));
-				write(sock, " ", 1);
-				write(sock, "\r\n", 2);
+			iov[0].iov_base = df_method;
+			iov[0].iov_len = strlen(df_method);
+			iov[2].iov_base = df_url;
+			iov[2].iov_len = strlen(df_url);
+			iov[4].iov_base = df_proto;
+			iov[4].iov_len = strlen(df_proto);
+			iov[1].iov_base = iov[3].iov_base = space;
+			iov[1].iov_len = iov[3].iov_len = 1;
+			iov[5].iov_base = crlf;
+			iov[5].iov_len = 2;
+			if (writev(sock, iov, 6) == -1) {
+				thread_log(0, errno, "writev()");
+				goto close;
+			}
 
-				if (strncmp(df_H, "HTTP/1.0", 8))
-					reopen = 1;
-
-				write(sock, "Host: ", 6);
-				if (df_Host) {
-					thread_log(1, 0, "Host: %s", df_Host);
-					write(sock, df_Host, strlen(df_Host));
+			for (i = 0; i < df_nhdr; ++i) {
+				thread_log(2, 0, "%d %s", i, df_hdr[i]);
+				iov[0].iov_base = df_hdr[i];
+				iov[0].iov_len = strlen(df_hdr[i]);
+				iov[1].iov_base = crlf;
+				iov[1].iov_len = 2;
+				if (writev(sock, iov, 2) == -1) {
+					thread_log(0, errno, "writev()");
+					goto close;
 				}
-				write(sock, "\r\n", 2);
-				if (df_c) {
-					thread_log(1, 0, "Connection: %s", df_c);
-					write(sock, "Connection: ", 12);
-					write(sock, df_c, strlen(df_c));
-					write(sock, "\r\n", 2);
-					if (isequal(df_c, "keep-alive", df_c + strlen(df_c)))
-						reopen = 0;
-				}
-				write(sock, "\r\n", 2);
-				if (!reopen)
-					reopen = receive_response(sock);
-				if (reopen)
-					close(sock);
 			}
+			if (write(sock, crlf, 2) == -1) {
+				thread_log(0, errno, "writev()");
+				goto close;
+			}
+			if (receive_response(sock) || reopen) {
+close:
+				thread_log(1, 0, "close");
+				close(sock);
+				sock = -1;
+			}
 		}
 
 		/* clean up */
-		freez(msg->ptr);
-		freez(msg);
-		freez(df_H);
-		freez(df_Host);
-		freez(df_Uq);
-		freez(df_m);
-		freez(df_c);
+		freez(df_method);
+		freez(df_url);
+		freez(df_proto);
+		freez(df_conn);
+		while (df_nhdr) {
+			--df_nhdr;
+			freez(df_hdr[df_nhdr]);
+		}
 		bogus = 0;
 	}
 
 	/* leftovers */
-	freez(msg->ptr);
-	freez(msg);
-	freez(df_H);
-	freez(df_Host);
-	freez(df_Uq);
-	freez(df_m);
-	freez(df_c);
+	freez(df_method);
+	freez(df_url);
+	freez(df_proto);
+	freez(df_conn);
+	while (df_nhdr) {
+		--df_nhdr;
+		freez(df_hdr[df_nhdr]);
+	}
+	freez(df_hdr);
 
 	return (0);
 }
 
-
 static int
 gen_traffic(void *priv, enum shmlogtag tag, unsigned fd,
     unsigned len, unsigned spec, const char *ptr)
@@ -623,7 +656,7 @@
 	if (fd == 0 || !(spec & VSL_S_CLIENT))
 		return (0);
 
-	thread_log(2, 0, "%d %s", fd, VSL_tags[tag]);
+	thread_log(3, 0, "%d %s", fd, VSL_tags[tag]);
 	thr = thread_get(fd, replay_thread);
 	msg = malloc(sizeof (struct message));
 	msg->tag = tag;
@@ -679,6 +712,8 @@
 
 	addr_info = init_connection(address);
 
+	signal(SIGPIPE, SIG_IGN);
+
 	while (VSL_Dispatch(vd, gen_traffic, NULL) == 0)
 		/* nothing */ ;
 	thread_close(0);

Modified: branches/1.1/include/varnishapi.h
===================================================================
--- branches/1.1/include/varnishapi.h	2008-03-09 15:26:31 UTC (rev 2578)
+++ branches/1.1/include/varnishapi.h	2008-03-09 15:26:55 UTC (rev 2579)
@@ -44,8 +44,8 @@
 typedef int vsl_handler(void *priv, enum shmlogtag tag, unsigned fd, unsigned len, unsigned spec, const char *ptr);
 #define VSL_S_CLIENT	(1 << 0)
 #define VSL_S_BACKEND	(1 << 1)
-#define VSL_ARGS	"bCcdI:i:r:s:X:x:"
-#define VSL_USAGE	"[-bCcd] [-i tag] [-I regexp] [-r file] [-s skip] [-X regexp] [-x tag]"
+#define VSL_ARGS	"bCcdI:i:k:r:s:X:x:"
+#define VSL_USAGE	"[-bCcd] [-i tag] [-I regexp] [-k keep] [-r file] [-s skip] [-X regexp] [-x tag]"
 vsl_handler VSL_H_Print;
 struct VSL_data;
 struct VSL_data *VSL_New(void);

Modified: branches/1.1/lib/libvarnishapi/shmlog.c
===================================================================
--- branches/1.1/lib/libvarnishapi/shmlog.c	2008-03-09 15:26:31 UTC (rev 2578)
+++ branches/1.1/lib/libvarnishapi/shmlog.c	2008-03-09 15:26:55 UTC (rev 2579)
@@ -87,6 +87,7 @@
 	regex_t			*regexcl;
 
 	unsigned long		skip;
+	unsigned long		keep;
 };
 
 #ifndef MAP_HASSEMAPHORE
@@ -294,6 +295,9 @@
 		if (vd->skip) {
 			--vd->skip;
 			continue;
+		} else if (vd->keep) {
+			if (--vd->keep == 0)
+				return (0);
 		}
 		if (vd->map[p[0]] & M_SELECT) {
 			*pp = p;
@@ -495,7 +499,26 @@
 	}
 	return (1);
 }
+/*--------------------------------------------------------------------*/
 
+static int
+vsl_k_arg(struct VSL_data *vd, const char *opt)
+{
+	char *end;
+
+	CHECK_OBJ_NOTNULL(vd, VSL_MAGIC);
+	if (*opt == '\0') {
+		fprintf(stderr, "number required for -k\n");
+		return (-1);
+	}
+	vd->keep = strtoul(opt, &end, 10);
+	if (*end != '\0') {
+		fprintf(stderr, "invalid number for -k\n");
+		return (-1);
+	}
+	return (1);
+}
+
 /*--------------------------------------------------------------------*/
 
 int
@@ -512,6 +535,7 @@
 	case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg));
 	case 'C': vd->regflags = REG_ICASE; return (1);
 	case 's': return (vsl_s_arg(vd, opt));
+	case 'k': return (vsl_k_arg(vd, opt));
 	default:
 		return (0);
 	}




More information about the varnish-commit mailing list