varnish-cache/bin/varnishtest/vtc_haproxy.c
0
/*-
1
 * Copyright (c) 2008-2018 Varnish Software AS
2
 * All rights reserved.
3
 *
4
 * Author: Frédéric Lécaille <flecaille@haproxy.com>
5
 *
6
 * SPDX-License-Identifier: BSD-2-Clause
7
 *
8
 * Redistribution and use in source and binary forms, with or without
9
 * modification, are permitted provided that the following conditions
10
 * are met:
11
 * 1. Redistributions of source code must retain the above copyright
12
 *    notice, this list of conditions and the following disclaimer.
13
 * 2. Redistributions in binary form must reproduce the above copyright
14
 *    notice, this list of conditions and the following disclaimer in the
15
 *    documentation and/or other materials provided with the distribution.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27
 * SUCH DAMAGE.
28
 */
29
30
#include "config.h"
31
32
#include <inttypes.h>
33
#include <stdio.h>
34
#include <stdlib.h>
35
#include <string.h>
36
#include <sys/stat.h> /* for MUSL (mode_t) */
37
#include <sys/types.h>
38
#include <sys/socket.h>
39
#include <unistd.h>
40
41
#include "vtc.h"
42
43
#include "vfil.h"
44
#include "vpf.h"
45
#include "vre.h"
46
#include "vtcp.h"
47
#include "vsa.h"
48
#include "vtim.h"
49
50
#define HAPROXY_PROGRAM_ENV_VAR "HAPROXY_PROGRAM"
51
#define HAPROXY_ARGS_ENV_VAR    "HAPROXY_ARGS"
52
#define HAPROXY_OPT_WORKER      "-W"
53
#define HAPROXY_OPT_MCLI        "-S"
54
#define HAPROXY_OPT_DAEMON      "-D"
55
#define HAPROXY_SIGNAL          SIGINT
56
#define HAPROXY_EXPECT_EXIT     (128 + HAPROXY_SIGNAL)
57
58
struct envar {
59
        VTAILQ_ENTRY(envar) list;
60
        char *name;
61
        char *value;
62
};
63
64
struct haproxy {
65
        unsigned                magic;
66
#define HAPROXY_MAGIC           0x8a45cf75
67
        char                    *name;
68
        struct vtclog           *vl;
69
        VTAILQ_ENTRY(haproxy)   list;
70
71
        const char              *filename;
72
        struct vsb              *args;
73
        int                     opt_worker;
74
        int                     opt_mcli;
75
        int                     opt_daemon;
76
        int                     opt_check_mode;
77
        char                    *pid_fn;
78
        pid_t                   pid;
79
        pid_t                   ppid;
80
        int                     fds[4];
81
        char                    *cfg_fn;
82
        struct vsb              *cfg_vsb;
83
84
        pthread_t               tp;
85
        int                     expect_exit;
86
        int                     expect_signal;
87
        int                     its_dead_jim;
88
89
        /* UNIX socket CLI. */
90
        char                    *cli_fn;
91
        /* TCP socket CLI. */
92
        struct haproxy_cli *cli;
93
94
        /* master CLI */
95
        struct haproxy_cli *mcli;
96
97
        char                    *workdir;
98
        struct vsb              *msgs;
99
        char                    closed_sock[256]; /* Closed TCP socket */
100
        VTAILQ_HEAD(,envar) envars;
101
};
102
103
static VTAILQ_HEAD(, haproxy)   haproxies =
104
    VTAILQ_HEAD_INITIALIZER(haproxies);
105
106
struct haproxy_cli {
107
        unsigned                magic;
108
#define HAPROXY_CLI_MAGIC       0xb09a4ed8
109
        struct vtclog           *vl;
110
        char                    running;
111
112
        char                    *spec;
113
114
        int                     sock;
115
        char                    connect[256];
116
117
        pthread_t               tp;
118
        size_t                  txbuf_sz;
119
        char                    *txbuf;
120
        size_t                  rxbuf_sz;
121
        char                    *rxbuf;
122
123
        double                  timeout;
124
};
125
126
static void haproxy_write_conf(struct haproxy *h);
127
128
static void
129 680
haproxy_add_envar(struct haproxy *h,
130
                  const char *name, const char *value)
131
{
132
        struct envar *e;
133
134 680
        e = malloc(sizeof *e);
135 680
        AN(e);
136 680
        e->name = strdup(name);
137 680
        e->value = strdup(value);
138 680
        AN(e->name);
139 680
        AN(e->value);
140 680
        VTAILQ_INSERT_TAIL(&h->envars, e, list);
141 680
}
142
143
static void
144 400
haproxy_delete_envars(struct haproxy *h)
145
{
146
        struct envar *e, *e2;
147 1080
        VTAILQ_FOREACH_SAFE(e, &h->envars, list, e2) {
148 680
                VTAILQ_REMOVE(&h->envars, e, list);
149 680
                free(e->name);
150 680
                free(e->value);
151 680
                free(e);
152 680
        }
153 400
}
154
155
static void
156 400
haproxy_build_env(const struct haproxy *h)
157
{
158
        struct envar *e;
159
160 1080
        VTAILQ_FOREACH(e, &h->envars, list) {
161 680
                if (setenv(e->name, e->value, 0) == -1)
162 0
                        vtc_fatal(h->vl, "setenv() failed: %s (%d)",
163 0
                                  strerror(errno), errno);
164 680
        }
165 400
}
166
167
/**********************************************************************
168
 * Socket connect (same as client_tcp_connect()).
169
 */
170
171
static int
172 40
haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, double tmo,
173
    const char **errp)
174
{
175
        int fd;
176
        char mabuf[VTCP_ADDRBUFSIZE], mpbuf[VTCP_PORTBUFSIZE];
177
178 40
        AN(addr);
179 40
        AN(errp);
180 40
        fd = VTCP_open(addr, NULL, tmo, errp);
181 40
        if (fd < 0)
182 0
                return (fd);
183 40
        VTCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf);
184 80
        vtc_log(vl, 3,
185 40
            "CLI connected fd %d from %s %s to %s", fd, mabuf, mpbuf, addr);
186 40
        return (fd);
187 40
}
188
189
/*
190
 * SECTION: haproxy.cli haproxy CLI Specification
191
 * SECTION: haproxy.cli.send
192
 * send STRING
193
 *         Push STRING on the CLI connection. STRING will be terminated by an
194
 *         end of line character (\n).
195
 */
196
static void v_matchproto_(cmd_f)
197 40
cmd_haproxy_cli_send(CMD_ARGS)
198
{
199
        struct vsb *vsb;
200
        struct haproxy_cli *hc;
201
        int j;
202
203 40
        (void)vl;
204 40
        CAST_OBJ_NOTNULL(hc, priv, HAPROXY_CLI_MAGIC);
205 40
        AZ(strcmp(av[0], "send"));
206 40
        AN(av[1]);
207 40
        AZ(av[2]);
208
209 40
        vsb = VSB_new_auto();
210 40
        AN(vsb);
211 40
        AZ(VSB_cat(vsb, av[1]));
212 40
        AZ(VSB_cat(vsb, "\n"));
213 40
        AZ(VSB_finish(vsb));
214 40
        if (hc->sock == -1) {
215
                int fd;
216
                const char *err;
217
                struct vsb *vsb_connect;
218
219 0
                vsb_connect = macro_expand(hc->vl, hc->connect);
220 0
                AN(vsb_connect);
221 0
                fd = haproxy_cli_tcp_connect(hc->vl,
222 0
                    VSB_data(vsb_connect), 10., &err);
223 0
                if (fd < 0)
224 0
                        vtc_fatal(hc->vl,
225 0
                            "CLI failed to open %s: %s", VSB_data(vsb), err);
226 0
                VSB_destroy(&vsb_connect);
227 0
                hc->sock = fd;
228 0
        }
229 40
        vtc_dump(hc->vl, 4, "CLI send", VSB_data(vsb), -1);
230
231 40
        if (VSB_tofile(vsb, hc->sock))
232 0
                vtc_fatal(hc->vl,
233 0
                    "CLI fd %d send error %s", hc->sock, strerror(errno));
234
235
        /* a CLI command must be followed by a SHUT_WR if we want HAProxy to
236
         * close after the response */
237 40
        j = shutdown(hc->sock, SHUT_WR);
238 40
        vtc_log(hc->vl, 3, "CLI shutting fd %d", hc->sock);
239 40
        if (!VTCP_Check(j))
240 0
                vtc_fatal(hc->vl, "Shutdown failed: %s", strerror(errno));
241
242 40
        VSB_destroy(&vsb);
243 40
}
244
245
#define HAPROXY_CLI_RECV_LEN (1 << 14)
246
static void
247 40
haproxy_cli_recv(struct haproxy_cli *hc)
248
{
249
        ssize_t ret;
250
        size_t rdz, left, off;
251
252 40
        rdz = ret = off = 0;
253
        /* We want to null terminate this buffer. */
254 40
        left = hc->rxbuf_sz - 1;
255 80
        while (!vtc_error && left > 0) {
256 80
                VTCP_set_read_timeout(hc->sock, hc->timeout);
257
258 80
                ret = recv(hc->sock, hc->rxbuf + off, HAPROXY_CLI_RECV_LEN, 0);
259 80
                if (ret < 0) {
260 0
                        if (errno == EINTR || errno == EAGAIN)
261 0
                                continue;
262
263 0
                        vtc_fatal(hc->vl,
264
                            "CLI fd %d recv() failed (%s)",
265 0
                            hc->sock, strerror(errno));
266
                }
267
                /* Connection closed. */
268 80
                if (ret == 0) {
269 40
                        if (rdz > 0 && hc->rxbuf[rdz - 1] != '\n')
270 0
                                vtc_fatal(hc->vl,
271
                                    "CLI rx timeout (fd: %d %.3fs ret: %zd)",
272 0
                                    hc->sock, hc->timeout, ret);
273
274 40
                        vtc_log(hc->vl, 4, "CLI connection normally closed");
275 40
                        vtc_log(hc->vl, 3, "CLI closing fd %d", hc->sock);
276 40
                        VTCP_close(&hc->sock);
277 40
                        break;
278
                }
279
280 40
                rdz += ret;
281 40
                left -= ret;
282 40
                off  += ret;
283
        }
284 40
        hc->rxbuf[rdz] = '\0';
285 40
        vtc_dump(hc->vl, 4, "CLI recv", hc->rxbuf, rdz);
286 40
}
287
288
/*
289
 * SECTION: haproxy.cli.expect
290
 * expect OP STRING
291
 *         Regex match the CLI reception buffer with STRING
292
 *         if OP is ~ or, on the contrary, if OP is !~ check that there is
293
 *         no regex match.
294
 */
295
static void v_matchproto_(cmd_f)
296 40
cmd_haproxy_cli_expect(CMD_ARGS)
297
{
298
        struct haproxy_cli *hc;
299
        struct vsb vsb[1];
300
        vre_t *vre;
301
        int error, erroroffset, i, ret;
302
        char *cmp, *spec, errbuf[VRE_ERROR_LEN];
303
304 40
        (void)vl;
305 40
        CAST_OBJ_NOTNULL(hc, priv, HAPROXY_CLI_MAGIC);
306 40
        AZ(strcmp(av[0], "expect"));
307 40
        av++;
308
309 40
        cmp = av[0];
310 40
        spec = av[1];
311 40
        AN(cmp);
312 40
        AN(spec);
313 40
        AZ(av[2]);
314
315 40
        assert(!strcmp(cmp, "~") || !strcmp(cmp, "!~"));
316
317 40
        haproxy_cli_recv(hc);
318
319 40
        vre = VRE_compile(spec, 0, &error, &erroroffset, 1);
320 40
        if (vre == NULL) {
321 0
                AN(VSB_init(vsb, errbuf, sizeof errbuf));
322 0
                AZ(VRE_error(vsb, error));
323 0
                AZ(VSB_finish(vsb));
324 0
                VSB_fini(vsb);
325 0
                vtc_fatal(hc->vl, "CLI regexp error: '%s' (@%d) (%s)",
326 0
                    errbuf, erroroffset, spec);
327
        }
328
329 40
        i = VRE_match(vre, hc->rxbuf, 0, 0, NULL);
330
331 40
        VRE_free(&vre);
332
333 40
        ret = (i >= 0 && *cmp == '~') || (i < 0 && *cmp == '!');
334 40
        if (!ret)
335 0
                vtc_fatal(hc->vl, "CLI expect failed %s \"%s\"", cmp, spec);
336
        else
337 40
                vtc_log(hc->vl, 4, "CLI expect match %s \"%s\"", cmp, spec);
338 40
}
339
340
static const struct cmds haproxy_cli_cmds[] = {
341
#define CMD_HAPROXY_CLI(n) { #n, cmd_haproxy_cli_##n },
342
        CMD_HAPROXY_CLI(send)
343
        CMD_HAPROXY_CLI(expect)
344
#undef CMD_HAPROXY_CLI
345
        { NULL, NULL }
346
};
347
348
/**********************************************************************
349
 * HAProxy CLI client thread
350
 */
351
352
static void *
353 40
haproxy_cli_thread(void *priv)
354
{
355
        struct haproxy_cli *hc;
356
        struct vsb *vsb;
357
        int fd;
358
        const char *err;
359
360 40
        CAST_OBJ_NOTNULL(hc, priv, HAPROXY_CLI_MAGIC);
361 40
        AN(*hc->connect);
362
363 40
        vsb = macro_expand(hc->vl, hc->connect);
364 40
        AN(vsb);
365
366 40
        fd = haproxy_cli_tcp_connect(hc->vl, VSB_data(vsb), 10., &err);
367 40
        if (fd < 0)
368 0
                vtc_fatal(hc->vl,
369 0
                    "CLI failed to open %s: %s", VSB_data(vsb), err);
370 40
        VTCP_blocking(fd);
371 40
        hc->sock = fd;
372 40
        parse_string(hc->vl, hc, hc->spec);
373 40
        vtc_log(hc->vl, 2, "CLI ending");
374 40
        VSB_destroy(&vsb);
375 40
        return (NULL);
376
}
377
378
/**********************************************************************
379
 * Wait for the CLI client thread to stop
380
 */
381
382
static void
383 40
haproxy_cli_wait(struct haproxy_cli *hc)
384
{
385
        void *res;
386
387 40
        CHECK_OBJ_NOTNULL(hc, HAPROXY_CLI_MAGIC);
388 40
        vtc_log(hc->vl, 2, "CLI waiting");
389 40
        PTOK(pthread_join(hc->tp, &res));
390 40
        if (res != NULL)
391 0
                vtc_fatal(hc->vl, "CLI returned \"%s\"", (char *)res);
392 40
        REPLACE(hc->spec, NULL);
393 40
        hc->tp = 0;
394 40
        hc->running = 0;
395 40
}
396
397
/**********************************************************************
398
 * Start the CLI client thread
399
 */
400
401
static void
402 40
haproxy_cli_start(struct haproxy_cli *hc)
403
{
404 40
        CHECK_OBJ_NOTNULL(hc, HAPROXY_CLI_MAGIC);
405 40
        vtc_log(hc->vl, 2, "CLI starting");
406 40
        PTOK(pthread_create(&hc->tp, NULL, haproxy_cli_thread, hc));
407 40
        hc->running = 1;
408
409 40
}
410
411
/**********************************************************************
412
 * Run the CLI client thread
413
 */
414
415
static void
416 40
haproxy_cli_run(struct haproxy_cli *hc)
417
{
418 40
        haproxy_cli_start(hc);
419 40
        haproxy_cli_wait(hc);
420 40
}
421
422
/**********************************************************************
423
 *
424
 */
425
426
static void
427 120
haproxy_wait_pidfile(struct haproxy *h)
428
{
429 120
        char buf_err[1024] = {0};
430 120
        int usleep_time = 1000;
431
        double t0;
432
        pid_t pid;
433
434 120
        vtc_log(h->vl, 3, "wait-pid-file");
435 4333
        for (t0 = VTIM_mono(); VTIM_mono() - t0 < 3;) {
436 4333
                if (vtc_error)
437 0
                        return;
438
439 4333
                if (VPF_Read(h->pid_fn, &pid) != 0) {
440 4213
                        bprintf(buf_err,
441
                            "Could not read PID file '%s'", h->pid_fn);
442 4213
                        usleep(usleep_time);
443 4213
                        continue;
444
                }
445
446 120
                if (!h->opt_daemon && pid != h->pid) {
447 0
                        bprintf(buf_err,
448
                            "PID file has different PID (%ld != %lld)",
449
                            (long)pid, (long long)h->pid);
450 0
                        usleep(usleep_time);
451 0
                        continue;
452
                }
453
454 120
                if (kill(pid, 0) < 0) {
455 0
                        bprintf(buf_err,
456
                            "Could not find PID %ld process", (long)pid);
457 0
                        usleep(usleep_time);
458 0
                        continue;
459
                }
460
461 120
                h->pid = pid;
462
463 240
                vtc_log(h->vl, 2, "haproxy PID %ld successfully started",
464 120
                    (long)pid);
465 120
                return;
466
        }
467 0
        vtc_fatal(h->vl, "haproxy %s PID file check failed:\n\t%s\n",
468 0
                  h->name, buf_err);
469 120
}
470
471
/**********************************************************************
472
 * Allocate and initialize a CLI client
473
 */
474
475
static struct haproxy_cli *
476 400
haproxy_cli_new(struct haproxy *h)
477
{
478
        struct haproxy_cli *hc;
479
480 400
        ALLOC_OBJ(hc, HAPROXY_CLI_MAGIC);
481 400
        AN(hc);
482
483 400
        hc->vl = h->vl;
484 400
        vtc_log_set_cmd(hc->vl, haproxy_cli_cmds);
485 400
        hc->sock = -1;
486 400
        bprintf(hc->connect, "${%s_cli_sock}", h->name);
487
488 400
        hc->txbuf_sz = hc->rxbuf_sz = 2048 * 1024;
489 400
        hc->txbuf = malloc(hc->txbuf_sz);
490 400
        AN(hc->txbuf);
491 400
        hc->rxbuf = malloc(hc->rxbuf_sz);
492 400
        AN(hc->rxbuf);
493
494 400
        return (hc);
495
}
496
497
/* creates a master CLI client (-mcli) */
498
static struct haproxy_cli *
499 400
haproxy_mcli_new(struct haproxy *h)
500
{
501
        struct haproxy_cli *hc;
502
503 400
        ALLOC_OBJ(hc, HAPROXY_CLI_MAGIC);
504 400
        AN(hc);
505
506 400
        hc->vl = h->vl;
507 400
        vtc_log_set_cmd(hc->vl, haproxy_cli_cmds);
508 400
        hc->sock = -1;
509 400
        bprintf(hc->connect, "${%s_mcli_sock}", h->name);
510
511 400
        hc->txbuf_sz = hc->rxbuf_sz = 2048 * 1024;
512 400
        hc->txbuf = malloc(hc->txbuf_sz);
513 400
        AN(hc->txbuf);
514 400
        hc->rxbuf = malloc(hc->rxbuf_sz);
515 400
        AN(hc->rxbuf);
516
517 400
        return (hc);
518
}
519
520
/* Bind an address/port for the master CLI (-mcli) */
521
static int
522 0
haproxy_create_mcli(struct haproxy *h)
523
{
524
        int sock;
525
        const char *err;
526
        char buf[128], addr[128], port[128];
527 0
        char vsabuf[vsa_suckaddr_len];
528
        const struct suckaddr *sua;
529
530 0
        sock = VTCP_listen_on(default_listen_addr, NULL, 100, &err);
531 0
        if (err != NULL)
532 0
                vtc_fatal(h->vl,
533 0
                          "Create listen socket failed: %s", err);
534 0
        assert(sock > 0);
535 0
        sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf);
536 0
        AN(sua);
537
538 0
        VTCP_name(sua, addr, sizeof addr, port, sizeof port);
539 0
        bprintf(buf, "%s_mcli", h->name);
540 0
        if (VSA_Get_Proto(sua) == AF_INET)
541 0
                macro_def(h->vl, buf, "sock", "%s:%s", addr, port);
542
        else
543 0
                macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port);
544 0
        macro_def(h->vl, buf, "addr", "%s", addr);
545 0
        macro_def(h->vl, buf, "port", "%s", port);
546
547 0
        return (sock);
548 0
}
549
550
static void
551 800
haproxy_cli_delete(struct haproxy_cli *hc)
552
{
553 800
        CHECK_OBJ_NOTNULL(hc, HAPROXY_CLI_MAGIC);
554 800
        REPLACE(hc->spec, NULL);
555 800
        REPLACE(hc->txbuf, NULL);
556 800
        REPLACE(hc->rxbuf, NULL);
557 800
        FREE_OBJ(hc);
558 800
}
559
560
/**********************************************************************
561
 * Allocate and initialize a haproxy
562
 */
563
564
static struct haproxy *
565 400
haproxy_new(const char *name)
566
{
567
        struct haproxy *h;
568
        struct vsb *vsb;
569
        char buf[PATH_MAX];
570
        int closed_sock;
571
        char addr[128], port[128];
572
        const char *err;
573
        const char *env_args;
574 400
        char vsabuf[vsa_suckaddr_len];
575
        const struct suckaddr *sua;
576
577 400
        ALLOC_OBJ(h, HAPROXY_MAGIC);
578 400
        AN(h);
579 400
        REPLACE(h->name, name);
580
581 400
        h->args = VSB_new_auto();
582 400
        env_args = getenv(HAPROXY_ARGS_ENV_VAR);
583 400
        if (env_args) {
584 0
                VSB_cat(h->args, env_args);
585 0
                VSB_cat(h->args, " ");
586 0
        }
587
588 400
        h->vl = vtc_logopen("%s", name);
589 400
        vtc_log_set_cmd(h->vl, haproxy_cli_cmds);
590 400
        AN(h->vl);
591
592 400
        h->filename = getenv(HAPROXY_PROGRAM_ENV_VAR);
593 400
        if (h->filename == NULL)
594 400
                h->filename = "haproxy";
595
596 400
        bprintf(buf, "${tmpdir}/%s", name);
597 400
        vsb = macro_expand(h->vl, buf);
598 400
        AN(vsb);
599 400
        h->workdir = strdup(VSB_data(vsb));
600 400
        AN(h->workdir);
601 400
        VSB_destroy(&vsb);
602
603 400
        bprintf(buf, "%s/stats.sock", h->workdir);
604 400
        h->cli_fn = strdup(buf);
605 400
        AN(h->cli_fn);
606
607 400
        bprintf(buf, "%s/cfg", h->workdir);
608 400
        h->cfg_fn = strdup(buf);
609 400
        AN(h->cfg_fn);
610
611
        /* Create a new TCP socket to reserve an IP:port and close it asap.
612
         * May be useful to simulate an unreachable server.
613
         */
614 400
        bprintf(h->closed_sock, "%s_closed", h->name);
615 400
        closed_sock = VTCP_listen_on("127.0.0.1:0", NULL, 100, &err);
616 400
        if (err != NULL)
617 0
                vtc_fatal(h->vl,
618 0
                        "Create listen socket failed: %s", err);
619 400
        assert(closed_sock > 0);
620 400
        sua = VSA_getsockname(closed_sock, vsabuf, sizeof vsabuf);
621 400
        AN(sua);
622 400
        VTCP_name(sua, addr, sizeof addr, port, sizeof port);
623 400
        if (VSA_Get_Proto(sua) == AF_INET)
624 400
                macro_def(h->vl, h->closed_sock, "sock", "%s:%s", addr, port);
625
        else
626 0
                macro_def(h->vl, h->closed_sock, "sock", "[%s]:%s", addr, port);
627 400
        macro_def(h->vl, h->closed_sock, "addr", "%s", addr);
628 400
        macro_def(h->vl, h->closed_sock, "port", "%s", port);
629 400
        VTCP_close(&closed_sock);
630
631 400
        h->cli = haproxy_cli_new(h);
632 400
        AN(h->cli);
633
634 400
        h->mcli = haproxy_mcli_new(h);
635 400
        AN(h->mcli);
636
637 400
        bprintf(buf, "rm -rf \"%s\" ; mkdir -p \"%s\"", h->workdir, h->workdir);
638 400
        AZ(system(buf));
639
640 400
        VTAILQ_INIT(&h->envars);
641 400
        VTAILQ_INSERT_TAIL(&haproxies, h, list);
642
643 400
        return (h);
644 400
}
645
646
/**********************************************************************
647
 * Delete a haproxy instance
648
 */
649
650
static void
651 400
haproxy_delete(struct haproxy *h)
652
{
653
        char buf[PATH_MAX];
654
655 400
        CHECK_OBJ_NOTNULL(h, HAPROXY_MAGIC);
656 400
        vtc_logclose(h->vl);
657
658 400
        if (!leave_temp) {
659 400
                bprintf(buf, "rm -rf \"%s\"", h->workdir);
660 400
                AZ(system(buf));
661 400
        }
662
663 400
        free(h->name);
664 400
        free(h->workdir);
665 400
        free(h->cli_fn);
666 400
        free(h->cfg_fn);
667 400
        free(h->pid_fn);
668 400
        VSB_destroy(&h->args);
669 400
        haproxy_cli_delete(h->cli);
670 400
        haproxy_cli_delete(h->mcli);
671
672
        /* XXX: MEMLEAK (?) */
673 400
        FREE_OBJ(h);
674 400
}
675
676
/**********************************************************************
677
 * HAProxy listener
678
 */
679
680
static void *
681 400
haproxy_thread(void *priv)
682
{
683
        struct haproxy *h;
684
685 400
        CAST_OBJ_NOTNULL(h, priv, HAPROXY_MAGIC);
686 400
        (void)vtc_record(h->vl, h->fds[0], h->msgs);
687 400
        h->its_dead_jim = 1;
688 400
        return (NULL);
689
}
690
691
692
/**********************************************************************
693
 * Start a HAProxy instance.
694
 */
695
696
static void
697 400
haproxy_start(struct haproxy *h)
698
{
699
        char buf[PATH_MAX];
700
        struct vsb *vsb;
701
702 400
        vtc_log(h->vl, 2, "%s", __func__);
703
704 400
        AZ(VSB_finish(h->args));
705 800
        vtc_log(h->vl, 4, "opt_worker %d opt_daemon %d opt_check_mode %d opt_mcli %d",
706 400
            h->opt_worker, h->opt_daemon, h->opt_check_mode, h->opt_mcli);
707
708 400
        vsb = VSB_new_auto();
709 400
        AN(vsb);
710
711 400
        VSB_printf(vsb, "exec \"%s\"", h->filename);
712 400
        if (h->opt_check_mode)
713 80
                VSB_cat(vsb, " -c");
714 320
        else if (h->opt_daemon)
715 120
                VSB_cat(vsb, " -D");
716
        else
717 200
                VSB_cat(vsb, " -d");
718
719 400
        if (h->opt_worker) {
720 0
                VSB_cat(vsb, " -W");
721 0
                if (h->opt_mcli) {
722
                        int sock;
723 0
                        sock = haproxy_create_mcli(h);
724 0
                        VSB_printf(vsb, " -S \"fd@%d\"", sock);
725 0
                }
726 0
        }
727
728 400
        VSB_printf(vsb, " %s", VSB_data(h->args));
729
730 400
        VSB_printf(vsb, " -f \"%s\" ", h->cfg_fn);
731
732 400
        if (h->opt_worker || h->opt_daemon) {
733 120
                bprintf(buf, "%s/pid", h->workdir);
734 120
                h->pid_fn = strdup(buf);
735 120
                AN(h->pid_fn);
736 120
                VSB_printf(vsb, " -p \"%s\"", h->pid_fn);
737 120
        }
738
739 400
        AZ(VSB_finish(vsb));
740 400
        vtc_dump(h->vl, 4, "argv", VSB_data(vsb), -1);
741
742 400
        if (h->opt_worker && !h->opt_daemon) {
743
                /*
744
                 * HAProxy master process must exit with status 128 + <signum>
745
                 * if signaled by <signum> signal.
746
                 */
747 0
                h->expect_exit = HAPROXY_EXPECT_EXIT;
748 0
        }
749
750 400
        haproxy_write_conf(h);
751
752 400
        AZ(pipe(&h->fds[0]));
753 400
        vtc_log(h->vl, 4, "XXX %d @%d", h->fds[1], __LINE__);
754 400
        AZ(pipe(&h->fds[2]));
755 400
        h->pid = h->ppid = fork();
756 400
        assert(h->pid >= 0);
757 800
        if (h->pid == 0) {
758 400
                haproxy_build_env(h);
759 400
                haproxy_delete_envars(h);
760 400
                AZ(chdir(h->name));
761 400
                AZ(dup2(h->fds[0], 0));
762 400
                assert(dup2(h->fds[3], 1) == 1);
763 400
                assert(dup2(1, 2) == 2);
764 400
                closefd(&h->fds[0]);
765 400
                closefd(&h->fds[1]);
766 400
                closefd(&h->fds[2]);
767 400
                closefd(&h->fds[3]);
768 400
                AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(vsb), (char*)0));
769 0
                exit(1);
770
        }
771 400
        VSB_destroy(&vsb);
772
773 400
        vtc_log(h->vl, 3, "PID: %ld", (long)h->pid);
774 400
        macro_def(h->vl, h->name, "pid", "%ld", (long)h->pid);
775 400
        macro_def(h->vl, h->name, "name", "%s", h->workdir);
776
777 400
        closefd(&h->fds[0]);
778 400
        closefd(&h->fds[3]);
779 400
        h->fds[0] = h->fds[2];
780 400
        h->fds[2] = h->fds[3] = -1;
781
782 400
        PTOK(pthread_create(&h->tp, NULL, haproxy_thread, h));
783
784 400
        if (h->pid_fn != NULL)
785 120
                haproxy_wait_pidfile(h);
786 400
}
787
788
789
/**********************************************************************
790
 * Wait for a HAProxy instance.
791
 */
792
793
static void
794 400
haproxy_wait(struct haproxy *h)
795
{
796
        void *p;
797
        int i, n, sig;
798
799 400
        vtc_log(h->vl, 2, "Wait");
800
801 400
        if (h->pid < 0)
802 0
                haproxy_start(h);
803
804 0
        if (h->cli->spec)
805 0
                haproxy_cli_run(h->cli);
806
807 0
        if (h->mcli->spec)
808 0
                haproxy_cli_run(h->mcli);
809
810 400
        closefd(&h->fds[1]);
811
812 400
        sig = SIGINT;
813 400
        n = 0;
814 400
        vtc_log(h->vl, 2, "Stop HAproxy pid=%ld", (long)h->pid);
815 9400
        while (h->opt_daemon || (!h->opt_check_mode && !h->its_dead_jim)) {
816 9120
                assert(h->pid > 0);
817 9120
                if (n == 0) {
818 760
                        i = kill(h->pid, sig);
819 760
                        if (i == 0)
820 640
                                h->expect_signal = -sig;
821 1160
                        if (i && errno == ESRCH)
822 120
                                break;
823 1280
                        vtc_log(h->vl, 4,
824 640
                            "Kill(%d)=%d: %s", sig, i, strerror(errno));
825 640
                }
826 840
                usleep(100000);
827 840
                if (++n == 20) {
828 440
                        switch (sig) {
829 320
                        case SIGINT:    sig = SIGTERM ; break;
830 120
                        case SIGTERM:   sig = SIGKILL ; break;
831 0
                        default:        break;
832
                        }
833 440
                        n = 0;
834 440
                }
835
        }
836
837 400
        PTOK(pthread_join(h->tp, &p));
838 400
        AZ(p);
839 400
        closefd(&h->fds[0]);
840 400
        if (!h->opt_daemon) {
841 280
                vtc_wait4(h->vl, h->ppid, h->expect_exit, h->expect_signal, 0);
842 280
                h->ppid = -1;
843 280
        }
844 400
        h->pid = -1;
845 400
}
846
847
#define HAPROXY_BE_FD_STR     "fd@${"
848
#define HAPROXY_BE_FD_STRLEN  strlen(HAPROXY_BE_FD_STR)
849
850
static int
851 400
haproxy_build_backends(struct haproxy *h, const char *vsb_data)
852
{
853
        char *s, *p, *q;
854
855 400
        s = strdup(vsb_data);
856 400
        if (!s)
857 0
                return (-1);
858
859 400
        p = s;
860 1080
        while (1) {
861
                int sock;
862
                char buf[128], addr[128], port[128];
863
                const char *err;
864 1080
                char vsabuf[vsa_suckaddr_len];
865
                const struct suckaddr *sua;
866
867 1080
                p = strstr(p, HAPROXY_BE_FD_STR);
868 1080
                if (!p)
869 400
                        break;
870
871 680
                q = p += HAPROXY_BE_FD_STRLEN;
872 2800
                while (*q && *q != '}')
873 2120
                        q++;
874 680
                if (*q != '}')
875 0
                        break;
876
877 680
                *q++ = '\0';
878 680
                sock = VTCP_listen_on("127.0.0.1:0", NULL, 100, &err);
879 680
                if (err != NULL)
880 0
                        vtc_fatal(h->vl,
881 0
                            "Create listen socket failed: %s", err);
882 680
                assert(sock > 0);
883 680
                sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf);
884 680
                AN(sua);
885
886 680
                VTCP_name(sua, addr, sizeof addr, port, sizeof port);
887 680
                bprintf(buf, "%s_%s", h->name, p);
888 680
                if (VSA_Get_Proto(sua) == AF_INET)
889 680
                        macro_def(h->vl, buf, "sock", "%s:%s", addr, port);
890
                else
891 0
                        macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port);
892 680
                macro_def(h->vl, buf, "addr", "%s", addr);
893 680
                macro_def(h->vl, buf, "port", "%s", port);
894
895 680
                bprintf(buf, "%d", sock);
896 680
                vtc_log(h->vl, 4, "setenv(%s, %s)", p, buf);
897 680
                haproxy_add_envar(h, p, buf);
898 680
                p = q;
899 1080
        }
900 400
        free(s);
901 400
        return (0);
902 400
}
903
904
static void
905 80
haproxy_check_conf(struct haproxy *h, const char *expect)
906
{
907
908 80
        h->msgs = VSB_new_auto();
909 80
        AN(h->msgs);
910 80
        h->opt_check_mode = 1;
911 80
        haproxy_start(h);
912 80
        haproxy_wait(h);
913 80
        AZ(VSB_finish(h->msgs));
914 80
        if (strstr(VSB_data(h->msgs), expect) == NULL)
915 0
                vtc_fatal(h->vl, "Did not find expected string '%s'", expect);
916 80
        vtc_log(h->vl, 2, "Found expected '%s'", expect);
917 80
        VSB_destroy(&h->msgs);
918 80
}
919
920
/**********************************************************************
921
 * Write a configuration for <h> HAProxy instance.
922
 */
923
924
static void
925 400
haproxy_store_conf(struct haproxy *h, const char *cfg, int auto_be)
926
{
927
        struct vsb *vsb, *vsb2;
928
929 400
        vsb = VSB_new_auto();
930 400
        AN(vsb);
931
932 400
        vsb2 = VSB_new_auto();
933 400
        AN(vsb2);
934
935 800
        VSB_printf(vsb, "    global\n\tstats socket \"%s\" "
936 400
                   "level admin mode 600\n", h->cli_fn);
937 400
        VSB_cat(vsb, "    stats socket \"fd@${cli}\" level admin\n");
938 400
        AZ(VSB_cat(vsb, cfg));
939
940 400
        if (auto_be)
941 40
                cmd_server_gen_haproxy_conf(vsb);
942
943 400
        AZ(VSB_finish(vsb));
944
945 400
        AZ(haproxy_build_backends(h, VSB_data(vsb)));
946
947 400
        h->cfg_vsb = macro_expand(h->vl, VSB_data(vsb));
948 400
        AN(h->cfg_vsb);
949
950 400
        VSB_destroy(&vsb2);
951 400
        VSB_destroy(&vsb);
952 400
}
953
954
static void
955 400
haproxy_write_conf(struct haproxy *h)
956
{
957
        struct vsb *vsb;
958
959 400
        vsb = macro_expand(h->vl, VSB_data(h->cfg_vsb));
960 400
        AN(vsb);
961 400
        assert(VSB_len(vsb) >= 0);
962
963 400
        vtc_dump(h->vl, 4, "conf", VSB_data(vsb), VSB_len(vsb));
964 1200
        if (VFIL_writefile(h->workdir, h->cfg_fn,
965 800
            VSB_data(vsb), VSB_len(vsb)) != 0)
966 0
                vtc_fatal(h->vl,
967
                    "failed to write haproxy configuration file: %s (%d)",
968 0
                    strerror(errno), errno);
969
970 400
        VSB_destroy(&vsb);
971 400
}
972
973
/* SECTION: haproxy haproxy
974
 *
975
 * Define and interact with haproxy instances.
976
 *
977
 * To define a haproxy server, you'll use this syntax::
978
 *
979
 *      haproxy hNAME -conf-OK CONFIG
980
 *      haproxy hNAME -conf-BAD ERROR CONFIG
981
 *      haproxy hNAME [-D] [-W] [-arg STRING] [-conf[+vcl] STRING]
982
 *
983
 * The first ``haproxy hNAME`` invocation will start the haproxy master
984
 * process in the background, waiting for the ``-start`` switch to actually
985
 * start the child.
986
 *
987
 * Arguments:
988
 *
989
 * hNAME
990
 *         Identify the HAProxy server with a string, it must starts with 'h'.
991
 *
992
 * \-conf-OK CONFIG
993
 *         Run haproxy in '-c' mode to check config is OK
994
 *         stdout/stderr should contain 'Configuration file is valid'
995
 *         The exit code should be 0.
996
 *
997
 * \-conf-BAD ERROR CONFIG
998
 *         Run haproxy in '-c' mode to check config is BAD.
999
 *         "ERROR" should be part of the diagnostics on stdout/stderr.
1000
 *         The exit code should be 1.
1001
 *
1002
 * \-D
1003
 *         Run HAproxy in daemon mode.  If not given '-d' mode used.
1004
 *
1005
 * \-W
1006
 *         Enable HAproxy in Worker mode.
1007
 *
1008
 * \-S
1009
 *         Enable HAproxy Master CLI in Worker mode
1010
 *
1011
 * \-arg STRING
1012
 *         Pass an argument to haproxy, for example "-h simple_list".
1013
 *
1014
 * \-cli STRING
1015
 *         Specify the spec to be run by the command line interface (CLI).
1016
 *
1017
 * \-mcli STRING
1018
 *         Specify the spec to be run by the command line interface (CLI)
1019
 *         of the Master process.
1020
 *
1021
 * \-conf STRING
1022
 *         Specify the configuration to be loaded by this HAProxy instance.
1023
 *
1024
 * \-conf+backend STRING
1025
 *         Specify the configuration to be loaded by this HAProxy instance,
1026
 *         all server instances will be automatically appended
1027
 *
1028
 * \-start
1029
 *         Start this HAProxy instance.
1030
 *
1031
 * \-wait
1032
 *         Stop this HAProxy instance.
1033
 *
1034
 * \-expectexit NUMBER
1035
 *         Expect haproxy to exit(3) with this value
1036
 *
1037
 */
1038
1039
void
1040 40600
cmd_haproxy(CMD_ARGS)
1041
{
1042
        struct haproxy *h, *h2;
1043
1044 40600
        (void)priv;
1045
1046 40600
        if (av == NULL) {
1047
                /* Reset and free */
1048 40560
                VTAILQ_FOREACH_SAFE(h, &haproxies, list, h2) {
1049 800
                        vtc_log(h->vl, 2,
1050
                            "Reset and free %s haproxy %ld",
1051 400
                            h->name, (long)h->pid);
1052 400
                        if (h->pid >= 0)
1053 280
                                haproxy_wait(h);
1054 400
                        VTAILQ_REMOVE(&haproxies, h, list);
1055 400
                        haproxy_delete(h);
1056 400
                }
1057 40160
                return;
1058
        }
1059
1060 440
        AZ(strcmp(av[0], "haproxy"));
1061 440
        av++;
1062
1063 440
        VTC_CHECK_NAME(vl, av[0], "haproxy", 'h');
1064 520
        VTAILQ_FOREACH(h, &haproxies, list)
1065 120
                if (!strcmp(h->name, av[0]))
1066 40
                        break;
1067 400
        if (h == NULL)
1068 400
                h = haproxy_new(av[0]);
1069 440
        av++;
1070
1071 1360
        for (; *av != NULL; av++) {
1072 920
                if (vtc_error)
1073 0
                        break;
1074
1075 920
                if (!strcmp(*av, "-conf-OK")) {
1076 40
                        AN(av[1]);
1077 40
                        haproxy_store_conf(h, av[1], 0);
1078 40
                        h->expect_exit = 0;
1079 40
                        haproxy_check_conf(h, "");
1080 40
                        av++;
1081 40
                        continue;
1082
                }
1083 880
                if (!strcmp(*av, "-conf-BAD")) {
1084 40
                        AN(av[1]);
1085 40
                        AN(av[2]);
1086 40
                        haproxy_store_conf(h, av[2], 0);
1087 40
                        h->expect_exit = 1;
1088 40
                        haproxy_check_conf(h, av[1]);
1089 40
                        av += 2;
1090 40
                        continue;
1091
                }
1092
1093 840
                if (!strcmp(*av, HAPROXY_OPT_DAEMON)) {
1094 120
                        h->opt_daemon = 1;
1095 120
                        continue;
1096
                }
1097 720
                if (!strcmp(*av, HAPROXY_OPT_WORKER)) {
1098 0
                        h->opt_worker = 1;
1099 0
                        continue;
1100
                }
1101 720
                if (!strcmp(*av, HAPROXY_OPT_MCLI)) {
1102 0
                        h->opt_mcli = 1;
1103 0
                        continue;
1104
                }
1105 720
                if (!strcmp(*av, "-arg")) {
1106 0
                        AN(av[1]);
1107 0
                        AZ(h->pid);
1108 0
                        VSB_cat(h->args, " ");
1109 0
                        VSB_cat(h->args, av[1]);
1110 0
                        av++;
1111 0
                        continue;
1112
                }
1113
1114 720
                if (!strcmp(*av, "-cli")) {
1115 40
                        REPLACE(h->cli->spec, av[1]);
1116 40
                        if (h->tp)
1117 40
                                haproxy_cli_run(h->cli);
1118 40
                        av++;
1119 40
                        continue;
1120
                }
1121
1122 680
                if (!strcmp(*av, "-mcli")) {
1123 0
                        REPLACE(h->mcli->spec, av[1]);
1124 0
                        if (h->tp)
1125 0
                                haproxy_cli_run(h->mcli);
1126 0
                        av++;
1127 0
                        continue;
1128
                }
1129
1130 680
                if (!strcmp(*av, "-conf")) {
1131 280
                        AN(av[1]);
1132 280
                        haproxy_store_conf(h, av[1], 0);
1133 280
                        av++;
1134 280
                        continue;
1135
                }
1136 400
                if (!strcmp(*av, "-conf+backend")) {
1137 40
                        AN(av[1]);
1138 40
                        haproxy_store_conf(h, av[1], 1);
1139 40
                        av++;
1140 40
                        continue;
1141
                }
1142
1143 360
                if (!strcmp(*av, "-expectexit")) {
1144 0
                        h->expect_exit = strtoul(av[1], NULL, 0);
1145 0
                        av++;
1146 0
                        continue;
1147
                }
1148 360
                if (!strcmp(*av, "-start")) {
1149 320
                        haproxy_start(h);
1150 320
                        continue;
1151
                }
1152 40
                if (!strcmp(*av, "-wait")) {
1153 40
                        haproxy_wait(h);
1154 40
                        continue;
1155
                }
1156 0
                vtc_fatal(h->vl, "Unknown haproxy argument: %s", *av);
1157
        }
1158 40600
}