New upstream version 1.4

1 files changed, 262 insertions, 69 deletions

diff --git a/tests/runtests.c b/tests/runtests.c
index 42a73ea..af15a5c 100644
--- a/tests/runtests.c
+++ b/tests/runtests.c
@@ -1,6 +1,37 @@
 /*
  * Run a set of tests, reporting results.
  *
+ * Test suite driver that runs a set of tests implementing a subset of the
+ * Test Anything Protocol (TAP) and reports the results.
+ *
+ * Any bug reports, bug fixes, and improvements are very much welcome and
+ * should be sent to the e-mail address below.  This program is part of C TAP
+ * Harness <https://www.eyrie.org/~eagle/software/c-tap-harness/>.
+ *
+ * Copyright 2000-2001, 2004, 2006-2018 Russ Allbery <eagle@eyrie.org>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+/*
  * Usage:
  *
  *      runtests [-hv] [-b <build-dir>] [-s <source-dir>] -l <test-list>
@@ -8,9 +39,10 @@
  *      runtests -o [-h] [-b <build-dir>] [-s <source-dir>] <test>
  *
  * In the first case, expects a list of executables located in the given file,
- * one line per executable.  For each one, runs it as part of a test suite,
- * reporting results.  In the second case, use the same infrastructure, but
- * run only the tests listed on the command line.
+ * one line per executable, possibly followed by a space-separated list of
+ * options.  For each one, runs it as part of a test suite, reporting results.
+ * In the second case, use the same infrastructure, but run only the tests
+ * listed on the command line.
  *
  * Test output should start with a line containing the number of tests
  * (numbered from 1 to this number), optionally preceded by "1..", although
@@ -48,41 +80,16 @@
  * output.  This is intended for use with failing tests so that the person
  * running the test suite can get more details about what failed.
  *
- * If built with the C preprocessor symbols SOURCE and BUILD defined, C TAP
- * Harness will export those values in the environment so that tests can find
- * the source and build directory and will look for tests under both
- * directories.  These paths can also be set with the -b and -s command-line
- * options, which will override anything set at build time.
+ * If built with the C preprocessor symbols C_TAP_SOURCE and C_TAP_BUILD
+ * defined, C TAP Harness will export those values in the environment so that
+ * tests can find the source and build directory and will look for tests under
+ * both directories.  These paths can also be set with the -b and -s
+ * command-line options, which will override anything set at build time.
  *
  * If the -v option is given, or the C_TAP_VERBOSE environment variable is set,
  * display the full output of each test as it runs rather than showing a
  * summary of the results of each test.
- *
- * Any bug reports, bug fixes, and improvements are very much welcome and
- * should be sent to the e-mail address below.  This program is part of C TAP
- * Harness <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
- *
- * Copyright 2000, 2001, 2004, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013,
- *     2014, 2015 Russ Allbery <eagle@eyrie.org>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
-*/
+ */
 
 /* Required for fdopen(), getopt(), and putenv(). */
 #if defined(__STRICT_ANSI__) || defined(PEDANTIC)
@@ -135,15 +142,17 @@
 
 /*
  * The source and build versions of the tests directory.  This is used to set
- * the SOURCE and BUILD environment variables and find test programs, if set.
- * Normally, this should be set as part of the build process to the test
- * subdirectories of $(abs_top_srcdir) and $(abs_top_builddir) respectively.
+ * the C_TAP_SOURCE and C_TAP_BUILD environment variables (and the SOURCE and
+ * BUILD environment variables set for backward compatibility) and find test
+ * programs, if set.  Normally, this should be set as part of the build
+ * process to the test subdirectories of $(abs_top_srcdir) and
+ * $(abs_top_builddir) respectively.
  */
-#ifndef SOURCE
-# define SOURCE NULL
+#ifndef C_TAP_SOURCE
+# define C_TAP_SOURCE NULL
 #endif
-#ifndef BUILD
-# define BUILD NULL
+#ifndef C_TAP_BUILD
+# define C_TAP_BUILD NULL
 #endif
 
 /* Test status codes. */
@@ -177,7 +186,7 @@ enum plan_status {
 /* Structure to hold data for a set of tests. */
 struct testset {
     char *file;                 /* The file name of the test. */
-    char *path;                 /* The path to the test program. */
+    char **command;             /* The argv vector to run the command. */
     enum plan_status plan;      /* The status of our plan. */
     unsigned long count;        /* Expected count of tests. */
     unsigned long current;      /* The last seen test number. */
@@ -188,7 +197,7 @@ struct testset {
     unsigned long allocated;    /* The size of the results table. */
     enum test_status *results;  /* Table of results by test number. */
     unsigned int aborted;       /* Whether the set was aborted. */
-    int reported;               /* Whether the results were reported. */
+    unsigned int reported;      /* Whether the results were reported. */
     int status;                 /* The exit status of the test. */
     unsigned int all_skipped;   /* Whether all tests were skipped. */
     char *reason;               /* Why all tests were skipped. */
@@ -240,6 +249,7 @@ Failed Set                 Fail/Total (%) Skip Stat  Failing Tests\n\
 #define xcalloc(n, size)      x_calloc((n), (size), __FILE__, __LINE__)
 #define xmalloc(size)         x_malloc((size), __FILE__, __LINE__)
 #define xstrdup(p)            x_strdup((p), __FILE__, __LINE__)
+#define xstrndup(p, size)     x_strndup((p), (size), __FILE__, __LINE__)
 #define xreallocarray(p, n, size) \
     x_reallocarray((p), (n), (size), __FILE__, __LINE__)
 
@@ -280,6 +290,8 @@ Failed Set                 Fail/Total (%) Skip Stat  Failing Tests\n\
 #endif
 
 /* Declare internal functions that benefit from compiler attributes. */
+static void die(const char *, ...)
+    __attribute__((__nonnull__, __noreturn__, __format__(printf, 1, 2)));
 static void sysdie(const char *, ...)
     __attribute__((__nonnull__, __noreturn__, __format__(printf, 1, 2)));
 static void *x_calloc(size_t, size_t, const char *, int)
@@ -290,6 +302,26 @@ static void *x_reallocarray(void *, size_t, size_t, const char *, int)
     __attribute__((__alloc_size__(2, 3), __malloc__, __nonnull__(4)));
 static char *x_strdup(const char *, const char *, int)
     __attribute__((__malloc__, __nonnull__));
+static char *x_strndup(const char *, size_t, const char *, int)
+    __attribute__((__malloc__, __nonnull__));
+
+
+/*
+ * Report a fatal error and exit.
+ */
+static void
+die(const char *format, ...)
+{
+    va_list args;
+
+    fflush(stdout);
+    fprintf(stderr, "runtests: ");
+    va_start(args, format);
+    vfprintf(stderr, format, args);
+    va_end(args);
+    fprintf(stderr, "\n");
+    exit(1);
+}
 
 
 /*
@@ -392,6 +424,35 @@ x_strdup(const char *s, const char *file, int line)
 
 
 /*
+ * Copy the first n characters of a string, reporting a fatal error and
+ * existing on failure.
+ *
+ * Avoid using the system strndup function since it may not exist (on Mac OS
+ * X, for example), and there's no need to introduce another portability
+ * requirement.
+ */
+char *
+x_strndup(const char *s, size_t size, const char *file, int line)
+{
+    const char *p;
+    size_t len;
+    char *copy;
+
+    /* Don't assume that the source string is nul-terminated. */
+    for (p = s; (size_t) (p - s) < size && *p != '\0'; p++)
+        ;
+    len = (size_t) (p - s);
+    copy = malloc(len + 1);
+    if (copy == NULL)
+        sysdie("failed to strndup %lu bytes at %s line %d",
+               (unsigned long) len, file, line);
+    memcpy(copy, s, len);
+    copy[len] = '\0';
+    return copy;
+}
+
+
+/*
  * Form a new string by concatenating multiple strings.  The arguments must be
  * terminated by (const char *) 0.
  *
@@ -447,7 +508,7 @@ concat(const char *first, ...)
 static double
 tv_seconds(const struct timeval *tv)
 {
-    return difftime(tv->tv_sec, 0) + tv->tv_usec * 1e-6;
+    return difftime(tv->tv_sec, 0) + (double) tv->tv_usec * 1e-6;
 }
 
 
@@ -485,12 +546,25 @@ skip_whitespace(const char *p)
 
 
 /*
+ * Given a pointer to a string, skip any non-whitespace characters and return
+ * a pointer to the first whitespace character, or to the end of the string.
+ */
+static const char *
+skip_non_whitespace(const char *p)
+{
+    while (*p != '\0' && !isspace((unsigned char)(*p)))
+        p++;
+    return p;
+}
+
+
+/*
  * Start a program, connecting its stdout to a pipe on our end and its stderr
  * to /dev/null, and storing the file descriptor to read from in the two
  * argument.  Returns the PID of the new process.  Errors are fatal.
  */
 static pid_t
-test_start(const char *path, int *fd)
+test_start(char *const *command, int *fd)
 {
     int fds[2], infd, errfd;
     pid_t child;
@@ -541,8 +615,9 @@ test_start(const char *path, int *fd)
         }
 
         /* Now, exec our process. */
-        if (execl(path, path, (char *) 0) == -1)
+        if (execv(command[0], command) == -1)
             _exit(CHILDERR_EXEC);
+        break;
 
     /* In parent.  Close the extra file descriptor. */
     default:
@@ -1035,7 +1110,7 @@ test_run(struct testset *ts, enum test_verbose verbose)
     char buffer[BUFSIZ];
 
     /* Run the test program. */
-    testpid = test_start(ts->path, &outfd);
+    testpid = test_start(ts->command, &outfd);
     output = fdopen(outfd, "r");
     if (!output) {
         puts("ABORTED");
@@ -1097,6 +1172,7 @@ test_fail_summary(const struct testlist *fails)
     struct testset *ts;
     unsigned int chars;
     unsigned long i, first, last, total;
+    double failed;
 
     puts(header);
 
@@ -1105,8 +1181,9 @@ test_fail_summary(const struct testlist *fails)
     for (; fails; fails = fails->next) {
         ts = fails->ts;
         total = ts->count - ts->skipped;
+        failed = (double) ts->failed;
         printf("%-26.26s %4lu/%-4lu %3.0f%% %4lu ", ts->file, ts->failed,
-               total, total ? (ts->failed * 100.0) / total : 0,
+               total, total ? (failed * 100.0) / (double) total : 0,
                ts->skipped);
         if (WIFEXITED(ts->status))
             printf("%4d  ", WEXITSTATUS(ts->status));
@@ -1203,19 +1280,111 @@ find_test(const char *name, const char *source, const char *build)
 
 
 /*
+ * Parse a single line of a test list and store the test name and command to
+ * execute it in the given testset struct.
+ *
+ * Normally, each line is just the name of the test, which is located in the
+ * test directory and turned into a command to run.  However, each line may
+ * have whitespace-separated options, which change the command that's run.
+ * Current supported options are:
+ *
+ * valgrind
+ *     Run the test under valgrind if C_TAP_VALGRIND is set.  The contents
+ *     of that environment variable are taken as the valgrind command (with
+ *     options) to run.  The command is parsed with a simple split on
+ *     whitespace and no quoting is supported.
+ *
+ * libtool
+ *     If running under valgrind, use libtool to invoke valgrind.  This avoids
+ *     running valgrind on the wrapper shell script generated by libtool.  If
+ *     set, C_TAP_LIBTOOL must be set to the full path to the libtool program
+ *     to use to run valgrind and thus the test.  Ignored if the test isn't
+ *     being run under valgrind.
+ */
+static void
+parse_test_list_line(const char *line, struct testset *ts, const char *source,
+                     const char *build)
+{
+    const char *p, *end, *option, *libtool;
+    const char *valgrind = NULL;
+    unsigned int use_libtool = 0;
+    unsigned int use_valgrind = 0;
+    size_t len, i;
+
+    /* Determine the name of the test. */
+    p = skip_non_whitespace(line);
+    ts->file = xstrndup(line, p - line);
+
+    /* Check if any test options are set. */
+    p = skip_whitespace(p);
+    while (*p != '\0') {
+        end = skip_non_whitespace(p);
+        if (strncmp(p, "libtool", end - p) == 0) {
+            use_libtool = 1;
+            p = end;
+        } else if (strncmp(p, "valgrind", end - p) == 0) {
+            valgrind = getenv("C_TAP_VALGRIND");
+            use_valgrind = (valgrind != NULL);
+            p = end;
+        } else {
+            option = xstrndup(p, end - p);
+            die("unknown test list option %s", option);
+        }
+        p = skip_whitespace(end);
+    }
+
+    /* Construct the argv to run the test.  First, find the length. */
+    len = 1;
+    if (use_valgrind && valgrind != NULL) {
+        p = skip_whitespace(valgrind);
+        while (*p != '\0') {
+            len++;
+            p = skip_whitespace(skip_non_whitespace(p));
+        }
+        if (use_libtool)
+            len += 2;
+    }
+
+    /* Now, build the command. */
+    ts->command = xcalloc(len + 1, sizeof(char *));
+    i = 0;
+    if (use_valgrind && valgrind != NULL) {
+        if (use_libtool) {
+            libtool = getenv("C_TAP_LIBTOOL");
+            if (libtool == NULL)
+                die("valgrind with libtool requested, but C_TAP_LIBTOOL is not"
+                    " set");
+            ts->command[i++] = xstrdup(libtool);
+            ts->command[i++] = xstrdup("--mode=execute");
+        }
+        p = skip_whitespace(valgrind);
+        while (*p != '\0') {
+            end = skip_non_whitespace(p);
+            ts->command[i++] = xstrndup(p, end - p);
+            p = skip_whitespace(end);
+        }
+    }
+    if (i != len - 1)
+        die("internal error while constructing command line");
+    ts->command[i++] = find_test(ts->file, source, build);
+    ts->command[i] = NULL;
+}
+
+
+/*
  * Read a list of tests from a file, returning the list of tests as a struct
  * testlist, or NULL if there were no tests (such as a file containing only
  * comments).  Reports an error to standard error and exits if the list of
  * tests cannot be read.
  */
 static struct testlist *
-read_test_list(const char *filename)
+read_test_list(const char *filename, const char *source, const char *build)
 {
     FILE *file;
     unsigned int line;
     size_t length;
     char buffer[BUFSIZ];
-    const char *testname;
+    const char *start;
     struct testlist *listhead, *current;
 
     /* Create the initial container list that will hold our results. */
@@ -1240,10 +1409,10 @@ read_test_list(const char *filename)
         buffer[length] = '\0';
 
         /* Skip comments, leading spaces, and blank lines. */
-        testname = skip_whitespace(buffer);
-        if (strlen(testname) == 0)
+        start = skip_whitespace(buffer);
+        if (strlen(start) == 0)
             continue;
-        if (testname[0] == '#')
+        if (start[0] == '#')
             continue;
 
         /* Allocate the new testset structure. */
@@ -1255,7 +1424,9 @@ read_test_list(const char *filename)
         }
         current->ts = xcalloc(1, sizeof(struct testset));
         current->ts->plan = PLAN_INIT;
-        current->ts->file = xstrdup(testname);
+
+        /* Parse the line and store the results in the testset struct. */
+        parse_test_list_line(start, current->ts, source, build);
     }
     fclose(file);
 
@@ -1277,7 +1448,7 @@ read_test_list(const char *filename)
  * freeing.
  */
 static struct testlist *
-build_test_list(char *argv[], int argc)
+build_test_list(char *argv[], int argc, const char *source, const char *build)
 {
     int i;
     struct testlist *listhead, *current;
@@ -1297,6 +1468,9 @@ build_test_list(char *argv[], int argc)
         current->ts = xcalloc(1, sizeof(struct testset));
         current->ts->plan = PLAN_INIT;
         current->ts->file = xstrdup(argv[i]);
+        current->ts->command = xcalloc(2, sizeof(char *));
+        current->ts->command[0] = find_test(current->ts->file, source, build);
+        current->ts->command[1] = NULL;
     }
 
     /* If there were no tests, current is still NULL. */
@@ -1314,8 +1488,12 @@ build_test_list(char *argv[], int argc)
 static void
 free_testset(struct testset *ts)
 {
+    size_t i;
+
     free(ts->file);
-    free(ts->path);
+    for (i = 0; ts->command[i] != NULL; i++)
+        free(ts->command[i]);
+    free(ts->command);
     free(ts->results);
     free(ts->reason);
     free(ts);
@@ -1330,8 +1508,7 @@ free_testset(struct testset *ts)
  * frees the test list that's passed in.
  */
 static int
-test_batch(struct testlist *tests, const char *source, const char *build,
-           enum test_verbose verbose)
+test_batch(struct testlist *tests, enum test_verbose verbose)
 {
     size_t length, i;
     size_t longest = 0;
@@ -1382,7 +1559,6 @@ test_batch(struct testlist *tests, const char *source, const char *build,
             fflush(stdout);
 
         /* Run the test. */
-        ts->path = find_test(ts->file, source, build);
         succeeded = test_run(ts, verbose);
         fflush(stdout);
         if (verbose)
@@ -1446,7 +1622,7 @@ test_batch(struct testlist *tests, const char *source, const char *build,
         fputs("All tests successful", stdout);
     else
         printf("Failed %lu/%lu tests, %.2f%% okay", failed, total,
-               (total - failed) * 100.0 / total);
+               (double) (total - failed) * 100.0 / (double) total);
     if (skipped != 0) {
         if (skipped == 1)
             printf(", %lu test skipped", skipped);
@@ -1479,8 +1655,9 @@ test_single(const char *program, const char *source, const char *build)
 
 
 /*
- * Main routine.  Set the SOURCE and BUILD environment variables and then,
- * given a file listing tests, run each test listed.
+ * Main routine.  Set the C_TAP_SOURCE, C_TAP_BUILD, SOURCE, and BUILD
+ * environment variables and then, given a file listing tests, run each test
+ * listed.
  */
 int
 main(int argc, char *argv[])
@@ -1489,13 +1666,15 @@ main(int argc, char *argv[])
     int status = 0;
     int single = 0;
     enum test_verbose verbose = CONCISE;
+    char *c_tap_source_env = NULL;
+    char *c_tap_build_env = NULL;
     char *source_env = NULL;
     char *build_env = NULL;
     const char *program;
     const char *shortlist;
     const char *list = NULL;
-    const char *source = SOURCE;
-    const char *build = BUILD;
+    const char *source = C_TAP_SOURCE;
+    const char *build = C_TAP_BUILD;
     struct testlist *tests;
 
     program = argv[0];
@@ -1537,13 +1716,23 @@ main(int argc, char *argv[])
     if (getenv("C_TAP_VERBOSE") != NULL)
         verbose = VERBOSE;
 
-    /* Set SOURCE and BUILD environment variables. */
+    /*
+     * Set C_TAP_SOURCE and C_TAP_BUILD environment variables.  Also set
+     * SOURCE and BUILD for backward compatibility, although we're trying to
+     * migrate to the ones with a C_TAP_* prefix.
+     */
     if (source != NULL) {
+        c_tap_source_env = concat("C_TAP_SOURCE=", source, (const char *) 0);
+        if (putenv(c_tap_source_env) != 0)
+            sysdie("cannot set C_TAP_SOURCE in the environment");
         source_env = concat("SOURCE=", source, (const char *) 0);
         if (putenv(source_env) != 0)
             sysdie("cannot set SOURCE in the environment");
     }
     if (build != NULL) {
+        c_tap_build_env = concat("C_TAP_BUILD=", build, (const char *) 0);
+        if (putenv(c_tap_build_env) != 0)
+            sysdie("cannot set C_TAP_BUILD in the environment");
         build_env = concat("BUILD=", build, (const char *) 0);
         if (putenv(build_env) != 0)
             sysdie("cannot set BUILD in the environment");
@@ -1559,20 +1748,24 @@ main(int argc, char *argv[])
         else
             shortlist++;
         printf(banner, shortlist);
-        tests = read_test_list(list);
-        status = test_batch(tests, source, build, verbose) ? 0 : 1;
+        tests = read_test_list(list, source, build);
+        status = test_batch(tests, verbose) ? 0 : 1;
     } else {
-        tests = build_test_list(argv, argc);
-        status = test_batch(tests, source, build, verbose) ? 0 : 1;
+        tests = build_test_list(argv, argc, source, build);
+        status = test_batch(tests, verbose) ? 0 : 1;
     }
 
     /* For valgrind cleanliness, free all our memory. */
     if (source_env != NULL) {
+        putenv((char *) "C_TAP_SOURCE=");
         putenv((char *) "SOURCE=");
+        free(c_tap_source_env);
         free(source_env);
     }
     if (build_env != NULL) {
+        putenv((char *) "C_TAP_BUILD=");
         putenv((char *) "BUILD=");
+        free(c_tap_build_env);
         free(build_env);
     }
     exit(status);