tests/lib/tap.c

   1 /*-
   2  * Copyright (c) 2004 Nik Clayton
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <ctype.h>
  28 #include <stdarg.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31
  32 #include "tap.h"
  33
  34 static int no_plan = 0;
  35 static int skip_all = 0;
  36 static int have_plan = 0;
  37 static unsigned int test_count = 0; /* Number of tests that have been run */
  38 static unsigned int e_tests = 0; /* Expected number of tests to run */
  39 static unsigned int failures = 0; /* Number of tests that failed */
  40 static char *todo_msg = NULL;
  41 static char *todo_msg_fixed = "libtap malloc issue";
  42 static int todo = 0;
  43 static int test_died = 0;
  44
  45 /* Encapsulate the pthread code in a conditional.  In the absence of
  46    libpthread the code does nothing */
  47 #ifdef HAVE_LIBPTHREAD
  48 #include <pthread.h>
  49 static pthread_mutex_t M = PTHREAD_MUTEX_INITIALIZER;
  50 # define LOCK pthread_mutex_lock(&M);
  51 # define UNLOCK pthread_mutex_unlock(&M);
  52 #else
  53 # define LOCK
  54 # define UNLOCK
  55 #endif
  56
  57 static void _expected_tests(unsigned int);
  58 static void _tap_init(void);
  59 static void _cleanup(void);
  60
  61 /*
  62  * Generate a test result.
  63  *
  64  * ok -- boolean, indicates whether or not the test passed.
  65  * test_name -- the name of the test, may be NULL
  66  * test_comment -- a comment to print afterwards, may be NULL
  67  */
  68 unsigned int
  69 _gen_result(int ok, const char *func, char *file, unsigned int line,
  70             char *test_name, ...)
  71 {
  72         va_list ap;
  73         char *local_test_name = NULL;
  74         char *c;
  75         int name_is_digits;
  76
  77         LOCK;
  78
  79         test_count++;
  80
  81         /* Start by taking the test name and performing any printf()
  82            expansions on it */
  83         if(test_name != NULL) {
  84                 va_start(ap, test_name);
  85                 vasprintf(&local_test_name, test_name, ap);
  86                 va_end(ap);
  87
  88                 /* Make sure the test name contains more than digits
  89                    and spaces.  Emit an error message and exit if it
  90                    does */
  91                 if(local_test_name) {
  92                         name_is_digits = 1;
  93                         for(c = local_test_name; *c != '\0'; c++) {
  94                                 if(!isdigit(*c) && !isspace(*c)) {
  95                                         name_is_digits = 0;
  96                                         break;
  97                                 }
  98                         }
  99
 100                         if(name_is_digits) {
 101                                 diag("    You named your test '%s'.  You shouldn't use numbers for your test names.", local_test_name);
 102                                 diag("    Very confusing.");
 103                         }
 104                 }
 105         }
 106
 107         if(!ok) {
 108                 printf("not ");
 109                 failures++;
 110         }
 111
 112         printf("ok %d", test_count);
 113
 114         if(test_name != NULL) {
 115                 printf(" - ");
 116
 117                 /* Print the test name, escaping any '#' characters it
 118                    might contain */
 119                 if(local_test_name != NULL) {
 120                         flockfile(stdout);
 121                         for(c = local_test_name; *c != '\0'; c++) {
 122                                 if(*c == '#')
 123                                         fputc('\\', stdout);
 124                                 fputc((int)*c, stdout);
 125                         }
 126                         funlockfile(stdout);
 127                 } else {        /* vasprintf() failed, use a fixed message */
 128                         printf("%s", todo_msg_fixed);
 129                 }
 130         }
 131
 132         /* If we're in a todo_start() block then flag the test as being
 133            TODO.  todo_msg should contain the message to print at this
 134            point.  If it's NULL then asprintf() failed, and we should
 135            use the fixed message.
 136
 137            This is not counted as a failure, so decrement the counter if
 138            the test failed. */
 139         if(todo) {
 140                 printf(" # TODO %s", todo_msg ? todo_msg : todo_msg_fixed);
 141                 if(!ok)
 142                         failures--;
 143         }
 144
 145         printf("\n");
 146
 147         if(!ok) {
 148                 if(getenv("HARNESS_ACTIVE") != NULL)
 149                         fputs("\n", stderr);
 150
 151                 diag("    Failed %stest (%s:%s() at line %d)",
 152                      todo ? "(TODO) " : "", file, func, line);
 153         }
 154         free(local_test_name);
 155
 156         UNLOCK;
 157
 158         /* We only care (when testing) that ok is positive, but here we
 159            specifically only want to return 1 or 0 */
 160         return ok ? 1 : 0;
 161 }
 162
 163 /*
 164  * Initialise the TAP library.  Will only do so once, however many times it's
 165  * called.
 166  */
 167 void
 168 _tap_init(void)
 169 {
 170         static int run_once = 0;
 171
 172         if(!run_once) {
 173                 atexit(_cleanup);
 174
 175                 /* stdout needs to be unbuffered so that the output appears
 176                    in the same place relative to stderr output as it does
 177                    with Test::Harness */
 178                 setbuf(stdout, 0);
 179                 run_once = 1;
 180         }
 181 }
 182
 183 /*
 184  * Note that there's no plan.
 185  */
 186 int
 187 plan_no_plan(void)
 188 {
 189
 190         LOCK;
 191
 192         _tap_init();
 193
 194         if(have_plan != 0) {
 195                 fprintf(stderr, "You tried to plan twice!\n");
 196                 test_died = 1;
 197                 UNLOCK;
 198                 exit(255);
 199         }
 200
 201         have_plan = 1;
 202         no_plan = 1;
 203
 204         UNLOCK;
 205
 206         return 1;
 207 }
 208
 209 /*
 210  * Note that the plan is to skip all tests
 211  */
 212 int
 213 plan_skip_all(char *reason)
 214 {
 215
 216         LOCK;
 217
 218         _tap_init();
 219
 220         skip_all = 1;
 221
 222         printf("1..0");
 223
 224         if(reason != NULL)
 225                 printf(" # Skip %s", reason);
 226
 227         printf("\n");
 228
 229         UNLOCK;
 230
 231         exit(0);
 232 }
 233
 234 /*
 235  * Note the number of tests that will be run.
 236  */
 237 int
 238 plan_tests(unsigned int tests)
 239 {
 240
 241         LOCK;
 242
 243         _tap_init();
 244
 245         if(have_plan != 0) {
 246                 fprintf(stderr, "You tried to plan twice!\n");
 247                 test_died = 1;
 248                 UNLOCK;
 249                 exit(255);
 250         }
 251
 252         if(tests == 0) {
 253                 fprintf(stderr, "You said to run 0 tests!  You've got to run something.\n");
 254                 test_died = 1;
 255                 UNLOCK;
 256                 exit(255);
 257         }
 258
 259         have_plan = 1;
 260
 261         _expected_tests(tests);
 262
 263         UNLOCK;
 264
 265         return e_tests;
 266 }
 267
 268 unsigned int
 269 diag(char *fmt, ...)
 270 {
 271         va_list ap;
 272
 273         fputs("# ", stderr);
 274
 275         va_start(ap, fmt);
 276         vfprintf(stderr, fmt, ap);
 277         va_end(ap);
 278
 279         fputs("\n", stderr);
 280
 281         return 0;
 282 }
 283
 284 void
 285 _expected_tests(unsigned int tests)
 286 {
 287
 288         printf("1..%d\n", tests);
 289         e_tests = tests;
 290 }
 291
 292 int
 293 skip(unsigned int n, char *fmt, ...)
 294 {
 295         va_list ap;
 296         char *skip_msg;
 297
 298         LOCK;
 299
 300         va_start(ap, fmt);
 301         asprintf(&skip_msg, fmt, ap);
 302         va_end(ap);
 303
 304         while(n-- > 0) {
 305                 test_count++;
 306                 printf("ok %d # skip %s\n", test_count,
 307                        skip_msg != NULL ?
 308                        skip_msg : "libtap():malloc() failed");
 309         }
 310
 311         free(skip_msg);
 312
 313         UNLOCK;
 314
 315         return 1;
 316 }
 317
 318 void
 319 todo_start(char *fmt, ...)
 320 {
 321         va_list ap;
 322
 323         LOCK;
 324
 325         va_start(ap, fmt);
 326         vasprintf(&todo_msg, fmt, ap);
 327         va_end(ap);
 328
 329         todo = 1;
 330
 331         UNLOCK;
 332 }
 333
 334 void
 335 todo_end(void)
 336 {
 337
 338         LOCK;
 339
 340         todo = 0;
 341         free(todo_msg);
 342
 343         UNLOCK;
 344 }
 345
 346 int
 347 exit_status(void)
 348 {
 349         int r;
 350
 351         LOCK;
 352
 353         /* If there's no plan, just return the number of failures */
 354         if(no_plan || !have_plan) {
 355                 UNLOCK;
 356                 return failures;
 357         }
 358
 359         /* Ran too many tests?  Return the number of tests that were run
 360            that shouldn't have been */
 361         if(e_tests < test_count) {
 362                 r = test_count - e_tests;
 363                 UNLOCK;
 364                 return r;
 365         }
 366
 367         /* Return the number of tests that failed + the number of tests
 368            that weren't run */
 369         r = failures + e_tests - test_count;
 370         UNLOCK;
 371
 372         return r;
 373 }
 374
 375 /*
 376  * Cleanup at the end of the run, produce any final output that might be
 377  * required.
 378  */
 379 void
 380 _cleanup(void)
 381 {
 382
 383         LOCK;
 384
 385         /* If plan_no_plan() wasn't called, and we don't have a plan,
 386            and we're not skipping everything, then something happened
 387            before we could produce any output */
 388         if(!no_plan && !have_plan && !skip_all) {
 389                 diag("Looks like your test died before it could output anything.");
 390                 UNLOCK;
 391                 return;
 392         }
 393
 394         if(test_died) {
 395                 diag("Looks like your test died just after %d.", test_count);
 396                 UNLOCK;
 397                 return;
 398         }
 399
 400
 401         /* No plan provided, but now we know how many tests were run, and can
 402            print the header at the end */
 403         if(!skip_all && (no_plan || !have_plan)) {
 404                 printf("1..%d\n", test_count);
 405         }
 406
 407         if((have_plan && !no_plan) && e_tests < test_count) {
 408                 diag("Looks like you planned %d %s but ran %d extra.",
 409                      e_tests, e_tests == 1 ? "test" : "tests", test_count - e_tests);
 410                 UNLOCK;
 411                 return;
 412         }
 413
 414         if((have_plan || !no_plan) && e_tests > test_count) {
 415                 diag("Looks like you planned %d %s but only ran %d.",
 416                      e_tests, e_tests == 1 ? "test" : "tests", test_count);
 417                 UNLOCK;
 418                 return;
 419         }
 420
 421         if(failures)
 422                 diag("Looks like you failed %d %s of %d.",
 423                      failures, failures == 1 ? "test" : "tests", test_count);
 424
 425         UNLOCK;
 426 }