xapian-core  1.4.23
testsuite.cc
Go to the documentation of this file.
1 
4 /* Copyright 1999,2000,2001 BrightStation PLC
5  * Copyright 2002 Ananova Ltd
6  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2012,2013,2015,2016,2017 Olly Betts
7  * Copyright 2007 Richard Boulton
8  *
9  * This program is free software; you can redistribute it and/or
10  * modify it under the terms of the GNU General Public License as
11  * published by the Free Software Foundation; either version 2 of the
12  * License, or (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
22  * USA
23  */
24 
25 #include <config.h>
26 
27 #include "testsuite.h"
28 
29 #ifndef NO_LIBXAPIAN
30 # include "backendmanager.h"
31 #endif
32 #include "fdtracker.h"
33 #include "testrunner.h"
34 #include "safeunistd.h"
35 
36 #ifdef HAVE_VALGRIND
37 # include <valgrind/memcheck.h>
38 # include <sys/types.h>
39 # include "safefcntl.h"
40 #endif
41 
42 #include <algorithm>
43 #include <ios>
44 #include <iostream>
45 #include <set>
46 
47 #include <cerrno>
48 #include <cfloat> // For DBL_DIG.
49 #include <cmath> // For ceil, fabs, log10.
50 #include <cstdio>
51 #include <cstdlib>
52 #include <cstring>
53 
54 #include "gnu_getopt.h"
55 
56 #include <setjmp.h>
57 #include <signal.h>
58 
59 #include <exception>
60 #ifdef USE_RTTI
61 # include <typeinfo>
62 # ifdef HAVE_CXXABI_H
63 # include <cxxabi.h>
64 # endif
65 #endif
66 
67 #ifndef NO_LIBXAPIAN
68 # include <xapian/error.h>
69 #endif
70 #include "errno_to_string.h"
71 #include "filetests.h"
72 #include "noreturn.h"
73 #include "stringutils.h"
74 
75 using namespace std;
76 
78 int verbose;
79 
80 #ifdef HAVE_VALGRIND
81 static int vg_log_fd = -1;
82 #endif
83 
84 #if HAVE_DECL_SIGSETJMP && HAVE_DECL_SIGLONGJMP
85 # define SIGSETJMP(ENV, SAVESIGS) sigsetjmp(ENV, SAVESIGS)
86 # define SIGLONGJMP(ENV, VAL) siglongjmp(ENV, VAL)
87 # define SIGJMP_BUF sigjmp_buf
88 #else
89 # define SIGSETJMP(ENV, SAVESIGS) setjmp(ENV)
90 # define SIGLONGJMP(ENV, VAL) longjmp(ENV, VAL)
91 # define SIGJMP_BUF jmp_buf
92 #endif
93 
95 // We use this to attempt to diagnose when the code fails to catch an
96 // exception when it should (due to a compiler or runtime fault in
97 // GCC 2.95 it seems)
98 const char * expected_exception = NULL;
99 
100 const char* expected_failure;
101 
103 std::ostringstream tout;
104 
105 int test_driver::runs = 0;
108 string test_driver::argv0;
109 string test_driver::opt_help;
110 map<int, string *> test_driver::short_opts;
111 vector<string> test_driver::test_names;
112 bool test_driver::abort_on_error = false;
115 bool test_driver::use_cr = false;
116 
117 void
119 {
120  const string & s = tout.str();
121  if (!s.empty()) {
122  out << '\n' << s;
123  tout.str(string());
124  }
125 }
126 
127 string
129 {
130  char *p = getenv("srcdir");
131  if (p != NULL) return string(p);
132 
133  // Default srcdir to the pathname of argv[0].
134  string srcdir(argv0);
135  string::size_type i = srcdir.find_last_of(DIR_SEPS);
136  string srcfile;
137  if (i != string::npos) {
138  srcfile.assign(srcdir, i + 1, string::npos);
139  srcdir.erase(i);
140  // libtool may put the real executable in .libs.
141  i = srcdir.find_last_of(DIR_SEPS);
142  if (srcdir.substr(i + 1) == ".libs") {
143  srcdir.erase(i);
144  // And it may have an "lt-" prefix.
145  if (startswith(srcfile, "lt-")) srcfile.erase(0, 3);
146  }
147  } else {
148  // No path of argv[0], so default srcdir to the current directory.
149  // This may not work if libtool is involved as the true executable is
150  // sometimes in ".libs".
151  srcfile = srcdir;
152  srcdir = ".";
153  }
154 
155  // Remove any trailing ".exe" suffix, since some platforms add this.
156  if (endswith(srcfile, ".exe")) srcfile.resize(srcfile.size() - 4);
157 
158  // Sanity check.
159  if (!file_exists(srcdir + '/' + srcfile + ".cc")) {
160  cout << argv0
161  << ": srcdir is not in the environment and I can't guess it!\n"
162  "Run test programs using the runtest script - see HACKING "
163  "for details\n";
164  exit(1);
165  }
166  return srcdir;
167 }
168 
170  : out(cout.rdbuf()), tests(tests_)
171 {
172  tout << boolalpha;
173 }
174 
175 static SIGJMP_BUF jb;
176 static int signum = 0;
177 static void * sigaddr = NULL;
178 
179 // Needs C linkage so we can pass it to sigaction()/signal() without problems.
180 extern "C" {
181 
182 #if defined HAVE_SIGACTION && defined SA_SIGINFO
183 XAPIAN_NORETURN(static void handle_sig(int signum_, siginfo_t *si, void *));
184 static void handle_sig(int signum_, siginfo_t *si, void *)
185 {
186  // Disable all our signal handlers to avoid problems if the signal
187  // handling code causes a signal.
188  struct sigaction sa;
189  sa.sa_handler = SIG_DFL;
190  sigemptyset(&sa.sa_mask);
191  sa.sa_flags = 0;
192  // We set the handlers with SA_RESETHAND, but that will only reset the
193  // handler for the signal which fired.
194  if (signum_ != SIGSEGV) sigaction(SIGSEGV, &sa, NULL);
195  if (signum_ != SIGFPE) sigaction(SIGFPE, &sa, NULL);
196  if (signum_ != SIGILL) sigaction(SIGILL, &sa, NULL);
197 # ifdef SIGBUS
198  if (signum_ != SIGBUS) sigaction(SIGBUS, &sa, NULL);
199 # endif
200 # ifdef SIGSTKFLT
201  if (signum_ != SIGSTKFLT) sigaction(SIGSTKFLT, &sa, NULL);
202 # endif
203  signum = signum_;
204  sigaddr = si->si_addr;
205  SIGLONGJMP(jb, 1);
206 }
207 
208 #else
209 
210 XAPIAN_NORETURN(static void handle_sig(int signum_));
211 static void handle_sig(int signum_)
212 {
213  // Disable all our signal handlers to avoid problems if the signal
214  // handling code causes a signal.
215  signal(SIGSEGV, SIG_DFL);
216  signal(SIGFPE, SIG_DFL);
217  signal(SIGILL, SIG_DFL);
218 #ifdef SIGBUS
219  signal(SIGBUS, SIG_DFL);
220 #endif
221 #ifdef SIGSTKFLT
222  signal(SIGSTKFLT, SIG_DFL);
223 #endif
224  signum = signum_;
225  SIGLONGJMP(jb, 1);
226 }
227 #endif
228 
229 }
230 
232  private:
233  bool active;
234  public:
235  SignalRedirector() : active(false) { }
236  void activate() {
237  active = true;
238  signum = 0;
239  sigaddr = NULL;
240  // SA_SIGINFO not universal (e.g. not present on Linux < 2.2 and Hurd).
241 #if defined HAVE_SIGACTION && defined SA_SIGINFO
242  struct sigaction sa;
243  sa.sa_sigaction = handle_sig;
244  sigemptyset(&sa.sa_mask);
245  sa.sa_flags = SA_RESETHAND|SA_SIGINFO;
246  sigaction(SIGSEGV, &sa, NULL);
247  sigaction(SIGFPE, &sa, NULL);
248  sigaction(SIGILL, &sa, NULL);
249 # ifdef SIGBUS
250  sigaction(SIGBUS, &sa, NULL);
251 # endif
252 # ifdef SIGSTKFLT
253  sigaction(SIGSTKFLT, &sa, NULL);
254 # endif
255 #else
256  signal(SIGSEGV, handle_sig);
257  signal(SIGFPE, handle_sig);
258  signal(SIGILL, handle_sig);
259 # ifdef SIGBUS
260  signal(SIGBUS, handle_sig);
261 # endif
262 # ifdef SIGSTKFLT
263  signal(SIGSTKFLT, handle_sig);
264 # endif
265 #endif
266  }
268  if (active) {
269 #if defined HAVE_SIGACTION && defined SA_SIGINFO
270  struct sigaction sa;
271  sa.sa_handler = SIG_DFL;
272  sigemptyset(&sa.sa_mask);
273  sa.sa_flags = 0;
274  sigaction(SIGSEGV, &sa, NULL);
275  sigaction(SIGFPE, &sa, NULL);
276  sigaction(SIGILL, &sa, NULL);
277 # ifdef SIGBUS
278  sigaction(SIGBUS, &sa, NULL);
279 # endif
280 # ifdef SIGSTKFLT
281  sigaction(SIGSTKFLT, &sa, NULL);
282 # endif
283 #else
284  signal(SIGSEGV, SIG_DFL);
285  signal(SIGFPE, SIG_DFL);
286  signal(SIGILL, SIG_DFL);
287 # ifdef SIGBUS
288  signal(SIGBUS, SIG_DFL);
289 # endif
290 # ifdef SIGSTKFLT
291  signal(SIGSTKFLT, SIG_DFL);
292 # endif
293 #endif
294  }
295  }
296 };
297 
298 // A wrapper around the tests to trap exceptions,
299 // and avoid having to catch them in every test function.
300 // If this test driver is used for anything other than
301 // Xapian tests, then this ought to be provided by
302 // the client, really.
303 // return: test_driver::PASS, test_driver::FAIL, test_driver::SKIP,
304 // test_driver::XFAIL or test_driver:XPASS.
307 {
308  // This is used to make a note of how many times we've run the test
309  volatile int runcount = 0;
310 
311  FDTracker fdtracker;
312  fdtracker.init();
313 
314  while (true) {
315  tout.str(string());
316  if (SIGSETJMP(jb, 1) == 0) {
317  SignalRedirector sig; // use object so signal handlers are reset
318  static bool catch_signals =
319  (getenv("XAPIAN_TESTSUITE_SIG_DFL") == NULL);
320  if (catch_signals) sig.activate();
321  try {
322  expected_exception = NULL;
323  expected_failure = NULL;
324 #ifdef HAVE_VALGRIND
325  int vg_errs = 0;
326  long vg_leaks = 0, vg_dubious = 0, vg_reachable = 0;
327  if (vg_log_fd != -1) {
328  VALGRIND_DO_LEAK_CHECK;
329  vg_errs = VALGRIND_COUNT_ERRORS;
330  long dummy;
331  VALGRIND_COUNT_LEAKS(vg_leaks, vg_dubious, vg_reachable, dummy);
332  (void)dummy;
333  // Skip past any unread log output.
334  lseek(vg_log_fd, 0, SEEK_END);
335  }
336 #endif
337  test->run();
338  if (verbose > 1)
340 #ifndef NO_LIBXAPIAN
341  if (backendmanager)
343 #endif
344 #ifdef HAVE_VALGRIND
345  if (vg_log_fd != -1) {
346  // We must empty tout before asking valgrind to perform its
347  // leak checks, otherwise the buffers holding the output
348  // may be identified as a memory leak (especially if >1K of
349  // output has been buffered it appears...)
350  tout.str(string());
351 #define REPORT_FAIL_VG(M) do { \
352  if (verbose) { \
353  while (true) { \
354  ssize_t c = read(vg_log_fd, buf, sizeof(buf)); \
355  if (c == 0 || (c < 0 && errno != EINTR)) break; \
356  if (c > 0) out << string(buf, c); \
357  } \
358  } \
359  out << " " << col_red << M << col_reset; \
360 } while (0)
361  // Record the current position so we can restore it so
362  // REPORT_FAIL_VG() gets the whole output.
363  off_t curpos = lseek(vg_log_fd, 0, SEEK_CUR);
364  char buf[4096];
365  while (true) {
366  ssize_t c = read(vg_log_fd, buf, sizeof(buf));
367  if (c == 0 || (c < 0 && errno != EINTR)) {
368  buf[0] = 0;
369  break;
370  }
371  if (c > 0) {
372  // Valgrind output has "==<pid>== \n" between
373  // report "records", so skip any lines like that,
374  // and also any warnings and continuation lines.
375  ssize_t i = 0;
376  while (true) {
377  const char * spc;
378  spc = static_cast<const char *>(
379  memchr(buf + i, ' ', c - i));
380  if (!spc) {
381  i = c;
382  break;
383  }
384  i = spc - buf;
385  if (++i >= c) break;
386  if (buf[i] == '\n')
387  continue;
388  if (c - i >= 8 &&
389  (memcmp(buf + i, "Warning:", 8) == 0 ||
390  memcmp(buf + i, " ", 3) == 0)) {
391  // Skip this line.
392  i += 3;
393  const char * nl;
394  nl = static_cast<const char *>(
395  memchr(buf + i, '\n', c - i));
396  if (!nl) {
397  i = c;
398  break;
399  }
400  i = nl - buf;
401  continue;
402  }
403  break;
404  }
405 
406  char *start = buf + i;
407  c -= i;
408  if (c > 128) c = 128;
409 
410  {
411  const char *p;
412  p = static_cast<const char*>(
413  memchr(start, '\n', c));
414  if (p != NULL) c = p - start;
415  }
416 
417  memmove(buf, start, c);
418  buf[c] = '\0';
419  break;
420  }
421  }
422  lseek(vg_log_fd, curpos, SEEK_SET);
423 
424  int vg_errs2 = VALGRIND_COUNT_ERRORS;
425  vg_errs = vg_errs2 - vg_errs;
426  VALGRIND_DO_LEAK_CHECK;
427  long vg_leaks2 = 0, vg_dubious2 = 0, vg_reachable2 = 0;
428  long dummy;
429  VALGRIND_COUNT_LEAKS(vg_leaks2, vg_dubious2, vg_reachable2,
430  dummy);
431  (void)dummy;
432  vg_leaks = vg_leaks2 - vg_leaks;
433  vg_dubious = vg_dubious2 - vg_dubious;
434  vg_reachable = vg_reachable2 - vg_reachable;
435  if (vg_errs) {
436  string fail_msg(buf);
437  if (fail_msg.empty())
438  fail_msg = "VALGRIND DETECTED A PROBLEM";
439  REPORT_FAIL_VG(fail_msg);
440  return FAIL;
441  }
442  if (vg_leaks > 0) {
443  REPORT_FAIL_VG("LEAKED " << vg_leaks << " BYTES");
444  return FAIL;
445  }
446  if (vg_dubious > 0) {
447  // If code deliberately holds onto blocks by a pointer
448  // not to the start (e.g. languages/utilities.c does)
449  // then we need to rerun the test to see if the leak is
450  // real...
451  if (runcount == 0) {
452  out << col_yellow << " PROBABLY LEAKED MEMORY - RETRYING TEST" << col_reset;
453  ++runcount;
454  // Ensure that any cached memory from fd tracking
455  // is allocated before we rerun the test.
456  (void)fdtracker.check();
457  continue;
458  }
459  REPORT_FAIL_VG("PROBABLY LEAKED " << vg_dubious << " BYTES");
460  return FAIL;
461  }
462  if (vg_reachable > 0) {
463  // C++ STL implementations often "horde" released
464  // memory - for GCC 3.4 and newer the runtest script
465  // sets GLIBCXX_FORCE_NEW=1 which will disable this
466  // behaviour so we avoid this issue, but for older
467  // GCC and other compilers this may be an issue.
468  //
469  // See also:
470  // https://valgrind.org/docs/manual/faq.html#faq.reports
471  //
472  // For now, just use runcount to rerun the test and see
473  // if more is leaked - hopefully this shouldn't give
474  // false positives.
475  if (runcount == 0) {
476  out << col_yellow << " POSSIBLE UNRELEASED MEMORY - RETRYING TEST" << col_reset;
477  ++runcount;
478  // Ensure that any cached memory from fd tracking
479  // is allocated before we rerun the test.
480  (void)fdtracker.check();
481  continue;
482  }
483  REPORT_FAIL_VG("FAILED TO RELEASE " << vg_reachable << " BYTES");
484  return FAIL;
485  }
486  }
487 #endif
488  if (!fdtracker.check()) {
489  if (runcount == 0) {
490  out << col_yellow << " POSSIBLE FDLEAK:" << fdtracker.get_message() << col_reset;
491  ++runcount;
492  continue;
493  }
494  out << col_red << " FDLEAK:" << fdtracker.get_message() << col_reset;
495  return FAIL;
496  }
497  } catch (const TestFail &) {
498  out << ' ';
499  if (expected_failure) {
500  out << col_yellow << "XFAIL (" << expected_failure << ")";
501  } else {
502  out << col_red << "FAILED";
503  }
504  out << col_reset;
506  return expected_failure ? XFAIL : FAIL;
507  } catch (const TestSkip &) {
508  out << col_yellow << " SKIPPED" << col_reset;
510  return SKIP;
511 #ifndef NO_LIBXAPIAN
512  } catch (const Xapian::Error &err) {
513  out << ' ';
514  string errclass = err.get_type();
515  if (expected_exception && expected_exception == errclass) {
516  out << col_yellow << "C++ FAILED TO CATCH " << errclass << col_reset;
517  return SKIP;
518  }
519  if (errclass == "NetworkError" &&
520  err.get_error_string() != NULL &&
521  err.get_error_string() == errno_to_string(ECHILD)) {
522  // ECHILD suggests we've run out of processes, and that's
523  // much more likely to be a system issue than a Xapian bug.
524  //
525  // We also see apparently spurious ECHILD on Debian
526  // buildds sometimes: https://bugs.debian.org/681941
527  out << col_yellow << "ECHILD in network code" << col_reset;
528  return SKIP;
529  }
530 
531  if (expected_failure) {
532  out << col_yellow << "XFAIL (" << expected_failure
533  << "): ";
534  } else {
535  out << col_red << "FAIL: ";
536  }
537  out << err.get_description() << col_reset;
539  return expected_failure ? XFAIL : FAIL;
540 #endif
541  } catch (const string & msg) {
542  out << ' ';
543  if (expected_failure) {
544  out << col_yellow << "XFAIL (" << expected_failure
545  << "): ";
546  } else {
547  out << col_red << "FAIL: ";
548  }
549  out << "EXCEPTION std::string " << msg << col_reset;
551  return expected_failure ? XFAIL : FAIL;
552  } catch (const std::exception & e) {
553  out << ' ';
554  if (expected_failure) {
555  out << col_yellow << "XFAIL (" << expected_failure
556  << "): ";
557  } else {
558  out << col_red << "FAIL: ";
559  }
560 #ifndef USE_RTTI
561  out << "std::exception";
562 #else
563  const char * name = typeid(e).name();
564 # ifdef HAVE_CXXABI_H
565  // __cxa_demangle() apparently requires GCC >= 3.1.
566  // Demangle the name which GCC returns for type_info::name().
567  int status;
568  char * realname = abi::__cxa_demangle(name, NULL, 0, &status);
569  if (realname) {
570  out << realname;
571  free(realname);
572  } else {
573  out << name;
574  }
575 # else
576  out << name;
577 # endif
578 #endif
579  out << ": " << e.what() << col_reset;
581  return expected_failure ? XFAIL : FAIL;
582  } catch (const char * msg) {
583  out << ' ';
584  if (expected_failure) {
585  out << col_yellow << "XFAIL (" << expected_failure
586  << "): ";
587  } else {
588  out << col_red << "FAIL: ";
589  }
590  if (msg) {
591  out << "EXCEPTION char* " << msg;
592  } else {
593  out << "EXCEPTION (char*)NULL";
594  }
595  out << col_reset;
597  return expected_failure ? XFAIL : FAIL;
598  } catch (...) {
599  out << ' ';
600  if (expected_failure) {
601  out << col_yellow << "XFAIL (" << expected_failure
602  << "): ";
603  } else {
604  out << col_red << "FAIL: ";
605  }
606  out << "UNKNOWN EXCEPTION" << col_reset;
608  return expected_failure ? XFAIL : FAIL;
609  }
610 
611  if (expected_failure) {
612  // Testcase marked as expected to fail but actually passed.
613  out << ' ' << col_red << "XPASS (" << expected_failure << ")"
614  << col_reset;
616  return XPASS;
617  }
618  return PASS;
619  }
620 
621  // Caught a signal.
622  const char *signame = "SIGNAL";
623  bool show_addr = true;
624  switch (signum) {
625  case SIGSEGV: signame = "SIGSEGV"; break;
626  case SIGFPE: signame = "SIGFPE"; break;
627  case SIGILL: signame = "SIGILL"; break;
628 #ifdef SIGBUS
629  case SIGBUS: signame = "SIGBUS"; break;
630 #endif
631 #ifdef SIGSTKFLT
632  case SIGSTKFLT:
633  signame = "SIGSTKFLT";
634  show_addr = false;
635  break;
636 #endif
637  }
638  out << " " << col_red << signame;
639  if (show_addr) {
640  char buf[40];
641  sprintf(buf, " at %p", sigaddr);
642  out << buf;
643  }
644  out << col_reset;
646  return FAIL;
647  }
648 }
649 
651 test_driver::run_tests(vector<string>::const_iterator b,
652  vector<string>::const_iterator e)
653 {
654  return do_run_tests(b, e);
655 }
656 
659 {
660  const vector<string> blank;
661  return do_run_tests(blank.begin(), blank.end());
662 }
663 
665 test_driver::do_run_tests(vector<string>::const_iterator b,
666  vector<string>::const_iterator e)
667 {
668  set<string> m(b, e);
669  bool check_name = !m.empty();
670 
672 
673  for (const test_desc *test = tests; test->name; ++test) {
674  bool do_this_test = !check_name;
675  if (!do_this_test && m.find(test->name) != m.end())
676  do_this_test = true;
677  if (!do_this_test) {
678  // if this test is "foo123" see if "foo" was listed
679  // this way "./testprog foo" can run foo1, foo2, etc.
680  string t = test->name;
681  string::size_type i;
682  i = t.find_last_not_of("0123456789") + 1;
683  if (i != string::npos) {
684  t.resize(i);
685  if (m.find(t) != m.end()) do_this_test = true;
686  }
687  }
688  if (do_this_test) {
689  out << "Running test: " << test->name << "...";
690  out.flush();
691  test_driver::test_result test_res = runtest(test);
692 #ifndef NO_LIBXAPIAN
693  if (backendmanager)
695 #endif
696  switch (test_res) {
697  case PASS:
698  ++res.succeeded;
699  if (verbose || !use_cr) {
700  out << col_green << " ok" << col_reset << '\n';
701  } else {
702  out << "\r \r";
703  }
704  break;
705  case XFAIL:
706  ++res.xfailed;
707  out << '\n';
708  break;
709  case FAIL:
710  ++res.failed;
711  out << '\n';
712  if (abort_on_error) {
713  throw "Test failed - aborting further tests";
714  }
715  break;
716  case XPASS:
717  ++res.xpassed;
718  out << '\n';
719  if (abort_on_error) {
720  throw "Test marked as XFAIL passed - aborting further tests";
721  }
722  break;
723  case SKIP:
724  ++res.skipped;
725  out << '\n';
726  // ignore the result of this test.
727  break;
728  }
729  }
730  }
731  return res;
732 }
733 
734 void
736 {
737  cout << "Usage: " << argv0 << " [-v|--verbose] [-o|--abort-on-error] "
738  << opt_help << "[TESTNAME]...\n"
739  " " << argv0 << " [-h|--help]\n";
740  exit(1);
741 }
742 
743 /* Needs C linkage so we can pass it to atexit() without problems. */
744 extern "C" {
745 // Call upon program exit if there's more than one test run.
746 static void
748 {
749  test_driver::report(test_driver::total, "total");
750 }
751 }
752 
753 void
754 test_driver::report(const test_driver::result &r, const string &desc)
755 {
756  // Report totals at the end if we reported two or more subtotals.
757  if (++runs == 2) atexit(report_totals);
758 
759  if (r.succeeded != 0 || r.failed != 0) {
760  cout << argv0 << " " << desc << ": ";
761 
762  if (r.failed == 0 && r.xpassed == 0)
763  cout << "All ";
764 
765  cout << col_green << r.succeeded << col_reset << " tests passed";
766 
767  if (r.failed != 0)
768  cout << ", " << col_red << r.failed << col_reset << " failed";
769 
770  if (r.xpassed != 0)
771  cout << ", " << col_red << r.xpassed << col_reset
772  << " expected failures passed";
773 
774  if (r.xfailed != 0)
775  cout << ", " << col_yellow << r.xfailed << col_reset
776  << " expected failures";
777 
778  if (r.skipped) {
779  cout << ", " << col_yellow << r.skipped << col_reset
780  << " skipped.\n";
781  } else {
782  cout << ".\n";
783  }
784  }
785 }
786 
787 void
788 test_driver::add_command_line_option(const string &l, char s, string * arg)
789 {
790  short_opts.insert(make_pair(int(s), arg));
791  opt_help += "[-";
792  opt_help += s;
793  opt_help += ' ';
794  opt_help += l;
795  opt_help += "] ";
796 }
797 
798 void
799 test_driver::parse_command_line(int argc, char **argv)
800 {
801  argv0 = argv[0];
802 
803 #ifdef HAVE_VALGRIND
804  if (RUNNING_ON_VALGRIND) {
805  if (getenv("XAPIAN_TESTSUITE_VALGRIND") != NULL) {
806  // Open the valgrind log file, and unlink it.
807  char fname[64];
808  sprintf(fname, ".valgrind.log.%lu",
809  static_cast<unsigned long>(getpid()));
810  vg_log_fd = open(fname, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
811  if (vg_log_fd != -1) unlink(fname);
812  }
813  }
814 #endif
815 
816 #ifndef __WIN32__
817  {
818  bool colourise = true;
819  const char *p = getenv("XAPIAN_TESTSUITE_OUTPUT");
820  if (p == NULL || !*p || strcmp(p, "auto") == 0) {
821  colourise = isatty(1);
822  } else if (strcmp(p, "plain") == 0) {
823  colourise = false;
824  }
825  if (colourise) {
826  col_red = "\x1b[1m\x1b[31m";
827  col_green = "\x1b[1m\x1b[32m";
828  col_yellow = "\x1b[1m\x1b[33m";
829  col_reset = "\x1b[0m";
830  use_cr = true;
831  }
832  }
833 #endif
834 
835  static const struct option long_opts[] = {
836  {"verbose", no_argument, 0, 'v'},
837  {"abort-on-error", no_argument, 0, 'o'},
838  {"help", no_argument, 0, 'h'},
839  {NULL, 0, 0, 0}
840  };
841 
842  string short_opts_string = "voh";
843  map<int, string *>::const_iterator i;
844  for (i = short_opts.begin(); i != short_opts.end(); ++i) {
845  short_opts_string += char(i->first);
846  short_opts_string += ':';
847  }
848  const char * opts = short_opts_string.c_str();
849 
850  int c;
851  while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
852  switch (c) {
853  case 'v':
854  ++verbose;
855  break;
856  case 'o':
857  abort_on_error = true;
858  break;
859  default: {
860  i = short_opts.find(c);
861  if (i != short_opts.end()) {
862  i->second->assign(optarg);
863  break;
864  }
865  // -h or unrecognised option
866  usage();
867  return; // usage() doesn't return ...
868  }
869  }
870  }
871 
872  if (verbose == 0) {
873  const char *p = getenv("VERBOSE");
874  if (p != NULL) {
875  verbose = atoi(p);
876  }
877  }
878 
879  while (argv[optind]) {
880  test_names.push_back(string(argv[optind]));
881  ++optind;
882  }
883 }
884 
885 int
887 {
888  test_driver driver(tests);
889 
890  test_driver::result myresult;
891  myresult = driver.run_tests(test_names.begin(), test_names.end());
892 
893  subtotal += myresult;
894 
895  // Return value is a Unix-style exit code, so 0 for success and 1 for
896  // failure.
897  return myresult.failed > 0 || myresult.xpassed > 0;
898 }
899 
900 bool
901 TEST_EQUAL_DOUBLE_(double a, double b)
902 {
903  if (a == b) return true;
904  return (ceil(log10(max(fabs(a), fabs(b)))) - log10(fabs(a - b)) > DBL_DIG);
905 }
static bool use_cr
Definition: testsuite.h:260
static std::string argv0
Definition: testsuite.h:253
bool endswith(const std::string &s, char sfx)
Definition: stringutils.h:75
static void usage()
Definition: testsuite.cc:735
Define the XAPIAN_NORETURN macro.
Run multiple tests for different backends.
Wrappers to allow GNU getopt to be used cleanly from C++ code.
int optind
Definition: getopt.cc:94
result run_tests()
Run all the tests supplied and return the results.
Definition: testsuite.cc:658
unsigned int xfailed
Number of tests with result XFAIL.
Definition: testsuite.h:130
static std::string col_yellow
Definition: testsuite.h:256
Class which is thrown when a test case fails.
Definition: testsuite.h:46
int gnu_getopt_long(int argc_, char *const *argv_, const char *shortopts_, const struct option *longopts_, int *optind_)
Definition: gnu_getopt.h:97
static void parse_command_line(int argc, char **argv)
Parse the command line arguments.
Definition: testsuite.cc:799
test_result runtest(const test_desc *test)
Runs the test function and returns its result.
Definition: testsuite.cc:306
static void report_totals(void)
Definition: testsuite.cc:747
static int signum
Definition: testsuite.cc:176
unsigned int xpassed
Number of tests with result XFAIL.
Definition: testsuite.h:136
Track leaked file descriptors.
static void report(const test_driver::result &r, const std::string &desc)
Print summary of tests passed, failed, and skipped.
Definition: testsuite.cc:754
static std::map< int, std::string * > short_opts
Definition: testsuite.h:216
static const char * opts
static result subtotal
Definition: testsuite.h:199
Convert errno value to std::string, thread-safe if possible.
#define DIR_SEPS
Definition: config.h:8
a generic test suite engine
WritableDatabase open()
Construct a WritableDatabase object for a new, empty InMemory database.
Definition: dbfactory.h:104
STL namespace.
result do_run_tests(std::vector< std::string >::const_iterator b, std::vector< std::string >::const_iterator e)
The implementation used by run_tests.
Definition: testsuite.cc:665
Utility functions for testing files.
#define O_CLOEXEC
Definition: safefcntl.h:90
const char * get_type() const
The type of this error (e.g. "DocNotFoundError".)
Definition: error.h:117
#define false
Definition: header.h:9
static std::string col_green
Definition: testsuite.h:256
static std::string get_srcdir()
Read srcdir from environment and if not present, make a valiant attempt to guess a value...
Definition: testsuite.cc:128
test_driver(const test_desc *tests_)
The constructor, which sets up the test driver.
Definition: testsuite.cc:169
static void add_command_line_option(const std::string &l, char s, std::string *arg)
Add a test-specific command line option.
Definition: testsuite.cc:788
static SIGJMP_BUF jb
Definition: testsuite.cc:175
#define no_argument
Definition: gnu_getopt.h:79
Hierarchy of classes which Xapian can throw as exceptions.
const char * dummy[]
Definition: version_h.cc:7
std::ostream out
Definition: testsuite.h:244
Base class for backend handling in test harness.
static std::string col_reset
Definition: testsuite.h:256
std::ostringstream tout
The debug printing stream.
Definition: testsuite.cc:103
void errno_to_string(int e, string &s)
unsigned int succeeded
The number of tests which succeeded.
Definition: testsuite.h:118
const char * expected_failure
Set to a string explanation for testcases expected to fail.
Definition: testsuite.cc:100
static result total
Definition: testsuite.h:202
unsigned int failed
The number of tests which failed.
Definition: testsuite.h:121
static string srcdir
Definition: stemtest.cc:43
BackendManager * backendmanager
backendmanager is global so that it can be accessed by individual tests.
Definition: testrunner.cc:42
char * optarg
Definition: getopt.cc:79
#define SIGJMP_BUF
Definition: testsuite.cc:91
bool startswith(const std::string &s, char pfx)
Definition: stringutils.h:51
static const test_desc tests[]
The lists of tests to perform.
bool TEST_EQUAL_DOUBLE_(double a, double b)
Helper function for TEST_EQUAL_DOUBLE macro.
Definition: testsuite.cc:901
static int runs
Definition: testsuite.h:250
static std::string opt_help
Definition: testsuite.h:218
static std::vector< std::string > test_names
Definition: testsuite.h:220
const test_desc * tests
Definition: testsuite.h:247
virtual void clean_up()
Called after each test, to perform any necessary cleanup.
std::string get_description() const
Return a string describing this object.
Definition: error.cc:93
#define SIGSETJMP(ENV, SAVESIGS)
Definition: testsuite.cc:89
static bool abort_on_error
Definition: testsuite.h:241
const std::string & get_message() const
Definition: fdtracker.h:64
char name[9]
Definition: dbcheck.cc:55
void write_and_clear_tout()
Write out anything in tout and clear it.
Definition: testsuite.cc:118
const char * get_error_string() const
Returns any system error string associated with this exception.
Definition: error.cc:50
#define SIGLONGJMP(ENV, VAL)
Definition: testsuite.cc:90
All exceptions thrown by Xapian are subclasses of Xapian::Error.
Definition: error.h:43
void(* run)()
The function to run to perform the test.
Definition: testsuite.h:82
static std::string col_red
Definition: testsuite.h:256
Various handy helpers which std::string really should provide.
void init()
Definition: fdtracker.cc:77
static int run(const test_desc *tests)
Definition: testsuite.cc:886
<unistd.h>, but with compat.
Definition: header.h:151
int verbose
The global verbose flag.
Definition: testsuite.cc:78
A structure used to report the summary of tests passed and failed.
Definition: testsuite.h:116
const char * expected_exception
The exception type we were expecting in TEST_EXCEPTION.
Definition: testsuite.cc:98
Structure holding a description of a test.
Definition: testsuite.h:77
bool check()
Definition: fdtracker.cc:108
The test driver. This class takes care of running the tests.
Definition: testsuite.h:108
const char * name
The name of the test.
Definition: testsuite.h:79
bool file_exists(const char *path)
Test if a file exists.
Definition: filetests.h:39
unsigned int skipped
The number of tests which were skipped.
Definition: testsuite.h:124
include <fcntl.h>, but working around broken platforms.
static void * sigaddr
Definition: testsuite.cc:177
Class which is thrown when a test case is to be skipped.
Definition: testsuite.h:53
static void handle_sig(int signum_)
Definition: testsuite.cc:211