xapian-core  1.4.19
perftest.cc
Go to the documentation of this file.
1 
4 /* Copyright 2008 Lemur Consulting Ltd
5  * Copyright 2008,2009,2010,2012,2013,2015 Olly Betts
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License as
9  * published by the Free Software Foundation; either version 2 of the
10  * License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
20  * USA
21  */
22 
23 #include <config.h>
24 #include "perftest.h"
25 
26 #include "backendmanager.h"
27 #include "freemem.h"
28 #include "omassert.h"
29 #include "perftest/perftest_all.h"
30 #include "realtime.h"
31 #include "runprocess.h"
32 #include "str.h"
33 #include "stringutils.h"
34 #include "testrunner.h"
35 #include "testsuite.h"
36 
37 #include <cstdlib>
38 #include <iostream>
39 
40 #include "safeunistd.h"
41 #ifdef HAVE_SYS_UTSNAME_H
42 # include <sys/utsname.h>
43 #endif
44 
45 #ifdef __WIN32__
46 # include "safewindows.h"
47 # include "safewinsock2.h"
48 #endif
49 
50 using namespace std;
51 
53 
54 static string
55 escape_xml(const string & str)
56 {
57  string res;
58  string::size_type p = 0;
59  while (p < str.size()) {
60  char ch = str[p++];
61  switch (ch) {
62  case '<':
63  res += "&lt;";
64  continue;
65  case '>':
66  res += "&gt;";
67  continue;
68  case '&':
69  res += "&amp;";
70  continue;
71  case '"':
72  res += "&quot;";
73  continue;
74  default:
75  res += ch;
76  }
77  }
78  return res;
79 }
80 
82  : testcase_started(false),
83  indexing_started(false),
84  searching_started(false)
85 {}
86 
88 {
89  close();
90 }
91 
93 static string
95 {
96 #ifdef __WIN32__
97  char buf[256];
98  WORD WSAVerReq = MAKEWORD(1, 1);
99  WSADATA WSAData;
100 
101  if (WSAStartup(WSAVerReq, &WSAData) != 0) {
102  // wrong winsock dlls?
103  return string();
104  }
105  if (gethostname(buf, sizeof(buf)) != 0) {
106  *buf = '\0';
107  }
108  WSACleanup();
109  return buf;
110 #elif defined HAVE_SYS_UTSNAME_H
111  struct utsname uname_buf;
112  if (uname(&uname_buf) != 0) {
113  uname_buf.nodename[0] = '\0';
114  }
115  return uname_buf.nodename;
116 #elif defined HAVE_GETHOSTNAME
117  char buf[256];
118  if (gethostname(buf, sizeof(buf)) != 0) {
119  *buf = '\0';
120  }
121  return buf;
122 #else
123  return string();
124 #endif
125 }
126 
128 static string
130 {
131 #ifdef __WIN32__
132  return string();
133 #else
134  string loadavg;
135  try {
136  loadavg = stdout_to_string("uptime 2>/dev/null | sed 's/.*: \\([0-9][0-9]*\\)/\\1/;s/, .*//'");
137  } catch (const NoSuchProgram&) {
138  } catch (const ReadError&) {
139  }
140  return loadavg;
141 #endif
142 }
143 
145 static string
147 {
148  string ncpus;
149 #ifdef __WIN32__
150  SYSTEM_INFO siSysInfo;
151  GetSystemInfo(&siSysInfo);
152  ncpus = str(siSysInfo.dwNumberOfProcessors);
153 #else
154  try {
155  // Works on Linux, at least back to kernel 2.2.26.
156  ncpus = stdout_to_string("getconf _NPROCESSORS_ONLN 2>/dev/null | grep -v '[^0-9]'");
157  } catch (const NoSuchProgram&) {
158  } catch (const ReadError&) {
159  }
160  if (ncpus.empty())
161  try {
162  // Works on OpenBSD (and apparently FreeBSD and Darwin).
163  ncpus = stdout_to_string("sysctl hw.ncpu 2>/dev/null | sed 's/.*=//'");
164  } catch (const NoSuchProgram&) {
165  } catch (const ReadError&) {
166  }
167  if (ncpus.empty())
168  try {
169  // Works on Solaris and OSF/1.
170  ncpus = stdout_to_string("PATH=/usr/sbin:$PATH psrinfo 2>/dev/null | grep -c on-line");
171  } catch (const NoSuchProgram&) {
172  } catch (const ReadError&) {
173  }
174  if (ncpus.empty())
175  try {
176  // Works on Linux, just in case the getconf version doesn't.
177  // Different architectures have different formats for /proc/cpuinfo
178  // so this won't work as widely as getconf _NPROCESSORS_ONLN will.
179  ncpus = stdout_to_string("grep -c processor /proc/cpuinfo 2>/dev/null");
180  } catch (const NoSuchProgram&) {
181  } catch (const ReadError&) {
182  }
183 #endif
184  return ncpus;
185 }
186 
188 static string
190 {
191  string distro;
192 #ifdef __WIN32__
193  OSVERSIONINFO osvi;
194  ZeroMemory(&osvi, sizeof(OSVERSIONINFO));
195  osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
196  GetVersionEx(&osvi);
197  distro = "Microsoft Windows v";
198  distro += str(osvi.dwMajorVersion);
199  distro += '.';
200  distro += str(osvi.dwMinorVersion);
201  distro += '.';
202  distro += str(osvi.dwBuildNumber);
203 #else
204  try {
205  distro = stdout_to_string("perftest/get_machine_info 2>/dev/null");
206  } catch (const NoSuchProgram&) {
207  } catch (const ReadError&) {
208  }
209 #endif
210  return distro;
211 }
212 
214 static string
216 {
217  string commit_ref;
218  try {
219  commit_ref = stdout_to_string("cd \"$srcdir\" && git log -n1 --abbrev-commit --format=%h");
220  } catch (const NoSuchProgram&) {
221  } catch (const ReadError&) {
222  }
223 
224  return commit_ref;
225 }
226 
227 bool
228 PerfTestLogger::open(const string & logpath)
229 {
230  out.open(logpath.c_str(), ios::out | ios::binary | ios::trunc);
231  if (!out.is_open()) {
232  cerr << "Couldn't open output logfile '" << logpath << "'" << endl;
233  return false;
234  }
235 
236  string loadavg = get_loadavg();
237  string hostname = get_hostname();
238  string ncpus = get_ncpus();
239  string distro = get_distro();
240 
241  // Write header, and details of the machine.
242  write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testrun>\n"
243  " <machineinfo>\n");
244  if (!hostname.empty())
245  write(" <hostname>" + hostname + "</hostname>\n");
246  if (!loadavg.empty())
247  write(" <loadavg>" + loadavg + "</loadavg>\n");
248  if (!ncpus.empty())
249  write(" <ncpus>" + ncpus + "</ncpus>\n");
250  if (!distro.empty())
251  write(" <distro>" + distro + "</distro>\n");
252  write(" <physmem>" + str(get_total_physical_memory()) + "</physmem>\n");
253  write(" </machineinfo>\n");
254 
255  const string & commit_ref = get_commit_ref();
256 
257  write(" <sourceinfo>\n");
258  if (!commit_ref.empty())
259  write(" <commitref>" + commit_ref + "</commitref>\n");
260  write(" <version>" + string(Xapian::version_string()) + "</version>\n");
261  write(" </sourceinfo>\n");
262 
263  return true;
264 }
265 
266 void
267 PerfTestLogger::write(const string & text)
268 {
269  out.write(text.data(), text.size());
270  out.flush();
271 }
272 
273 void
275 {
276  repetition_end();
277  if (out.is_open()) {
278  write("</testrun>\n");
279  out.close();
280  }
281 }
282 
283 void
284 PerfTestLogger::indexing_begin(const string & dbname,
285  const std::map<std::string, std::string> & params)
286 {
287  searching_end();
288  indexing_end();
289  write(" <indexrun dbname=\"" + dbname + "\">\n <params>\n");
290  std::map<std::string, std::string>::const_iterator i;
291  for (i = params.begin(); i != params.end(); ++i) {
292  write(" <param name=\"" + i->first + "\">" + escape_xml(i->second) + "</param>\n");
293  }
294  i = params.find("flush_threshold");
295  if (i == params.end()) {
296  Xapian::doccount flush_threshold = 0;
297  const char *p = getenv("XAPIAN_FLUSH_THRESHOLD");
298  if (p)
299  flush_threshold = atoi(p);
300  if (flush_threshold == 0)
301  flush_threshold = 10000;
302  write(" <param name=\"flush_threshold\">" +
303  escape_xml(str(flush_threshold)) + "</param>\n");
304  }
305  write(" </params>\n");
306  indexing_addcount = 0;
310  indexing_started = true;
311 
312  indexing_log();
313 }
314 
315 void
317 {
320  double elapsed(last_indexlog_timer - indexing_timer);
321  write(" <item>"
322  "<time>" + str(elapsed) + "</time>"
323  "<adds>" + str(indexing_addcount) + "</adds>"
324  "</item>\n");
326 }
327 
328 void
330 {
333  // Log every 1000 documents
334  if (indexing_addcount % 1000 == 0) {
335  indexing_log();
336  } else {
337  // Or after 5 seconds
338  double now = RealTime::now();
339  if (now > last_indexlog_timer + 5)
340  indexing_log();
341  }
342 }
343 
344 void
346 {
347  if (indexing_started) {
348  indexing_log();
349  write(" </indexrun>\n");
350  indexing_started = false;
351  }
352 }
353 
354 void
355 PerfTestLogger::searching_start(const string & description)
356 {
357  indexing_end();
358  searching_end();
359  write(" <searchrun>\n"
360  " <description>" + escape_xml(description) + "</description>\n");
361  searching_started = true;
362  search_start();
363 }
364 
365 void
367 {
369 }
370 
371 void
373  const Xapian::MSet & mset)
374 {
376  double elapsed(RealTime::now() - searching_timer);
377  write(" <search>"
378  "<time>" + str(elapsed) + "</time>"
379  "<query>" + escape_xml(query.get_description()) + "</query>"
380  "<mset>"
381  "<size>" + str(mset.size()) + "</size>"
382  "<lb>" + str(mset.get_matches_lower_bound()) + "</lb>"
383  "<est>" + str(mset.get_matches_estimated()) + "</est>"
384  "<ub>" + str(mset.get_matches_upper_bound()) + "</ub>"
385  "</mset>"
386  "</search>\n");
387  search_start();
388 }
389 
390 void
392 {
393  if (searching_started) {
394  write(" </searchrun>\n");
395  searching_started = false;
396  }
397 }
398 
399 void
401 {
402  testcase_end();
403  write(" <testcase name=\"" + testcase + "\" backend=\"" +
404  backendmanager->get_dbtype() + "\" repnum=\"" +
405  str(repetition_number) + "\">\n");
406  testcase_started = true;
407 }
408 
409 void
411 {
412  indexing_end();
413  if (testcase_started) {
414  write(" </testcase>\n");
415  testcase_started = false;
416  }
417 }
418 
419 void
421 {
422  repetition_end();
423  repetition_number = num;
424 }
425 
426 void
428 {
429  testcase_end();
430 }
431 
432 
434 {
436  mutable bool repetitions_parsed;
437  mutable int repetitions;
438  public:
440  : repetitions_parsed(false), repetitions(5)
441  {
442  test_driver::add_command_line_option("repetitions", 'r',
443  &repetitions_string);
444  }
445 
446  int run() const {
447  int result = 0;
448  if (!repetitions_parsed) {
449  if (!repetitions_string.empty()) {
450  repetitions = atoi(repetitions_string.c_str());
451  }
452  repetitions_parsed = true;
453  }
454  for (int i = 0; i != repetitions; ++i) {
455  logger.repetition_begin(i + 1);
457  logger.repetition_end();
458  }
459  return result;
460  }
461 };
462 
463 int main(int argc, char **argv)
464 {
465  if (!logger.open("perflog.xml"))
466  return 1;
467 
468  PerfTestRunner runner;
469 
470  return runner.run_tests(argc, argv);
471 }
static string get_loadavg()
Get the load average.
Definition: perftest.cc:129
Xapian::doccount size() const
Return number of items in this MSet object.
Definition: omenquire.cc:318
#define Assert(COND)
Definition: omassert.h:122
bool searching_started
Definition: perftest.h:45
Run multiple tests for different backends.
static string get_ncpus()
Get the number of processors.
Definition: perftest.cc:146
void searching_start(const std::string &description)
Log the start of a search run.
Definition: perftest.cc:355
void repetition_begin(int num)
Start a repetition of the tests.
Definition: perftest.cc:420
void search_start()
Log the start of a search.
Definition: perftest.cc:366
Xapian::doccount get_matches_lower_bound() const
Lower bound on the total number of matching documents.
Definition: omenquire.cc:246
static string get_commit_ref()
Get the git commit for HEAD.
Definition: perftest.cc:215
performance tests for Xapian.
run an external process and capture its output in a string.
std::ofstream out
Definition: perftest.h:33
void indexing_add()
Log the addition of a document in an indexing run.
Definition: perftest.cc:329
a generic test suite engine
Class representing a list of search results.
Definition: mset.h:44
bool indexing_unlogged_changes
Definition: perftest.h:41
PerfTestLogger logger
Definition: perftest.cc:52
STL namespace.
Convert types to std::string.
long long get_total_physical_memory()
Determine how much physical memory there is.
Definition: freemem.cc:122
Xapian::doccount indexing_addcount
Definition: perftest.h:40
static string escape_xml(const string &str)
Definition: perftest.cc:55
#define false
Definition: header.h:9
double searching_timer
Definition: perftest.h:46
double indexing_timer
Definition: perftest.h:42
void testcase_begin(const std::string &testcase)
Start a testcase.
Definition: perftest.cc:400
static void add_command_line_option(const std::string &l, char s, std::string *arg)
Add a test-specific command line option.
Definition: testsuite.cc:788
const char * version_string()
Report the version string of the library which the program is linked with.
Definition: xapian.h:115
Xapian::doccount get_matches_upper_bound() const
Upper bound on the total number of matching documents.
Definition: omenquire.cc:262
int run_tests(int argc, char **argv)
Run all the tests.
Definition: testrunner.cc:137
Base class for backend handling in test harness.
bool indexing_started
Definition: perftest.h:39
Exception thrown if we encounter a read error.
Definition: runprocess.h:27
string stdout_to_string(const string &cmd)
Run command cmd, capture its stdout, and return it as a std::string.
Definition: runprocess.cc:39
bool open(const std::string &logpath)
Open a file to log to.
Definition: perftest.cc:228
BackendManager * backendmanager
backendmanager is global so that it can be accessed by individual tests.
Definition: testrunner.cc:42
string str(int value)
Convert int to std::string.
Definition: str.cc:90
include <winsock2.h> but working around problems.
A test runner, which runs the tests (implemented by subclassing it) with a variety of backends...
Definition: testrunner.h:36
Exception thrown if the program isn&#39;t found.
Definition: runprocess.h:30
static string get_distro()
Get details of the OS and distribution.
Definition: perftest.cc:189
virtual std::string get_dbtype() const
Get the database type currently in use.
void indexing_end()
Log the end of an indexing run.
Definition: perftest.cc:345
bool repetitions_parsed
Definition: perftest.cc:436
include <windows.h> without all the bloat and damage.
static string get_hostname()
Get the hostname.
Definition: perftest.cc:94
bool testcase_started
Definition: perftest.h:37
void close()
Flush and close the log file.
Definition: perftest.cc:274
static Xapian::Query query(Xapian::Query::op op, const string &t1=string(), const string &t2=string(), const string &t3=string(), const string &t4=string(), const string &t5=string(), const string &t6=string(), const string &t7=string(), const string &t8=string(), const string &t9=string(), const string &t10=string())
Definition: api_anydb.cc:63
int run() const
Run the tests with a particular backend.
Definition: perftest.cc:446
Xapian::doccount get_matches_estimated() const
Estimate of the total number of matching documents.
Definition: omenquire.cc:253
int repetition_number
Definition: perftest.h:35
double last_indexlog_timer
Definition: perftest.h:43
std::string get_description() const
Return a string describing this object.
Definition: query.cc:232
unsigned XAPIAN_DOCID_BASE_TYPE doccount
A count of documents.
Definition: types.h:38
void repetition_end()
End a repetition of the tests.
Definition: perftest.cc:427
double now()
Return the current time.
Definition: realtime.h:49
void search_end(const Xapian::Query &query, const Xapian::MSet &mset)
Log the completion of a search.
Definition: perftest.cc:372
void indexing_log()
Write a log entry for the current indexing run.
Definition: perftest.cc:316
Various handy helpers which std::string really should provide.
void searching_end()
Log the end of a search run.
Definition: perftest.cc:391
<unistd.h>, but with compat.
void write(const std::string &text)
Definition: perftest.cc:267
void testcase_end()
End a testcase.
Definition: perftest.cc:410
Various assertion macros.
Functions for handling a time or time interval in a double.
Class representing a query.
Definition: query.h:46
determine how much free physical memory there is.
string repetitions_string
Definition: perftest.cc:435
int main(int argc, char **argv)
Definition: perftest.cc:463
void indexing_begin(const std::string &dbname, const std::map< std::string, std::string > &params)
Log the start of an indexing run.
Definition: perftest.cc:284