xapian-core  1.4.18
api_backend.cc
Go to the documentation of this file.
1 
4 /* Copyright (C) 2008,2009,2010,2011,2012,2013,2014,2015,2016,2018,2019 Olly Betts
5  * Copyright (C) 2010 Richard Boulton
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License as
9  * published by the Free Software Foundation; either version 2 of the
10  * License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
20  * USA
21  */
22 
23 #include <config.h>
24 
25 #include "api_backend.h"
26 
27 #define XAPIAN_DEPRECATED(X) X
28 #include <xapian.h>
29 
30 #include "backendmanager.h"
31 #include "errno_to_string.h"
32 #include "filetests.h"
33 #include "str.h"
34 #include "testrunner.h"
35 #include "testsuite.h"
36 #include "testutils.h"
37 #include "unixcmds.h"
38 
39 #include "apitest.h"
40 
41 #include "safefcntl.h"
42 #include "safesysstat.h"
43 #include "safeunistd.h"
44 #ifdef HAVE_SOCKETPAIR
45 # include "safesyssocket.h"
46 # include <signal.h>
47 # include "safesyswait.h"
48 #endif
49 
50 #include <cerrno>
51 #include <fstream>
52 #include <iterator>
53 
54 using namespace std;
55 
57 DEFINE_TESTCASE(lockfileumask1, chert || glass) {
58 #if !defined __WIN32__ && !defined __CYGWIN__ && !defined __OS2__
59  mode_t old_umask = umask(022);
60  try {
62 
63  string path = get_named_writable_database_path("lockfileumask1");
64  path += "/flintlock";
65 
66  struct stat statbuf;
67  TEST(stat(path.c_str(), &statbuf) == 0);
68  TEST_EQUAL(statbuf.st_mode & 0777, 0644);
69  } catch (...) {
70  umask(old_umask);
71  throw;
72  }
73 
74  umask(old_umask);
75 #endif
76 }
77 
79 DEFINE_TESTCASE(totaldoclen1, writable) {
81  Xapian::Document doc;
82  doc.add_posting("foo", 1, 2000000000);
83  db.add_document(doc);
84  Xapian::Document doc2;
85  doc2.add_posting("bar", 1, 2000000000);
86  db.add_document(doc2);
87  TEST_EQUAL(db.get_avlength(), 2000000000);
88  TEST_EQUAL(db.get_total_length(), 4000000000ull);
89  db.commit();
90  TEST_EQUAL(db.get_avlength(), 2000000000);
91  TEST_EQUAL(db.get_total_length(), 4000000000ull);
92  for (int i = 0; i != 20; ++i) {
93  Xapian::Document doc3;
94  doc3.add_posting("count" + str(i), 1, 2000000000);
95  db.add_document(doc3);
96  }
97  TEST_EQUAL(db.get_avlength(), 2000000000);
98  TEST_EQUAL(db.get_total_length(), 44000000000ull);
99  db.commit();
100  TEST_EQUAL(db.get_avlength(), 2000000000);
101  TEST_EQUAL(db.get_total_length(), 44000000000ull);
102  if (get_dbtype() != "inmemory") {
103  // InMemory doesn't support get_writable_database_as_database().
105  TEST_EQUAL(dbr.get_avlength(), 2000000000);
106  TEST_EQUAL(dbr.get_total_length(), 44000000000ull);
107  }
108 }
109 
110 // Check that exceeding 32bit in combined database doesn't cause a problem
111 // when using 64bit docids.
112 DEFINE_TESTCASE(exceed32bitcombineddb1, writable) {
113  // Test case is for 64-bit Xapian::docid.
114  // FIXME: Though we should check that the overflow is handled gracefully
115  // for 32-bit...
116  if (sizeof(Xapian::docid) == 4) return;
117 
118  // The InMemory backend uses a vector for the documents, so trying to add
119  // a document with the maximum docid is likely to fail because we can't
120  // allocate enough memory!
121  SKIP_TEST_FOR_BACKEND("inmemory");
122 
125  Xapian::Document doc;
126  doc.set_data("prose");
127  doc.add_term("word");
128 
129  Xapian::docid max_id = 0xffffffff;
130 
131  db1.replace_document(max_id, doc);
132  db2.replace_document(max_id, doc);
133 
134  Xapian::Database db;
135  db.add_database(db1);
136  db.add_database(db2);
137 
138  Xapian::Enquire enquire(db);
140  Xapian::MSet mymset = enquire.get_mset(0, 10);
141 
142  TEST_EQUAL(2, mymset.size());
143 
144  for (Xapian::MSetIterator i = mymset.begin(); i != mymset.end(); ++i) {
145  TEST_EQUAL("prose", i.get_document().get_data());
146  }
147 }
148 
149 DEFINE_TESTCASE(dbstats1, backend) {
150  Xapian::Database db = get_database("etext");
151 
152  // Use precalculated values to avoid expending CPU cycles to calculate
153  // these every time without improving test coverage.
154  const Xapian::termcount min_len = 2;
155  const Xapian::termcount max_len = 532;
156  const Xapian::termcount max_wdf = 22;
157 
158  if (get_dbtype() != "inmemory") {
159  // Should be exact as no deletions have happened.
160  TEST_EQUAL(db.get_doclength_upper_bound(), max_len);
161  TEST_EQUAL(db.get_doclength_lower_bound(), min_len);
162  } else {
163  // For inmemory, we usually give rather loose bounds.
164  TEST_REL(db.get_doclength_upper_bound(),>=,max_len);
165  TEST_REL(db.get_doclength_lower_bound(),<=,min_len);
166  }
167 
168  if (get_dbtype() != "inmemory" &&
169  get_dbtype().find("remote") == string::npos) {
170  TEST_EQUAL(db.get_wdf_upper_bound("the"), max_wdf);
171  } else {
172  // For inmemory and remote backends, we usually give rather loose
173  // bounds (remote matches use tighter bounds, but querying the
174  // wdf bound gives a looser one).
175  TEST_REL(db.get_wdf_upper_bound("the"),>=,max_wdf);
176  }
177 
178  // This failed with an assertion during development between 1.3.1 and
179  // 1.3.2.
180  TEST_EQUAL(db.get_wdf_upper_bound(""), 0);
181 }
182 
183 // Check stats with a single document. In a multi-database situation, this
184 // gave 0 for get-_doclength_lower_bound() in 1.3.2.
185 DEFINE_TESTCASE(dbstats2, backend) {
186  Xapian::Database db = get_database("apitest_onedoc");
187 
188  // Use precalculated values to avoid expending CPU cycles to calculate
189  // these every time without improving test coverage.
190  const Xapian::termcount min_len = 15;
191  const Xapian::termcount max_len = 15;
192  const Xapian::termcount max_wdf = 7;
193 
194  if (get_dbtype() != "inmemory") {
195  // Should be exact as no deletions have happened.
196  TEST_EQUAL(db.get_doclength_upper_bound(), max_len);
197  TEST_EQUAL(db.get_doclength_lower_bound(), min_len);
198  } else {
199  // For inmemory, we usually give rather loose bounds.
200  TEST_REL(db.get_doclength_upper_bound(),>=,max_len);
201  TEST_REL(db.get_doclength_lower_bound(),<=,min_len);
202  }
203 
204  if (get_dbtype() != "inmemory" &&
205  get_dbtype().find("remote") == string::npos) {
206  TEST_EQUAL(db.get_wdf_upper_bound("word"), max_wdf);
207  } else {
208  // For inmemory and remote backends, we usually give rather loose
209  // bounds (remote matches use tighter bounds, but querying the
210  // wdf bound gives a looser one).
211  TEST_REL(db.get_wdf_upper_bound("word"),>=,max_wdf);
212  }
213 
214  TEST_EQUAL(db.get_wdf_upper_bound(""), 0);
215 }
216 
218 DEFINE_TESTCASE(alldocspl3, backend) {
219  Xapian::Database db = get_database(string());
220 
221  TEST_EQUAL(db.get_termfreq(string()), 0);
222  TEST_EQUAL(db.get_collection_freq(string()), 0);
223  TEST(db.postlist_begin(string()) == db.postlist_end(string()));
224 }
225 
227 DEFINE_TESTCASE(modifiedpostlist1, writable) {
229  Xapian::Document a, b;
230  Xapian::Enquire enq(db);
231 
232  a.add_term("T");
233  enq.set_query(Xapian::Query("T"));
234 
235  db.replace_document(2, a);
236  db.commit();
237  db.replace_document(1, a);
238  db.replace_document(1, b);
239 
240  mset_expect_order(enq.get_mset(0, 2), 2);
241 }
242 
244 DEFINE_TESTCASE(doclenaftercommit1, writable) {
249  db.commit();
250  TEST_EQUAL(db.get_doclength(1), 0);
251  TEST_EQUAL(db.get_unique_terms(1), 0);
252 }
253 
254 DEFINE_TESTCASE(valuesaftercommit1, writable) {
256  Xapian::Document doc;
257  doc.add_value(0, "value");
258  db.replace_document(2, doc);
259  db.commit();
260  db.replace_document(1, doc);
261  db.replace_document(3, doc);
262  TEST_EQUAL(db.get_document(3).get_value(0), "value");
263  db.commit();
264  TEST_EQUAL(db.get_document(3).get_value(0), "value");
265 }
266 
267 DEFINE_TESTCASE(lockfilefd0or1, chert || glass) {
268 #if !defined __WIN32__ && !defined __CYGWIN__ && !defined __OS2__
269  int old_stdin = dup(0);
270  int old_stdout = dup(1);
271  try {
272  // With fd 0 available.
273  close(0);
274  {
278  }
279  // With fd 0 and fd 1 available.
280  close(1);
281  {
285  }
286  // With fd 1 available.
287  dup2(old_stdin, 0);
288  {
292  }
293  } catch (...) {
294  dup2(old_stdin, 0);
295  dup2(old_stdout, 1);
296  close(old_stdin);
297  close(old_stdout);
298  throw;
299  }
300 
301  dup2(old_stdout, 1);
302  close(old_stdin);
303  close(old_stdout);
304 #endif
305 }
306 
308 DEFINE_TESTCASE(lockfilealreadyopen1, chert || glass) {
309  // Ensure database has been created.
310  (void)get_named_writable_database("lockfilealreadyopen1");
311  string path = get_named_writable_database_path("lockfilealreadyopen1");
312  int fd = ::open((path + "/flintlock").c_str(), O_RDONLY);
313  TEST(fd != -1);
314  try {
318  );
319  } catch (...) {
320  close(fd);
321  throw;
322  }
323  close(fd);
324 }
325 
327 DEFINE_TESTCASE(testlock1, chert || glass) {
328  Xapian::Database rdb;
329  TEST(!rdb.locked());
330  {
332  TEST(db.locked());
333  Xapian::Database db_as_database = db;
334  TEST(db_as_database.locked());
335  TEST(!rdb.locked());
337  TEST(db.locked());
338  TEST(db_as_database.locked());
339  try {
340  TEST(rdb.locked());
341  } catch (const Xapian::FeatureUnavailableError&) {
342  SKIP_TEST("Database::locked() not supported on this platform");
343  }
344  db_as_database = rdb;
345  TEST(db.locked());
346  TEST(db_as_database.locked());
347  TEST(rdb.locked());
348  db_as_database.close();
349  TEST(db.locked());
350  TEST(rdb.locked());
351  // After close(), locked() should either work as if close() hadn't been
352  // called or throw Xapian::DatabaseClosedError.
353  try {
354  TEST(db_as_database.locked());
355  } catch (const Xapian::DatabaseClosedError&) {
356  }
357  db.close();
358  TEST(!rdb.locked());
359  try {
360  TEST(!db_as_database.locked());
361  } catch (const Xapian::DatabaseClosedError&) {
362  }
363  }
364  TEST(!rdb.locked());
365 }
366 
371 DEFINE_TESTCASE(testlock2, backend && !writable) {
372  Xapian::Database db = get_database("apitest_simpledata");
373  TEST(!db.locked());
374  db.close();
375  TEST(!db.locked());
376 }
377 
390 DEFINE_TESTCASE(testlock3, inmemory) {
391  Xapian::Database db = get_database("apitest_simpledata");
392  TEST(db.locked());
393  db.close();
394  TEST(!db.locked());
395 }
396 
398 DEFINE_TESTCASE(testlock4, chert || glass) {
399  Xapian::Database db = get_writable_database("apitest_simpledata");
400  // Even if we don't have a way to test the lock on the current platform,
401  // this should know the database is locked because this object holds the
402  // lock.
403  TEST(db.locked());
404  db.close();
405  try {
406  TEST(!db.locked());
407  } catch (const Xapian::FeatureUnavailableError&) {
408  SKIP_TEST("Database::locked() not supported on this platform");
409  }
410 }
411 
413  mutable bool called;
414 
415  public:
416  CheckMatchDecider() : called(false) { }
417 
418  bool operator()(const Xapian::Document &) const {
419  called = true;
420  return true;
421  }
422 
423  bool was_called() const { return called; }
424 };
425 
427 DEFINE_TESTCASE(matchdecider4, remote) {
428  Xapian::Database db(get_database("apitest_simpledata"));
429  Xapian::Enquire enquire(db);
430  enquire.set_query(Xapian::Query("paragraph"));
431 
432  CheckMatchDecider mdecider;
433  Xapian::MSet mset;
434 
436  mset = enquire.get_mset(0, 10, NULL, &mdecider));
437  TEST(!mdecider.was_called());
438 }
439 
443 DEFINE_TESTCASE(replacedoc7, writable && !inmemory && !remote) {
444  // The inmemory backend doesn't batch changes, so there's nothing to
445  // check there.
446  //
447  // The remote backend doesn't implement the lazy replacement of documents
448  // optimisation currently.
450  Xapian::Document doc;
451  doc.set_data("fish");
452  doc.add_term("Hlocalhost");
453  doc.add_posting("hello", 1);
454  doc.add_posting("world", 2);
455  doc.add_value(1, "myvalue");
456  db.add_document(doc);
457  db.commit();
458 
459  // We add a second document, and then replace the first document with
460  // itself 10000 times. If the document count for the database reopened
461  // read-only is 2, then we triggered an automatic commit.
462 
463  doc.add_term("XREV2");
464  db.add_document(doc);
465 
466  for (int i = 0; i < 10000; ++i) {
467  doc = db.get_document(1);
468  db.replace_document(1, doc);
469  }
470 
472  TEST_EQUAL(rodb.get_doccount(), 1);
473 
474  db.flush();
475  TEST(rodb.reopen());
476 
477  TEST_EQUAL(rodb.get_doccount(), 2);
478 }
479 
484 DEFINE_TESTCASE(replacedoc8, writable) {
486  {
487  Xapian::Document doc;
488  doc.set_data("fish");
489  doc.add_term("takeaway");
490  db.add_document(doc);
491  }
492  db.delete_document(1);
493  {
494  Xapian::Document doc;
495  doc.set_data("chips");
496  doc.add_term("takeaway", 2);
497  db.replace_document(1, doc);
498  }
499  db.flush();
500  TEST_EQUAL(db.get_collection_freq("takeaway"), 2);
501  Xapian::PostingIterator p = db.postlist_begin("takeaway");
502  TEST(p != db.postlist_end("takeaway"));
503  TEST_EQUAL(p.get_wdf(), 2);
504 }
505 
507 DEFINE_TESTCASE(databasemodified1, writable && !inmemory && !remote && !multi) {
508  // The inmemory backend doesn't support revisions.
509  //
510  // The remote backend doesn't work as expected here, I think due to
511  // test harness issues.
512  //
513  // With multi, DatabaseModifiedError doesn't trigger as easily.
515  Xapian::Document doc;
516  doc.set_data("cargo");
517  doc.add_term("abc");
518  doc.add_term("def");
519  doc.add_term("ghi");
520  const int N = 500;
521  for (int i = 0; i < N; ++i) {
522  db.add_document(doc);
523  }
524  db.commit();
525 
527  db.add_document(doc);
528  db.commit();
529 
530  db.add_document(doc);
531  db.commit();
532 
533  db.add_document(doc);
534  try {
535  TEST_EQUAL(*rodb.termlist_begin(N - 1), "abc");
536  FAIL_TEST("Expected DatabaseModifiedError wasn't thrown");
537  } catch (const Xapian::DatabaseModifiedError &) {
538  }
539 
540  try {
541  Xapian::Enquire enq(rodb);
542  enq.set_query(Xapian::Query("abc"));
543  Xapian::MSet mset = enq.get_mset(0, 10);
544  FAIL_TEST("Expected DatabaseModifiedError wasn't thrown");
545  } catch (const Xapian::DatabaseModifiedError &) {
546  }
547 }
548 
550 DEFINE_TESTCASE(qpmemoryleak1, writable && !inmemory) {
551  // Inmemory never throws DatabaseModifiedError.
553  Xapian::Document doc;
554 
555  doc.add_term("foo");
556  for (int i = 100; i < 120; ++i) {
557  doc.add_term(str(i));
558  }
559 
560  for (int j = 0; j < 50; ++j) {
561  wdb.add_document(doc);
562  }
563  wdb.commit();
564 
566  Xapian::QueryParser queryparser;
567  queryparser.set_database(database);
569  for (int k = 0; k < 1000; ++k) {
570  wdb.add_document(doc);
571  wdb.commit();
572  (void)queryparser.parse_query("1", queryparser.FLAG_PARTIAL);
573  }
574  SKIP_TEST("didn't manage to trigger DatabaseModifiedError");
575  );
576 }
577 
578 static void
580 {
581  const char * value0 =
582  "ABBCDEFGHIJKLMMNOPQQRSTTUUVVWXYZZaabcdefghhijjkllmnopqrsttuvwxyz";
583  const char * value1 =
584  "EMLEMMMMMMMNMMLMELEDNLEDMLMLDMLMLMLMEDGFHPOPBAHJIQJNGRKCGF";
585  while (*value0) {
586  Xapian::Document doc;
587  doc.add_value(0, string(1, *value0++));
588  if (*value1) {
589  doc.add_value(1, string(1, *value1++));
590  doc.add_term("K1");
591  }
592  db.add_document(doc);
593  }
594 }
595 
597 DEFINE_TESTCASE(msize1, generated) {
599  Xapian::Enquire enq(db);
600  enq.set_sort_by_value(1, false);
601  enq.set_collapse_key(0);
602  enq.set_query(Xapian::Query("K1"));
603 
604  Xapian::MSet mset = enq.get_mset(0, 60);
608  TEST_EQUAL(lb, ub);
609  TEST_EQUAL(lb, est);
610 
611  Xapian::MSet mset2 = enq.get_mset(50, 10, 1000);
615  TEST_EQUAL(lb2, ub2);
616  TEST_EQUAL(lb2, est2);
617  TEST_EQUAL(est, est2);
618 
619  Xapian::MSet mset3 = enq.get_mset(0, 10, 1000);
623  TEST_EQUAL(lb3, ub3);
624  TEST_EQUAL(lb3, est3);
625  TEST_EQUAL(est, est3);
626 }
627 
628 static void
630 {
631  const char * value0 = "AAABCDEEFGHIIJJKLLMNNOOPPQQRSTTUVWXYZ";
632  const char * value1 = "MLEMNMLMLMEDEDEMLEMLMLMLPOAHGF";
633  while (*value0) {
634  Xapian::Document doc;
635  doc.add_value(0, string(1, *value0++));
636  if (*value1) {
637  doc.add_value(1, string(1, *value1++));
638  doc.add_term("K1");
639  }
640  db.add_document(doc);
641  }
642 }
643 
645 DEFINE_TESTCASE(msize2, generated) {
647  Xapian::Enquire enq(db);
648  enq.set_sort_by_value(1, false);
649  enq.set_collapse_key(0);
650  enq.set_query(Xapian::Query("K1"));
651 
652  Xapian::MSet mset = enq.get_mset(0, 60);
656  TEST_EQUAL(lb, ub);
657  TEST_EQUAL(lb, est);
658 
659  Xapian::MSet mset2 = enq.get_mset(50, 10, 1000);
663  TEST_EQUAL(lb2, ub2);
664  TEST_EQUAL(lb2, est2);
665  TEST_EQUAL(est, est2);
666 
667  Xapian::MSet mset3 = enq.get_mset(0, 10, 1000);
671  TEST_EQUAL(lb3, ub3);
672  TEST_EQUAL(lb3, est3);
673  TEST_EQUAL(est, est3);
674 }
675 
676 static void
678 {
679  for (int n = 1; n != 50; ++n) {
680  Xapian::Document doc;
681  for (int i = 1; i != 50; ++i) {
682  if (n % i == 0)
683  doc.add_term("N" + str(i));
684  }
685  db.add_document(doc);
686  }
687 }
688 
690 DEFINE_TESTCASE(xordecay1, generated) {
692  Xapian::Enquire enq(db);
694  Xapian::Query("N10"),
696  Xapian::Query("N2"),
697  Xapian::Query("N3"))));
698  Xapian::MSet mset1 = enq.get_mset(0, 1);
699  Xapian::MSet msetall = enq.get_mset(0, db.get_doccount());
700 
701  TEST(mset_range_is_same(mset1, 0, msetall, 0, mset1.size()));
702 }
703 
704 static void
706 {
707  const char * p = "VJ=QC]LUNTaARLI;715RR^];A4O=P4ZG<2CS4EM^^VS[A6QENR";
708  for (int d = 0; p[d]; ++d) {
709  int l = int(p[d] - '0');
710  Xapian::Document doc;
711  for (int n = 1; n < l; ++n) {
712  doc.add_term("N" + str(n));
713  if (n % (d + 1) == 0) {
714  doc.add_term("M" + str(n));
715  }
716  }
717  db.add_document(doc);
718  }
719 }
720 
722 DEFINE_TESTCASE(ordecay1, generated) {
724  Xapian::Enquire enq(db);
726  Xapian::Query("N20"),
727  Xapian::Query("N21")));
728 
729  Xapian::MSet msetall = enq.get_mset(0, db.get_doccount());
730  for (unsigned int i = 1; i < msetall.size(); ++i) {
731  Xapian::MSet submset = enq.get_mset(0, i);
732  TEST(mset_range_is_same(submset, 0, msetall, 0, submset.size()));
733  }
734 }
735 
739 DEFINE_TESTCASE(ordecay2, generated) {
741  Xapian::Enquire enq(db);
742  std::vector<Xapian::Query> q;
743  q.push_back(Xapian::Query("M20"));
744  q.push_back(Xapian::Query("N21"));
745  q.push_back(Xapian::Query("N22"));
747  Xapian::Query("N25"),
749  q.begin(),
750  q.end())));
751 
752  Xapian::MSet msetall = enq.get_mset(0, db.get_doccount());
753  for (unsigned int i = 1; i < msetall.size(); ++i) {
754  Xapian::MSet submset = enq.get_mset(0, i);
755  TEST(mset_range_is_same(submset, 0, msetall, 0, submset.size()));
756  }
757 }
758 
759 static void
761 {
762  static const unsigned t1[] = {2, 4, 6, 8, 10};
763  static const unsigned t2[] = {6, 7, 8, 11, 12, 13, 14, 15, 16, 17};
764  static const unsigned t3[] = {3, 7, 8, 11, 12, 13, 14, 15, 16, 17};
765 
766  for (unsigned i = 1; i <= 17; ++i) {
767  Xapian::Document doc;
768  db.replace_document(i, doc);
769  }
770  for (unsigned i : t1) {
771  Xapian::Document doc(db.get_document(i));
772  doc.add_term("T1");
773  db.replace_document(i, doc);
774  }
775  for (unsigned i : t2) {
776  Xapian::Document doc(db.get_document(i));
777  doc.add_term("T2");
778  if (i < 17) {
779  doc.add_term("T2_lowfreq");
780  }
781  doc.add_value(2, "1");
782  db.replace_document(i, doc);
783  }
784  for (unsigned i : t3) {
785  Xapian::Document doc(db.get_document(i));
786  doc.add_term("T3");
787  if (i < 17) {
788  doc.add_term("T3_lowfreq");
789  }
790  doc.add_value(3, "1");
791  db.replace_document(i, doc);
792  }
793 }
794 
798 DEFINE_TESTCASE(orcheck1, generated) {
800  Xapian::Enquire enq(db);
801  Xapian::Query q1("T1");
802  Xapian::Query q2("T2");
803  Xapian::Query q2l("T2_lowfreq");
804  Xapian::Query q3("T3");
805  Xapian::Query q3l("T3_lowfreq");
808 
809  tout << "Checking q2 OR q3\n";
812  mset_expect_order(enq.get_mset(0, db.get_doccount()), 8, 6);
813 
814  tout << "Checking q2l OR q3\n";
817  mset_expect_order(enq.get_mset(0, db.get_doccount()), 8, 6);
818 
819  tout << "Checking q2 OR q3l\n";
822  mset_expect_order(enq.get_mset(0, db.get_doccount()), 8, 6);
823 
824  tout << "Checking v2 OR q3\n";
827  mset_expect_order(enq.get_mset(0, db.get_doccount()), 8, 6);
828 
829  tout << "Checking q2 OR v3\n";
832  // Order of results in this one is different, because v3 gives no weight,
833  // both documents are in q2, and document 8 has a higher length.
834  mset_expect_order(enq.get_mset(0, db.get_doccount()), 6, 8);
835 
836 }
837 
842 DEFINE_TESTCASE(failedreplace1, chert || glass) {
844  Xapian::Document doc;
845  doc.add_term("foo");
846  db.add_document(doc);
847  Xapian::docid did = db.add_document(doc);
848  doc.add_term("abc");
849  doc.add_term(string(1000, 'm'));
850  doc.add_term("xyz");
852  db.commit();
853  TEST_EQUAL(db.get_doccount(), 0);
854  TEST_EQUAL(db.get_termfreq("foo"), 0);
855 }
856 
857 DEFINE_TESTCASE(failedreplace2, chert || glass) {
858  Xapian::WritableDatabase db(get_writable_database("apitest_simpledata"));
859  db.commit();
860  Xapian::doccount db_size = db.get_doccount();
861  Xapian::Document doc;
862  doc.set_data("wibble");
863  doc.add_term("foo");
864  doc.add_value(0, "seven");
865  db.add_document(doc);
866  Xapian::docid did = db.add_document(doc);
867  doc.add_term("abc");
868  doc.add_term(string(1000, 'm'));
869  doc.add_term("xyz");
870  doc.add_value(0, "six");
872  db.commit();
873  TEST_EQUAL(db.get_doccount(), db_size);
874  TEST_EQUAL(db.get_termfreq("foo"), 0);
875 }
876 
878 DEFINE_TESTCASE(phrase3, positional) {
879  Xapian::Database db = get_database("apitest_phrase");
880 
881  static const char * const phrase_words[] = { "phrase", "near" };
882  Xapian::Query q(Xapian::Query::OP_NEAR, phrase_words, phrase_words + 2, 12);
884 
885  Xapian::Enquire enquire(db);
886  enquire.set_query(q);
887  Xapian::MSet mset = enquire.get_mset(0, 5);
888 
889 }
890 
892 // Regression test for fix in 1.2.4.
893 DEFINE_TESTCASE(msetfirst2, backend) {
894  Xapian::Database db(get_database("apitest_simpledata"));
895  Xapian::Enquire enquire(db);
896  enquire.set_query(Xapian::Query("paragraph"));
897  Xapian::MSet mset;
898  // Before the fix, this tried to allocate too much memory.
899  mset = enquire.get_mset(0xfffffff0, 1);
900  TEST_EQUAL(mset.get_firstitem(), 0xfffffff0);
901  // Check that the number of documents gets clamped too.
902  mset = enquire.get_mset(1, 0xfffffff0);
903  TEST_EQUAL(mset.get_firstitem(), 1);
904  // Another regression test - MatchNothing used to give an MSet with
905  // get_firstitem() returning 0.
907  mset = enquire.get_mset(1, 1);
908  TEST_EQUAL(mset.get_firstitem(), 1);
909 }
910 
911 DEFINE_TESTCASE(bm25weight2, backend) {
912  Xapian::Database db(get_database("etext"));
913  Xapian::Enquire enquire(db);
914  enquire.set_query(Xapian::Query("the"));
915  enquire.set_weighting_scheme(Xapian::BM25Weight(0, 0, 0, 0, 1));
916  Xapian::MSet mset = enquire.get_mset(0, 100);
917  TEST_REL(mset.size(),>=,2);
918  double weight0 = mset[0].get_weight();
919  for (size_t i = 1; i != mset.size(); ++i) {
920  TEST_EQUAL(weight0, mset[i].get_weight());
921  }
922 }
923 
924 DEFINE_TESTCASE(unigramlmweight2, backend) {
925  Xapian::Database db(get_database("etext"));
926  Xapian::Enquire enquire(db);
927  enquire.set_query(Xapian::Query("the"));
929  Xapian::MSet mset = enquire.get_mset(0, 100);
930  TEST_REL(mset.size(),>=,2);
931 }
932 
933 DEFINE_TESTCASE(tradweight2, backend) {
934  Xapian::Database db(get_database("etext"));
935  Xapian::Enquire enquire(db);
936  enquire.set_query(Xapian::Query("the"));
938  Xapian::MSet mset = enquire.get_mset(0, 100);
939  TEST_REL(mset.size(),>=,2);
940  double weight0 = mset[0].get_weight();
941  for (size_t i = 1; i != mset.size(); ++i) {
942  TEST_EQUAL(weight0, mset[i].get_weight());
943  }
944 }
945 
946 // Regression test for bug fix in 1.2.9.
947 DEFINE_TESTCASE(emptydb1, backend) {
948  Xapian::Database db(get_database(string()));
949  static const Xapian::Query::op ops[] = {
961  };
962  for (Xapian::Query::op op : ops) {
963  tout << op << endl;
964  Xapian::Enquire enquire(db);
966  enquire.set_query(query);
967  Xapian::MSet mset = enquire.get_mset(0, 10);
971  }
972 }
973 
980 DEFINE_TESTCASE(multiargop1, backend) {
981  Xapian::Database db(get_database("apitest_simpledata"));
982  static const struct { unsigned hits; Xapian::Query::op op; } tests[] = {
983  { 0, Xapian::Query::OP_AND },
984  { 6, Xapian::Query::OP_OR },
986  { 5, Xapian::Query::OP_XOR },
989  { 0, Xapian::Query::OP_NEAR },
993  { 6, Xapian::Query::OP_MAX }
994  };
995  static const char* terms[] = {"two", "all", "paragraph", "banana"};
996  Xapian::Enquire enquire(db);
997  for (auto& test : tests) {
998  Xapian::Query::op op = test.op;
999  Xapian::doccount hits = test.hits;
1000  tout << op << " should give " << hits << " hits\n";
1001  Xapian::Query query(op, terms, terms + 4);
1002  enquire.set_query(query);
1003  Xapian::MSet mset = enquire.get_mset(0, 10);
1004  TEST_EQUAL(mset.get_matches_estimated(), hits);
1005  TEST_EQUAL(mset.get_matches_upper_bound(), hits);
1006  TEST_EQUAL(mset.get_matches_lower_bound(), hits);
1007  }
1008 }
1009 
1011 // Regression test for bug fixed in 1.3.1 and 1.2.11.
1012 DEFINE_TESTCASE(stubdb7, !backend) {
1014  Xapian::Database("nosuchdirectory", Xapian::DB_BACKEND_STUB));
1016  Xapian::WritableDatabase("nosuchdirectory",
1018 }
1019 
1021 // This runs for multi_* too, so serves to check that we get the same weights
1022 // with multiple databases as without.
1023 DEFINE_TESTCASE(msetweights1, backend) {
1024  Xapian::Database db = get_database("apitest_simpledata");
1025  Xapian::Enquire enq(db);
1027  Xapian::Query("paragraph"),
1028  Xapian::Query("word"));
1029  enq.set_query(q);
1030  // 5 documents match, and the 4th and 5th have the same weight, so ask for
1031  // 4 as that's a good test that we get the right one in this case.
1032  Xapian::MSet mset = enq.get_mset(0, 4);
1033 
1034  static const struct { Xapian::docid did; double wt; } expected[] = {
1035  { 2, 1.2058248004573934864 },
1036  { 4, 0.81127876655507624726 },
1037  { 1, 0.17309550762546158098 },
1038  { 3, 0.14609528172558261527 }
1039  };
1040 
1041  TEST_EQUAL(mset.size(), sizeof(expected) / sizeof(expected[0]));
1042  for (size_t i = 0; i < mset.size(); ++i) {
1043  TEST_EQUAL(*mset[i], expected[i].did);
1044  TEST_EQUAL_DOUBLE(mset[i].get_weight(), expected[i].wt);
1045  }
1046 
1047  // Now test a query which matches only even docids, so in the multi case
1048  // one subdatabase doesn't match.
1049  enq.set_query(Xapian::Query("one"));
1050  mset = enq.get_mset(0, 3);
1051 
1052  static const struct { Xapian::docid did; double wt; } expected2[] = {
1053  { 6, 0.73354729848273669823 },
1054  { 2, 0.45626501034348893038 }
1055  };
1056 
1057  TEST_EQUAL(mset.size(), sizeof(expected2) / sizeof(expected2[0]));
1058  for (size_t i = 0; i < mset.size(); ++i) {
1059  TEST_EQUAL(*mset[i], expected2[i].did);
1060  TEST_EQUAL_DOUBLE(mset[i].get_weight(), expected2[i].wt);
1061  }
1062 }
1063 
1064 DEFINE_TESTCASE(itorskiptofromend1, backend) {
1065  Xapian::Database db = get_database("apitest_simpledata");
1066 
1068  t.skip_to("zzzzz");
1069  TEST(t == db.termlist_end(1));
1070  // This worked in 1.2.x but segfaulted in 1.3.1.
1071  t.skip_to("zzzzzz");
1072 
1074  p.skip_to(99999);
1075  TEST(p == db.postlist_end("one"));
1076  // This segfaulted prior to 1.3.2.
1077  p.skip_to(999999);
1078 
1080  i.skip_to(99999);
1081  TEST(i == db.positionlist_end(6, "one"));
1082  // This segfaulted prior to 1.3.2.
1083  i.skip_to(999999);
1084 
1086  v.skip_to(99999);
1087  TEST(v == db.valuestream_end(1));
1088  // These segfaulted prior to 1.3.2.
1089  v.skip_to(999999);
1090  v.check(9999999);
1091 }
1092 
1094 // Regression test for bug fixed in 1.2.17 and 1.3.2 - the size gets fixed
1095 // but the uncorrected size was passed to the base file. Also, abort() was
1096 // called on 0.
1097 DEFINE_TESTCASE(blocksize1, chert || glass) {
1098  string db_dir = "." + get_dbtype();
1099  mkdir(db_dir.c_str(), 0755);
1100  db_dir += "/db__blocksize1";
1101  int flags;
1102  if (get_dbtype() == "chert") {
1104  } else {
1106  }
1107  static const unsigned bad_sizes[] = {
1108  65537, 8000, 2000, 1024, 16, 7, 3, 1, 0
1109  };
1110  for (size_t i = 0; i < sizeof(bad_sizes) / sizeof(bad_sizes[0]); ++i) {
1111  size_t block_size = bad_sizes[i];
1112  rm_rf(db_dir);
1113  Xapian::WritableDatabase db(db_dir, flags, block_size);
1114  Xapian::Document doc;
1115  doc.add_term("XYZ");
1116  doc.set_data("foo");
1117  db.add_document(doc);
1118  db.commit();
1119  }
1120 }
1121 
1123 DEFINE_TESTCASE(notermlist1, glass) {
1124  string db_dir = "." + get_dbtype();
1125  mkdir(db_dir.c_str(), 0755);
1126  db_dir += "/db__notermlist1";
1128  if (get_dbtype() == "chert") {
1129  flags |= Xapian::DB_BACKEND_CHERT;
1130  } else {
1131  flags |= Xapian::DB_BACKEND_GLASS;
1132  }
1133  rm_rf(db_dir);
1134  Xapian::WritableDatabase db(db_dir, flags);
1135  Xapian::Document doc;
1136  doc.add_term("hello");
1137  doc.add_value(42, "answer");
1138  db.add_document(doc);
1139  db.commit();
1140  TEST(!file_exists(db_dir + "/termlist.glass"));
1142 }
1143 
1145 DEFINE_TESTCASE(newfreelistblock1, writable) {
1146  Xapian::Document doc;
1147  doc.add_term("foo");
1148  for (int i = 100; i < 120; ++i) {
1149  doc.add_term(str(i));
1150  }
1151 
1153  for (int j = 0; j < 50; ++j) {
1154  wdb.add_document(doc);
1155  }
1156  wdb.commit();
1157 
1158  for (int k = 0; k < 1000; ++k) {
1159  wdb.add_document(doc);
1160  wdb.commit();
1161  }
1162 }
1163 
1169 DEFINE_TESTCASE(readonlyparentdir1, chert || glass) {
1170 #if !defined __WIN32__ && !defined __CYGWIN__ && !defined __OS2__
1171  string path = get_named_writable_database_path("readonlyparentdir1");
1172  // Fix permissions if the previous test was killed.
1173  (void)chmod(path.c_str(), 0700);
1174  mkdir(path.c_str(), 0777);
1175  mkdir((path + "/sub").c_str(), 0777);
1176  Xapian::WritableDatabase db = get_named_writable_database("readonlyparentdir1/sub");
1177  TEST(chmod(path.c_str(), 0500) == 0);
1178  try {
1179  Xapian::Document doc;
1180  doc.add_term("hello");
1181  doc.set_data("some text");
1182  db.add_document(doc);
1183  db.commit();
1184  } catch (...) {
1185  // Attempt to fix the permissions, otherwise things like "rm -rf" on
1186  // the source tree will fail.
1187  (void)chmod(path.c_str(), 0700);
1188  throw;
1189  }
1190  TEST(chmod(path.c_str(), 0700) == 0);
1191 #endif
1192 }
1193 
1194 static void
1196 {
1197  Xapian::Document doc;
1198  doc.add_posting("hurricane", 199881);
1199  doc.add_posting("hurricane", 203084);
1200  doc.add_posting("katrina", 199882);
1201  doc.add_posting("katrina", 202473);
1202  doc.add_posting("katrina", 203085);
1203  db.add_document(doc);
1204 }
1205 
1207 DEFINE_TESTCASE(phrasebug1, generated && positional) {
1208  Xapian::Database db = get_database("phrasebug1", make_phrasebug1_db);
1209  static const char * const qterms[] = { "katrina", "hurricane" };
1210  Xapian::Enquire e(db);
1211  Xapian::Query q(Xapian::Query::OP_PHRASE, qterms, qterms + 2, 5);
1212  e.set_query(q);
1213  Xapian::MSet mset = e.get_mset(0, 100);
1214  TEST_EQUAL(mset.size(), 0);
1215  static const char * const qterms2[] = { "hurricane", "katrina" };
1216  Xapian::Query q2(Xapian::Query::OP_PHRASE, qterms2, qterms2 + 2, 5);
1217  e.set_query(q2);
1218  mset = e.get_mset(0, 100);
1219  TEST_EQUAL(mset.size(), 1);
1220 }
1221 
1223 DEFINE_TESTCASE(retrylock1, writable && path) {
1224  // FIXME: Can't see an easy way to test this for remote databases - the
1225  // harness doesn't seem to provide a suitable way to reopen a remote.
1226 #if defined HAVE_FORK && defined HAVE_SOCKETPAIR
1227  int fds[2];
1228  if (socketpair(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, PF_UNSPEC, fds) < 0) {
1229  FAIL_TEST("socketpair() failed");
1230  }
1231  if (fds[1] >= FD_SETSIZE)
1232  SKIP_TEST("socketpair() gave fd >= FD_SETSIZE");
1233  if (fcntl(fds[1], F_SETFL, O_NONBLOCK) < 0)
1234  FAIL_TEST("fcntl() failed to set O_NONBLOCK");
1235  pid_t child = fork();
1236  if (child == -1)
1237  FAIL_TEST("fork() failed");
1238  if (child == 0) {
1239  // Wait for signal that parent has opened the database.
1240  char ch;
1241  while (read(fds[0], &ch, 1) < 0) { }
1242 
1243  try {
1246  if (write(fds[0], "y", 1)) { }
1247  } catch (const Xapian::DatabaseLockError &) {
1248  if (write(fds[0], "l", 1)) { }
1249  } catch (const Xapian::Error &e) {
1250  const string & m = e.get_description();
1251  if (write(fds[0], m.data(), m.size())) { }
1252  } catch (...) {
1253  if (write(fds[0], "o", 1)) { }
1254  }
1255  _exit(0);
1256  }
1257 
1258  close(fds[0]);
1259 
1261  if (write(fds[1], "", 1) != 1)
1262  FAIL_TEST("Failed to signal to child process");
1263 
1264  char result[256];
1265  int r = read(fds[1], result, sizeof(result));
1266  if (r == -1) {
1267  if (errno == EAGAIN) {
1268  // Good.
1269  result[0] = 'y';
1270  } else {
1271  // Error.
1272  tout << "errno=" << errno << ": " << errno_to_string(errno) << endl;
1273  result[0] = 'e';
1274  }
1275  r = 1;
1276  } else if (r >= 1) {
1277  if (result[0] == 'y') {
1278  // Child process managed to also get write lock!
1279  result[0] = '!';
1280  }
1281  } else {
1282  // EOF.
1283  result[0] = 'z';
1284  r = 1;
1285  }
1286 
1287  try {
1288  db.close();
1289  } catch (...) {
1290  kill(child, SIGKILL);
1291  int status;
1292  while (waitpid(child, &status, 0) < 0) {
1293  if (errno != EINTR) break;
1294  }
1295  throw;
1296  }
1297 
1298  if (result[0] == 'y') {
1299 retry:
1300  struct timeval tv;
1301  tv.tv_sec = 3;
1302  tv.tv_usec = 0;
1303  fd_set fdset;
1304  FD_ZERO(&fdset);
1305  FD_SET(fds[1], &fdset);
1306  int sr = select(fds[1] + 1, &fdset, NULL, NULL, &tv);
1307  if (sr == 0) {
1308  // Timed out.
1309  result[0] = 'T';
1310  r = 1;
1311  } else if (sr == -1) {
1312  if (errno == EINTR || errno == EAGAIN)
1313  goto retry;
1314  tout << "select() failed with errno=" << errno << ": "
1315  << errno_to_string(errno) << endl;
1316  result[0] = 'S';
1317  r = 1;
1318  } else {
1319  r = read(fds[1], result, sizeof(result));
1320  if (r == -1) {
1321  // Error.
1322  tout << "read() failed with errno=" << errno << ": "
1323  << errno_to_string(errno) << endl;
1324  result[0] = 'R';
1325  r = 1;
1326  } else if (r == 0) {
1327  // EOF.
1328  result[0] = 'Z';
1329  r = 1;
1330  }
1331  }
1332  }
1333 
1334  close(fds[1]);
1335 
1336  kill(child, SIGKILL);
1337  int status;
1338  while (waitpid(child, &status, 0) < 0) {
1339  if (errno != EINTR) break;
1340  }
1341 
1342  tout << string(result, r) << endl;
1343  TEST_EQUAL(result[0], 'y');
1344 #endif
1345 }
1346 
1347 // Opening a WritableDatabase with low fds available - it should avoid them.
1348 DEFINE_TESTCASE(dbfilefd012, chert || glass) {
1349 #if !defined __WIN32__ && !defined __CYGWIN__ && !defined __OS2__
1350  int oldfds[3];
1351  for (int i = 0; i < 3; ++i) {
1352  oldfds[i] = dup(i);
1353  }
1354  try {
1355  for (int j = 0; j < 3; ++j) {
1356  close(j);
1357  TEST_REL(lseek(j, 0, SEEK_CUR), <, 0);
1358  TEST_EQUAL(errno, EBADF);
1359  }
1360 
1362 
1363  // Check we didn't use any of those low fds for tables, as that risks
1364  // data corruption if some other code in the same process tries to
1365  // write to them (see #651).
1366  for (int fd = 0; fd < 3; ++fd) {
1367  // Check that the fd is still closed, or isn't open O_RDWR (the
1368  // lock file gets opened O_WRONLY), or it's a pipe (if we're using
1369  // a child process to hold a non-OFD fcntl lock).
1370  int flags = fcntl(fd, F_GETFL);
1371  if (flags == -1) {
1372  TEST_EQUAL(errno, EBADF);
1373  } else if ((flags & O_ACCMODE) != O_RDWR) {
1374  // OK.
1375  } else {
1376  struct stat sb;
1377  TEST_NOT_EQUAL(fstat(fd, &sb), -1);
1378 #ifdef S_ISSOCK
1379  TEST(S_ISSOCK(sb.st_mode));
1380 #else
1381  // If we can't check it is a socket, at least check it is not a
1382  // regular file.
1383  TEST(!S_ISREG(sb.st_mode));
1384 #endif
1385  }
1386  }
1387  } catch (...) {
1388  for (int j = 0; j < 3; ++j) {
1389  dup2(oldfds[j], j);
1390  close(oldfds[j]);
1391  }
1392  throw;
1393  }
1394 
1395  for (int j = 0; j < 3; ++j) {
1396  dup2(oldfds[j], j);
1397  close(oldfds[j]);
1398  }
1399 #endif
1400 }
1401 
1403 DEFINE_TESTCASE(cursorbug1, chert || glass) {
1406  Xapian::Enquire enq(db);
1408  Xapian::MSet mset;
1409  // The original problem triggers for chert and glass on repeat==7.
1410  for (int repeat = 0; repeat < 10; ++repeat) {
1411  tout.str(string());
1412  tout << "iteration #" << repeat << endl;
1413 
1414  const int ITEMS = 10;
1415  int free_id = db.get_doccount();
1416  int offset = max(free_id, ITEMS * 2) - (ITEMS * 2);
1417  int limit = offset + (ITEMS * 2);
1418 
1419  mset = enq.get_mset(offset, limit);
1420  for (Xapian::MSetIterator m1 = mset.begin(); m1 != mset.end(); ++m1) {
1421  (void)m1.get_document().get_value(0);
1422  }
1423 
1424  for (int i = free_id; i <= free_id + ITEMS; ++i) {
1425  Xapian::Document doc;
1426  const string & id = str(i);
1427  string qterm = "Q" + id;
1428  doc.add_value(0, id);
1429  doc.add_boolean_term(qterm);
1430  wdb.replace_document(qterm, doc);
1431  }
1432  wdb.commit();
1433 
1434  db.reopen();
1435  mset = enq.get_mset(offset, limit);
1436  for (Xapian::MSetIterator m2 = mset.begin(); m2 != mset.end(); ++m2) {
1437  (void)m2.get_document().get_value(0);
1438  }
1439  }
1440 }
1441 
1442 // Regression test for #674, fixed in 1.2.21 and 1.3.3.
1443 DEFINE_TESTCASE(sortvalue2, backend) {
1444  Xapian::Database db = get_database("apitest_simpledata");
1445  db.add_database(get_database("apitest_simpledata2"));
1446  Xapian::Enquire enq(db);
1448  enq.set_sort_by_value(0, false);
1449  Xapian::MSet mset = enq.get_mset(0, 50);
1450 
1451  // Check all results are in key order - the bug was that they were sorted
1452  // by docid instead with multiple remote databases.
1453  string old_key;
1454  for (Xapian::MSetIterator i = mset.begin(); i != mset.end(); ++i) {
1455  string key = db.get_document(*i).get_value(0);
1456  TEST(old_key <= key);
1457  swap(old_key, key);
1458  }
1459 }
1460 
1462 DEFINE_TESTCASE(enquiregetquery1, backend) {
1463  Xapian::Database db = get_database("apitest_simpledata");
1464  Xapian::Enquire enq(db);
1465  TEST_EQUAL(enq.get_query().get_description(), "Query()");
1466 }
1467 
1468 DEFINE_TESTCASE(embedded1, singlefile) {
1469  // In reality you should align the embedded database to a multiple of
1470  // database block size, but any offset is meant to work.
1471  off_t offset = 1234;
1472 
1473  Xapian::Database db = get_database("apitest_simpledata");
1474  const string & db_path = get_database_path("apitest_simpledata");
1475  const string & tmp_path = db_path + "-embedded";
1476  ofstream out(tmp_path, fstream::trunc|fstream::binary);
1477  out.seekp(offset);
1478  out << ifstream(db_path, fstream::binary).rdbuf();
1479  out.close();
1480 
1481  {
1482  int fd = open(tmp_path.c_str(), O_RDONLY|O_BINARY);
1483  lseek(fd, offset, SEEK_SET);
1484  Xapian::Database db_embedded(fd);
1485  TEST_EQUAL(db.get_doccount(), db_embedded.get_doccount());
1486  }
1487 
1488  {
1489  int fd = open(tmp_path.c_str(), O_RDONLY|O_BINARY);
1490  lseek(fd, offset, SEEK_SET);
1491  size_t check_errors =
1493  TEST_EQUAL(check_errors, 0);
1494  }
1495 }
1496 
1498 DEFINE_TESTCASE(exactxor1, backend) {
1499  Xapian::Database db = get_database("apitest_simpledata");
1500  Xapian::Enquire enq(db);
1501 
1502  static const char * const words[4] = {
1503  "blank", "test", "paragraph", "banana"
1504  };
1505  Xapian::Query q(Xapian::Query::OP_XOR, words, words + 4);
1506  enq.set_query(q);
1508  Xapian::MSet mset = enq.get_mset(0, 0);
1509  // A reversed conditional gave us 5 in this case.
1511  // Test improved lower bound in 1.3.7 (earlier versions gave 0).
1513 
1514  static const char * const words2[4] = {
1515  "queri", "test", "paragraph", "word"
1516  };
1517  Xapian::Query q2(Xapian::Query::OP_XOR, words2, words2 + 4);
1518  enq.set_query(q2);
1520  mset = enq.get_mset(0, 0);
1521  // A reversed conditional gave us 6 in this case.
1523  // Test improved lower bound in 1.3.7 (earlier versions gave 0).
1525 }
1526 
1528 DEFINE_TESTCASE(getrevision1, chert || glass) {
1530  TEST_EQUAL(db.get_revision(), 0);
1531  db.commit();
1532  TEST_EQUAL(db.get_revision(), 0);
1533  Xapian::Document doc;
1534  doc.add_term("hello");
1535  db.add_document(doc);
1536  TEST_EQUAL(db.get_revision(), 0);
1537  db.commit();
1538  TEST_EQUAL(db.get_revision(), 1);
1539  db.commit();
1540  TEST_EQUAL(db.get_revision(), 1);
1541  db.add_document(doc);
1542  db.commit();
1543  TEST_EQUAL(db.get_revision(), 2);
1544 }
1545 
1547 DEFINE_TESTCASE(getrevision2, glass) {
1548  Xapian::Database db;
1549  TEST_EQUAL(db.get_revision(), 0);
1550  Xapian::Database wdb;
1551  TEST_EQUAL(wdb.get_revision(), 0);
1552 }
1553 
1555 DEFINE_TESTCASE(getdocumentlazy1, backend) {
1556  Xapian::Database db = get_database("apitest_simpledata");
1558  Xapian::Document doc = db.get_document(2);
1559  TEST_EQUAL(doc.get_data(), doc_lazy.get_data());
1560  TEST_EQUAL(doc.get_value(0), doc_lazy.get_value(0));
1561 }
1562 
1564 DEFINE_TESTCASE(getdocumentlazy2, backend) {
1565  Xapian::Database db = get_database("apitest_simpledata");
1566  Xapian::Document doc;
1567  try {
1569  } catch (const Xapian::DocNotFoundError&) {
1570  // DOC_ASSUME_VALID is really just a hint, so ignoring is OK (the
1571  // remote backend currently does).
1572  }
1573  TEST(doc.get_data().empty());
1575  doc = db.get_document(db.get_lastdocid() + 1);
1576  );
1577 }
1578 
1579 static void
1581 {
1582  Xapian::Document doc;
1583  doc.add_term("foo");
1584  doc.add_boolean_term("bar");
1585  db.add_document(doc);
1586  Xapian::Document doc2;
1587  doc2.add_posting("foo", 0, 2);
1588  doc2.add_term("foo2");
1589  doc2.add_boolean_term("baz");
1590  doc2.add_boolean_term("baz2");
1591  db.add_document(doc2);
1592 }
1593 
1594 DEFINE_TESTCASE(getuniqueterms1, generated) {
1595  Xapian::Database db =
1596  get_database("uniqterms_gt_doclen", gen_uniqterms_gt_doclen_db);
1597 
1598  auto unique1 = db.get_unique_terms(1);
1599  TEST_REL(unique1, <=, db.get_doclength(1));
1600  TEST_REL(unique1, <, db.get_document(1).termlist_count());
1601  // Ideally it'd be equal to 1, and in this case it is, but the current
1602  // backends can't always efficiently ensure an exact answer.
1603  TEST_REL(unique1, >=, 1);
1604 
1605  auto unique2 = db.get_unique_terms(2);
1606  TEST_REL(unique2, <=, db.get_doclength(2));
1607  TEST_REL(unique2, <, db.get_document(2).termlist_count());
1608  // Ideally it'd be equal to 2, but the current backends can't always
1609  // efficiently ensure an exact answer and here it is actually 3.
1610  TEST_REL(unique2, >=, 2);
1611 }
1612 
1619 DEFINE_TESTCASE(nopositionbug1, generated) {
1620  Xapian::Database db =
1621  get_database("uniqterms_gt_doclen", gen_uniqterms_gt_doclen_db);
1622 
1623  // Test both orders.
1624  static const char* const terms1[] = { "foo", "baz" };
1625  static const char* const terms2[] = { "baz", "foo" };
1626 
1627  Xapian::Enquire enq(db);
1629  begin(terms1), end(terms1), 10));
1630  TEST_EQUAL(enq.get_mset(0, 5).size(), 0);
1631 
1633  begin(terms2), end(terms2), 10));
1634  TEST_EQUAL(enq.get_mset(0, 5).size(), 0);
1635 
1637  begin(terms1), end(terms1), 10));
1638  TEST_EQUAL(enq.get_mset(0, 5).size(), 0);
1639 
1641  begin(terms2), end(terms2), 10));
1642  TEST_EQUAL(enq.get_mset(0, 5).size(), 0);
1643 
1644  // Exercise exact phrase case too.
1646  begin(terms1), end(terms1), 2));
1647  TEST_EQUAL(enq.get_mset(0, 5).size(), 0);
1648 
1650  begin(terms2), end(terms2), 2));
1651  TEST_EQUAL(enq.get_mset(0, 5).size(), 0);
1652 }
1653 
1658 DEFINE_TESTCASE(checkatleast4, backend) {
1659  Xapian::Database db = get_database("apitest_simpledata");
1660  Xapian::Enquire enq(db);
1661  enq.set_query(Xapian::Query("paragraph"));
1662  // This used to cause access to an element in an empty vector.
1663  Xapian::MSet mset = enq.get_mset(0, 0, 4);
1664  TEST_EQUAL(mset.size(), 0);
1665 }
1666 
1668 DEFINE_TESTCASE(nodocs1, transactions && !remote) {
1669  {
1671  db.set_metadata("foo", "bar");
1672  db.commit();
1673  Xapian::Document doc;
1674  doc.add_term("baz");
1675  db.add_document(doc);
1676  db.commit();
1677  }
1678 
1679  size_t check_errors =
1682  TEST_EQUAL(check_errors, 0);
1683 }
1684 
1686 DEFINE_TESTCASE(splitpostings1, writable) {
1688  Xapian::Document doc;
1689  // Add postings to create a split internally.
1690  for (Xapian::termpos pos = 0; pos <= 100; pos += 10) {
1691  doc.add_posting("foo", pos);
1692  }
1693  for (Xapian::termpos pos = 5; pos <= 100; pos += 20) {
1694  doc.add_posting("foo", pos);
1695  }
1696  db.add_document(doc);
1697  db.commit();
1698 
1699  Xapian::termpos expect = 0;
1700  Xapian::termpos pos = 0;
1701  for (auto p = db.positionlist_begin(1, "foo");
1702  p != db.positionlist_end(1, "foo"); ++p) {
1703  TEST_REL(expect, <=, 100);
1704  pos = *p;
1705  TEST_EQUAL(pos, expect);
1706  expect += 5;
1707  if (expect % 20 == 15) expect += 5;
1708  }
1709  TEST_EQUAL(pos, 100);
1710 }
1711 
1713 DEFINE_TESTCASE(multidb1, backend) {
1714  Xapian::Database db;
1715  TEST_EQUAL(db.size(), 0);
1716  Xapian::Database db2 = get_database("apitest_simpledata");
1717  TEST(db2.size() != 0);
1718  db.add_database(db2);
1719  TEST_EQUAL(db.size(), db2.size());
1720  db.add_database(db2);
1721  // Regression test for bug introduced and fixed in git master before 1.5.0.
1722  // Adding a multi database to an empty database incorrectly worked just
1723  // like assigning the database object. The list of shards is now copied
1724  // instead.
1725  TEST_EQUAL(db.size(), db2.size() * 2);
1727  TEST_EQUAL(db.size(), db2.size() * 2);
1728 }
int close(FD &fd)
Definition: fd.h:63
Xapian::doccount size() const
Return number of items in this MSet object.
Definition: omenquire.cc:318
Xapian::Document get_document(Xapian::docid did) const
Get a document from the database, given its document id.
Definition: omdatabase.cc:490
Xapian::docid add_document(const Xapian::Document &document)
Add a new document to the database.
Definition: omdatabase.cc:899
PositionIterator positionlist_end(Xapian::docid, const std::string &) const
Corresponding end iterator to positionlist_begin().
Definition: database.h:252
Run multiple tests for different backends.
void add_value(Xapian::valueno slot, const std::string &value)
Add a new value.
Definition: omdocument.cc:107
static size_t check(const std::string &path, int opts=0, std::ostream *out=NULL)
Check the integrity of a database or database table.
Definition: database.h:560
TermIterator termlist_begin(Xapian::docid did) const
An iterator pointing to the start of the termlist for a given document.
Definition: omdatabase.cc:198
Indicates an attempt to access a closed database.
Definition: error.h:1097
#define TEST(a)
Test a condition, without an additional explanation for failure.
Definition: testsuite.h:275
void skip_to(Xapian::termpos termpos)
Advance the iterator to term position termpos.
This class is used to access a database, or a group of databases.
Definition: database.h:68
void set_sort_by_value(Xapian::valueno sort_key, bool reverse)
Set the sorting to be by value only.
Definition: omenquire.cc:869
Xapian::termcount termlist_count() const
The length of the termlist - i.e.
Definition: omdocument.cc:191
static const Xapian::Query MatchAll
A query matching all documents.
Definition: query.h:75
Match documents which an odd number of subqueries match.
Definition: query.h:107
const int DB_CREATE
Create a new database.
Definition: constants.h:44
PositionIterator positionlist_begin(Xapian::docid did, const std::string &tname) const
An iterator pointing to the start of the position list for a given term in a given document...
Definition: omdatabase.cc:250
bool mset_range_is_same(const Xapian::MSet &mset1, unsigned int first1, const Xapian::MSet &mset2, unsigned int first2, unsigned int count)
Definition: testutils.cc:46
op
Query operators.
Definition: query.h:78
Xapian::termcount get_doclength_lower_bound() const
Get a lower bound on the length of a document in this DB.
Definition: omdatabase.cc:401
Xapian::doccount get_matches_lower_bound() const
Lower bound on the total number of matching documents.
Definition: omenquire.cc:246
Xapian::WritableDatabase get_writable_database(const string &dbname)
Definition: apitest.cc:87
include <sys/socket.h> with portability workarounds.
Xapian::docid get_lastdocid() const
Get the highest document id which has been used in the database.
Definition: omdatabase.cc:279
Convert errno value to std::string, thread-safe if possible.
Build a Xapian::Query object from a user query string.
Definition: queryparser.h:770
#define O_BINARY
Definition: safefcntl.h:81
a generic test suite engine
bool reopen()
Re-open the database.
Definition: omdatabase.cc:125
C++ function versions of useful Unix commands.
WritableDatabase open()
Construct a WritableDatabase object for a new, empty InMemory database.
Definition: dbfactory.h:104
Class for iterating over document values.
Definition: valueiterator.h:40
Class representing a list of search results.
Definition: mset.h:44
void skip_to(const std::string &term)
Advance the iterator to term term.
Xapian::WritableDatabase get_writable_database_again()
Definition: apitest.cc:125
STL namespace.
Pick the maximum weight of any subquery.
Definition: query.h:240
MSet get_mset(Xapian::doccount first, Xapian::doccount maxitems, Xapian::doccount checkatleast=0, const RSet *omrset=0, const MatchDecider *mdecider=0) const
Get (a portion of) the match set for the current query.
Definition: omenquire.cc:932
Convert types to std::string.
static double est(double l, double r, double n)
Definition: orpostlist.cc:306
const int DB_CREATE_OR_OPEN
Create database if it doesn&#39;t already exist.
Definition: constants.h:35
Utility functions for testing files.
Xapian::rev get_revision() const
Get the revision of the database.
Definition: omdatabase.cc:802
void replace_document(Xapian::docid did, const Xapian::Document &document)
Replace a given document in the database.
Definition: omdatabase.cc:949
void set_metadata(const std::string &key, const std::string &metadata)
Set the user-specified metadata associated with a given key.
Definition: omdatabase.cc:1061
#define false
Definition: header.h:9
Xapian::doccount get_doccount() const
Get the number of documents in the database.
Definition: omdatabase.cc:267
const int DOC_ASSUME_VALID
Assume document id is valid.
Definition: constants.h:280
DEFINE_TESTCASE(lockfileumask1, chert||glass)
Regression test - lockfile should honour umask, was only user-readable.
Definition: api_backend.cc:57
Xapian::totallength get_total_length() const
Get the total length of all the documents in the database.
Definition: omdatabase.cc:312
include <sys/stat.h> with portability enhancements
#define SOCK_CLOEXEC
Definition: safesyssocket.h:68
static void make_msize2_db(Xapian::WritableDatabase &db, const string &)
Definition: api_backend.cc:629
const int DB_BACKEND_GLASS
Use the glass backend.
Definition: constants.h:158
Xapian::WritableDatabase get_named_writable_database(const std::string &name, const std::string &source)
Definition: apitest.cc:93
Enable partial matching.
Definition: queryparser.h:829
test functionality of the Xapian API
void rm_rf(const string &filename)
Remove a directory and contents, just like the Unix "rm -rf" command.
Definition: unixcmds.cc:111
Xapian::doccount get_matches_upper_bound() const
Upper bound on the total number of matching documents.
Definition: omenquire.cc:262
Xapian::doclength get_avlength() const
Get the average length of the documents in the database.
Definition: omdatabase.cc:293
static void make_xordecay1_db(Xapian::WritableDatabase &db, const string &)
Definition: api_backend.cc:677
Class for iterating over a list of terms.
Definition: termiterator.h:41
unsigned XAPIAN_TERMCOUNT_BASE_TYPE termcount
A counts of terms.
Definition: types.h:72
#define TEST_REL(A, REL, B)
Test a relation holds,e.g. TEST_REL(a,>,b);.
Definition: testmacros.h:32
Class for iterating over a list of terms.
#define TEST_NOT_EQUAL(a, b)
Test for non-equality of two things.
Definition: testsuite.h:305
InvalidArgumentError indicates an invalid parameter value was passed to the API.
Definition: error.h:241
Xapian::termcount get_doclength_upper_bound() const
Get an upper bound on the length of a document in this DB.
Definition: omdatabase.cc:421
Base class for backend handling in test harness.
string get_database_path(const string &dbname)
Definition: apitest.cc:72
Class implementing a "boolean" weighting scheme.
Definition: weight.h:400
Xapian::doccount get_firstitem() const
Rank of first item in this MSet.
Definition: omenquire.cc:239
ValueIterator valuestream_end(Xapian::valueno) const
Return end iterator corresponding to valuestream_begin().
Definition: database.h:359
std::string get_named_writable_database_path(const std::string &name)
Definition: apitest.cc:99
DatabaseModifiedError indicates a database was modified.
Definition: error.h:539
DatabaseLockError indicates failure to lock a database.
Definition: error.h:493
Pick the best N subqueries and combine with OP_OR.
Definition: query.h:206
const int DB_OPEN
Open an existing database.
Definition: constants.h:50
This class provides read/write access to a database.
Definition: database.h:785
static void make_phrasebug1_db(Xapian::WritableDatabase &db, const string &)
std::ostringstream tout
The debug printing stream.
Definition: testsuite.cc:103
void errno_to_string(int e, string &s)
Iterator over a Xapian::MSet.
Definition: mset.h:351
Match only documents where all subqueries match near and in order.
Definition: query.h:152
Indicates an attempt to use a feature which is unavailable.
Definition: error.h:719
Match the first subquery taking extra weight from other subqueries.
Definition: query.h:118
Public interfaces for the Xapian library.
bool operator()(const Xapian::Document &) const
Decide whether we want this document to be in the MSet.
Definition: api_backend.cc:418
Match like OP_AND but only taking weight from the first subquery.
Definition: query.h:128
bool locked() const
Test if this database is currently locked for writing.
Definition: omdatabase.cc:790
#define S_ISREG(ST_MODE)
Definition: safesysstat.h:60
void delete_document(Xapian::docid did)
Delete a document from the database.
Definition: omdatabase.cc:922
#define TEST_EXCEPTION(TYPE, CODE)
Check that CODE throws exactly Xapian exception TYPE.
Definition: testutils.h:109
bool was_called() const
Definition: api_backend.cc:423
Match only documents where a value slot is within a given range.
Definition: query.h:158
std::string get_dbtype()
Definition: apitest.cc:42
string str(int value)
Convert int to std::string.
Definition: str.cc:90
MSetIterator begin() const
Return iterator pointing to the first item in this MSet.
Definition: mset.h:607
MSetIterator end() const
Return iterator pointing to just after the last item in this MSet.
Definition: mset.h:612
Xapian::termcount get_doclength(Xapian::docid did) const
Get the length of a document.
Definition: omdatabase.cc:461
void commit()
Commit any pending modifications made to the database.
Definition: omdatabase.cc:854
Xapian::Weight subclass implementing the traditional probabilistic formula.
Definition: weight.h:746
Xapian::Database get_writable_database_as_database()
Definition: apitest.cc:119
Class for iterating over term positions.
static const test_desc tests[]
The lists of tests to perform.
Indicates an attempt to access a database not present.
Definition: error.h:1055
Query parse_query(const std::string &query_string, unsigned flags=FLAG_DEFAULT, const std::string &default_prefix=std::string())
Parse a query.
Definition: queryparser.cc:161
TermIterator termlist_end(Xapian::docid) const
Corresponding end iterator to termlist_begin().
Definition: database.h:238
#define TEST_EQUAL_DOUBLE(a, b)
Test two doubles for near equality.
Definition: testsuite.h:295
#define SKIP_TEST_FOR_BACKEND(B)
Definition: apitest.h:75
ValueIterator valuestream_begin(Xapian::valueno slot) const
Return an iterator over the value in slot slot for each document.
Definition: omdatabase.cc:450
void add_database(const Database &database)
Add an existing database (or group of databases) to those accessed by this object.
Definition: omdatabase.cc:148
void set_query(const Xapian::Query &query, Xapian::termcount qlen=0)
Set the query to run.
Definition: omenquire.cc:793
Indicates an attempt to access a document not present in the database.
Definition: error.h:674
size_t size() const
Return number of shards in this Database object.
Definition: database.h:93
Match like OP_OR but weighting as if a single term.
Definition: query.h:230
std::string get_description() const
Return a string describing this object.
Definition: error.cc:93
Base class for matcher decision functor.
Definition: enquire.h:118
bool check(Xapian::docid docid)
Check if the specified docid occurs.
void add_posting(const std::string &tname, Xapian::termpos tpos, Xapian::termcount wdfinc=1)
Add an occurrence of a term at a particular position.
Definition: omdocument.cc:128
#define FAIL_TEST(MSG)
Fail the current testcase with message MSG.
Definition: testsuite.h:68
static void gen_uniqterms_gt_doclen_db(Xapian::WritableDatabase &db, const string &)
Match only documents which all subqueries match.
Definition: query.h:84
static Xapian::Query query(Xapian::Query::op op, const string &t1=string(), const string &t2=string(), const string &t3=string(), const string &t4=string(), const string &t5=string(), const string &t6=string(), const string &t7=string(), const string &t8=string(), const string &t9=string(), const string &t10=string())
Definition: api_anydb.cc:63
Xapian::Database get_database(const string &dbname)
Definition: apitest.cc:48
Xapian::doccount get_matches_estimated() const
Estimate of the total number of matching documents.
Definition: omenquire.cc:253
const Xapian::Query & get_query() const
Get the current query.
Definition: omenquire.cc:800
const int DB_BACKEND_STUB
Open a stub database file.
Definition: constants.h:179
void set_database(const Database &db)
Specify the database being searched.
Definition: queryparser.cc:141
#define SKIP_TEST(MSG)
Skip the current testcase with message MSG.
Definition: testsuite.h:74
std::string get_description() const
Return a string describing this object.
Definition: query.cc:232
This class provides an interface to the information retrieval system for the purpose of searching...
Definition: enquire.h:152
unsigned XAPIAN_DOCID_BASE_TYPE doccount
A count of documents.
Definition: types.h:38
Match only documents where all subqueries match near each other.
Definition: query.h:140
void flush()
Pre-1.1.0 name for commit().
Definition: database.h:927
All exceptions thrown by Xapian are subclasses of Xapian::Error.
Definition: error.h:43
Match documents which the first subquery matches but no others do.
Definition: query.h:99
Match documents which at least one subquery matches.
Definition: query.h:92
void skip_to(Xapian::docid did)
Advance the iterator to document did.
Xapian-specific test helper functions and macros.
unsigned XAPIAN_TERMPOS_BASE_TYPE termpos
A term position within a document or query.
Definition: types.h:83
static const Xapian::Query MatchNothing
A query matching no documents.
Definition: query.h:65
const int DB_NO_TERMLIST
When creating a database, don&#39;t create a termlist table.
Definition: constants.h:136
Definition: quest.cc:108
<unistd.h>, but with compat.
include <sys/wait.h>, with portability stuff.
void add_boolean_term(const std::string &term)
Add a boolean filter term to the document.
Definition: document.h:191
void mset_expect_order(const Xapian::MSet &A, Xapian::docid d1, Xapian::docid d2, Xapian::docid d3, Xapian::docid d4, Xapian::docid d5, Xapian::docid d6, Xapian::docid d7, Xapian::docid d8, Xapian::docid d9, Xapian::docid d10, Xapian::docid d11, Xapian::docid d12)
Definition: testutils.cc:225
static void make_msize1_db(Xapian::WritableDatabase &db, const string &)
Definition: api_backend.cc:579
const int DB_RETRY_LOCK
If the database is already locked, retry the lock.
Definition: constants.h:145
void set_weighting_scheme(const Weight &weight_)
Set the weighting scheme to use for queries.
Definition: omenquire.cc:819
unsigned XAPIAN_DOCID_BASE_TYPE docid
A unique identifier for a document.
Definition: types.h:52
Class representing a query.
Definition: query.h:46
std::string get_data() const
Get data stored in the document.
Definition: omdocument.cc:71
#define TEST_EQUAL(a, b)
Test for equality of two things.
Definition: testsuite.h:278
Xapian::termcount get_unique_terms(Xapian::docid did) const
Get the number of unique terms in document.
Definition: omdatabase.cc:476
PostingIterator postlist_end(const std::string &) const
Corresponding end iterator to postlist_begin().
Definition: database.h:225
Xapian::Weight subclass implementing the Language Model formula.
Definition: weight.h:1384
static void make_ordecay_db(Xapian::WritableDatabase &db, const string &)
Definition: api_backend.cc:705
void set_data(const std::string &data)
Set data stored in the document.
Definition: omdocument.cc:78
bool file_exists(const char *path)
Test if a file exists.
Definition: filetests.h:39
void set_collapse_key(Xapian::valueno collapse_key, Xapian::doccount collapse_max=1)
Set the collapse key to use for queries.
Definition: omenquire.cc:842
std::string get_value(Xapian::valueno slot) const
Get value by number.
Definition: omdocument.cc:64
virtual void close()
Close the database.
Definition: omdatabase.cc:138
include <fcntl.h>, but working around broken platforms.
Xapian::doccount get_termfreq(const std::string &tname) const
Get the number of documents in the database indexed by a given term.
Definition: omdatabase.cc:323
A handle representing a document in a Xapian database.
Definition: document.h:61
const int DB_BACKEND_CHERT
Use the chert backend.
Definition: constants.h:170
Xapian::Weight subclass implementing the BM25 probabilistic formula.
Definition: weight.h:513
UnimplementedError indicates an attempt to use an unimplemented feature.
Definition: error.h:325
PostingIterator postlist_begin(const std::string &tname) const
An iterator pointing to the start of the postlist for a given term.
Definition: omdatabase.cc:162
void skip_to(Xapian::docid docid_or_slot)
Advance the iterator to document id or value slot docid_or_slot.
void add_term(const std::string &tname, Xapian::termcount wdfinc=1)
Add a term to the document, without positional information.
Definition: omdocument.cc:140
Xapian::termcount get_collection_freq(const std::string &tname) const
Return the total number of occurrences of the given term.
Definition: omdatabase.cc:339
Xapian::termcount get_wdf_upper_bound(const std::string &term) const
Get an upper bound on the wdf of term term.
Definition: omdatabase.cc:435
static void make_orcheck_db(Xapian::WritableDatabase &db, const string &)
Definition: api_backend.cc:760
const int DBCHECK_SHOW_STATS
Show statistics for the B-tree.
Definition: constants.h:228