xapian-core  1.4.25
api_replicate.cc
Go to the documentation of this file.
1 
4 /* Copyright 2008 Lemur Consulting Ltd
5  * Copyright 2009,2010,2011,2012,2013,2014,2015,2016,2017,2020 Olly Betts
6  * Copyright 2010 Richard Boulton
7  * Copyright 2011 Dan Colish
8  *
9  * This program is free software; you can redistribute it and/or
10  * modify it under the terms of the GNU General Public License as
11  * published by the Free Software Foundation; either version 2 of the
12  * License, or (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
22  * USA
23  */
24 
25 #include <config.h>
26 
27 #include "api_replicate.h"
28 
29 #include <xapian.h>
30 #include "api/replication.h"
31 
32 #include "apitest.h"
33 #include "dbcheck.h"
34 #include "errno_to_string.h"
35 #include "fd.h"
36 #include "filetests.h"
37 #include "safedirent.h"
38 #include "safefcntl.h"
39 #include "safesysstat.h"
40 #include "safeunistd.h"
41 #include "setenv.h"
42 #include "testsuite.h"
43 #include "testutils.h"
44 #include "unixcmds.h"
45 
46 #include <sys/types.h>
47 
48 #include <cerrno>
49 #include <cstdlib>
50 #include <string>
51 
52 using namespace std;
53 
54 #ifdef XAPIAN_HAS_REMOTE_BACKEND
55 
56 static void rmtmpdir(const string & path) {
57  rm_rf(path);
58 }
59 
60 static void mktmpdir(const string & path) {
61  rmtmpdir(path);
62  if (mkdir(path.c_str(), 0700) == -1 && errno != EEXIST) {
63  FAIL_TEST("Can't make temporary directory");
64  }
65 }
66 
67 static off_t get_file_size(const string & path) {
68  off_t size = file_size(path);
69  if (errno) {
70  FAIL_TEST("Can't stat '" << path << "'");
71  }
72  return size;
73 }
74 
75 static size_t do_read(int fd, char * p, size_t desired)
76 {
77  size_t total = 0;
78  while (desired) {
79  ssize_t c = read(fd, p, desired);
80  if (c == 0) return total;
81  if (c < 0) {
82  if (errno == EINTR) continue;
83  FAIL_TEST("Error reading from file");
84  }
85  p += c;
86  total += c;
87  desired -= c;
88  }
89  return total;
90 }
91 
92 static void do_write(int fd, const char * p, size_t n)
93 {
94  while (n) {
95  ssize_t c = write(fd, p, n);
96  if (c < 0) {
97  if (errno == EINTR) continue;
98  FAIL_TEST("Error writing to file");
99  }
100  p += c;
101  n -= c;
102  }
103 }
104 
105 // Make a truncated copy of a file.
106 static off_t
107 truncated_copy(const string & srcpath, const string & destpath, off_t tocopy)
108 {
109  FD fdin(open(srcpath.c_str(), O_RDONLY | O_BINARY));
110  if (fdin == -1) {
111  FAIL_TEST("Open failed (when opening '" << srcpath << "')");
112  }
113 
114  FD fdout(open(destpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
115  if (fdout == -1) {
116  FAIL_TEST("Open failed (when creating '" << destpath << "')");
117  }
118 
119  const int BUFSIZE = 1024;
120  char buf[BUFSIZE];
121  size_t total_bytes = 0;
122  while (tocopy > 0) {
123  size_t thiscopy = tocopy > BUFSIZE ? BUFSIZE : tocopy;
124  size_t bytes = do_read(fdin, buf, thiscopy);
125  if (thiscopy != bytes) {
126  FAIL_TEST("Couldn't read desired number of bytes from changeset");
127  }
128  tocopy -= bytes;
129  total_bytes += bytes;
130  do_write(fdout, buf, bytes);
131  }
132 
133  if (close(fdout) == -1)
134  FAIL_TEST("Error closing file");
135 
136  return total_bytes;
137 }
138 
139 static void
140 get_changeset(const string & changesetpath,
141  Xapian::DatabaseMaster & master,
142  Xapian::DatabaseReplica & replica,
143  int expected_changesets,
144  int expected_fullcopies,
145  bool expected_changed,
146  bool full_copy = false)
147 {
148  FD fd(open(changesetpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666));
149  if (fd == -1) {
150  FAIL_TEST("Open failed (when creating a new changeset file at '"
151  << changesetpath << "')");
152  }
154  master.write_changesets_to_fd(fd,
155  full_copy ? "" : replica.get_revision_info(),
156  &info1);
157 
158  TEST_EQUAL(info1.changeset_count, expected_changesets);
159  TEST_EQUAL(info1.fullcopy_count, expected_fullcopies);
160  TEST_EQUAL(info1.changed, expected_changed);
161 }
162 
163 static int
164 apply_changeset(const string & changesetpath,
165  Xapian::DatabaseReplica & replica,
166  int expected_changesets,
167  int expected_fullcopies,
168  bool expected_changed)
169 {
170  FD fd(open(changesetpath.c_str(), O_RDONLY | O_BINARY));
171  if (fd == -1) {
172  FAIL_TEST("Open failed (when reading changeset file at '"
173  << changesetpath << "')");
174  }
175 
176  int count = 1;
177  replica.set_read_fd(fd);
180  bool client_changed = false;
181  while (replica.apply_next_changeset(&info2, 0)) {
182  ++count;
183  info1.changeset_count += info2.changeset_count;
184  info1.fullcopy_count += info2.fullcopy_count;
185  if (info2.changed)
186  client_changed = true;
187  }
188  info1.changeset_count += info2.changeset_count;
189  info1.fullcopy_count += info2.fullcopy_count;
190  if (info2.changed)
191  client_changed = true;
192 
193  TEST_EQUAL(info1.changeset_count, expected_changesets);
194  TEST_EQUAL(info1.fullcopy_count, expected_fullcopies);
195  TEST_EQUAL(client_changed, expected_changed);
196  return count;
197 }
198 
199 // Replicate from the master to the replica.
200 // Returns the number of changesets which were applied.
201 static int
203  Xapian::DatabaseReplica & replica,
204  const string & tempdir,
205  int expected_changesets,
206  int expected_fullcopies,
207  bool expected_changed,
208  bool full_copy = false)
209 {
210  string changesetpath = tempdir + "/changeset";
211  get_changeset(changesetpath, master, replica,
212  expected_changesets,
213  expected_fullcopies,
214  expected_changed,
215  full_copy);
216  return apply_changeset(changesetpath, replica,
217  expected_changesets,
218  expected_fullcopies,
219  expected_changed);
220 }
221 
222 // Check that the databases held at the given path are identical.
223 static void
224 check_equal_dbs(const string & masterpath, const string & replicapath)
225 {
226  Xapian::Database master(masterpath);
227  Xapian::Database replica(replicapath);
228 
229  TEST_EQUAL(master.get_uuid(), master.get_uuid());
230  dbcheck(replica, master.get_doccount(), master.get_lastdocid());
231 
232  for (Xapian::TermIterator t = master.allterms_begin();
233  t != master.allterms_end(); ++t) {
234  TEST_EQUAL(postlist_to_string(master, *t),
235  postlist_to_string(replica, *t));
236  }
237 }
238 
239 #define set_max_changesets(N) setenv("XAPIAN_MAX_CHANGESETS", #N, 1)
240 
244 };
245 
246 // Ensure that we don't leave generation of changesets on for the next
247 // testcase, even if this one exits with an exception.
248 #define UNSET_MAX_CHANGESETS_AFTERWARDS unset_max_changesets_helper_ ezlxq
249 
250 #endif
251 
252 // #######################################################################
253 // # Tests start here
254 
255 // Basic test of replication functionality.
256 DEFINE_TESTCASE(replicate1, replicas) {
257 #ifdef XAPIAN_HAS_REMOTE_BACKEND
259  string tempdir = ".replicatmp";
260  mktmpdir(tempdir);
261  string masterpath = get_named_writable_database_path("master");
262 
263  set_max_changesets(10);
264 
265  Xapian::Document doc1;
266  doc1.set_data(string("doc1"));
267  doc1.add_posting("doc", 1);
268  doc1.add_posting("one", 1);
269 
271  Xapian::DatabaseMaster master(masterpath);
272  string replicapath = tempdir + "/replica";
273  {
274  Xapian::DatabaseReplica replica(replicapath);
275 
276  // Add a document to the original database.
277  orig.add_document(doc1);
278  orig.commit();
279 
280  // Apply the replication - we don't have changesets stored, so this
281  // should just do a database copy, and return a count of 1.
282  int count = replicate(master, replica, tempdir, 0, 1, true);
283  TEST_EQUAL(count, 1);
284  {
285  Xapian::Database dbcopy(replicapath);
286  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
287  }
288 
289  // Repeating the replication should return a count of 1, since no
290  // further changes should need to be applied.
291  count = replicate(master, replica, tempdir, 0, 0, false);
292  TEST_EQUAL(count, 1);
293  {
294  Xapian::Database dbcopy(replicapath);
295  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
296  }
297  }
298  {
299  // Regression test - if the replica was reopened, a full copy always
300  // used to occur, whether it was needed or not. Fixed in revision
301  // #10117.
302  Xapian::DatabaseReplica replica(replicapath);
303  int count = replicate(master, replica, tempdir, 0, 0, false);
304  TEST_EQUAL(count, 1);
305  {
306  Xapian::Database dbcopy(replicapath);
307  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
308  }
309 
310  orig.add_document(doc1);
311  orig.commit();
312  orig.add_document(doc1);
313  orig.commit();
314 
315  count = replicate(master, replica, tempdir, 2, 0, true);
316  TEST_EQUAL(count, 3);
317  {
318  Xapian::Database dbcopy(replicapath);
319  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
320  }
321 
322  check_equal_dbs(masterpath, replicapath);
323 
324  // We need this inner scope to we close the replica before we remove
325  // the temporary directory on Windows.
326  }
327 
328  TEST_EQUAL(Xapian::Database::check(masterpath), 0);
329 
330  rmtmpdir(tempdir);
331 #endif
332 }
333 
334 // Test replication from a replicated copy.
335 DEFINE_TESTCASE(replicate2, replicas) {
336 #ifdef XAPIAN_HAS_REMOTE_BACKEND
337  SKIP_TEST_FOR_BACKEND("glass"); // Glass doesn't currently support this.
339 
340  string tempdir = ".replicatmp";
341  mktmpdir(tempdir);
342  string masterpath = get_named_writable_database_path("master");
343 
344  set_max_changesets(10);
345 
346  {
348  Xapian::DatabaseMaster master(masterpath);
349  string replicapath = tempdir + "/replica";
350  Xapian::DatabaseReplica replica(replicapath);
351 
352  Xapian::DatabaseMaster master2(replicapath);
353  string replica2path = tempdir + "/replica2";
354  Xapian::DatabaseReplica replica2(replica2path);
355 
356  // Add a document to the original database.
357  Xapian::Document doc1;
358  doc1.set_data(string("doc1"));
359  doc1.add_posting("doc", 1);
360  doc1.add_posting("one", 1);
361  orig.add_document(doc1);
362  orig.commit();
363 
364  // Apply the replication - we don't have changesets stored, so this
365  // should just do a database copy, and return a count of 1.
366  TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
367  check_equal_dbs(masterpath, replicapath);
368 
369  // Replicate from the replica.
370  TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
371  check_equal_dbs(masterpath, replica2path);
372 
373  orig.add_document(doc1);
374  orig.commit();
375  orig.add_document(doc1);
376  orig.commit();
377 
378  // Replicate from the replica - should have no changes.
379  TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 0, false), 1);
380  check_equal_dbs(replicapath, replica2path);
381 
382  // Replicate, and replicate from the replica - should have 2 changes.
383  TEST_EQUAL(replicate(master, replica, tempdir, 2, 0, 1), 3);
384  check_equal_dbs(masterpath, replicapath);
385  TEST_EQUAL(replicate(master2, replica2, tempdir, 2, 0, 1), 3);
386  check_equal_dbs(masterpath, replica2path);
387 
388  // Stop writing changesets, and make a modification
390  orig.close();
392  orig.add_document(doc1);
393  orig.commit();
394 
395  // Replication should do a full copy.
396  TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
397  check_equal_dbs(masterpath, replicapath);
398  TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
399  check_equal_dbs(masterpath, replica2path);
400 
401  // Start writing changesets, but only keep 1 in history, and make a
402  // modification.
404  orig.close();
406  orig.add_document(doc1);
407  orig.commit();
408 
409  // Replicate, and replicate from the replica - should have 1 changes.
410  TEST_EQUAL(replicate(master, replica, tempdir, 1, 0, 1), 2);
411  check_equal_dbs(masterpath, replicapath);
412  TEST_EQUAL(replicate(master2, replica2, tempdir, 1, 0, 1), 2);
413  check_equal_dbs(masterpath, replica2path);
414 
415  // Make two changes - only one changeset should be preserved.
416  orig.add_document(doc1);
417  orig.commit();
418 
419  // Replication should do a full copy, since one of the needed
420  // changesets is missing.
421 
422  // FIXME - the following tests are commented out because the backends
423  // don't currently tidy up old changesets correctly.
424  // TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
425  // check_equal_dbs(masterpath, replicapath);
426  // TEST_EQUAL(replicate(master2, replica2, tempdir, 0, 1, true), 1);
427  // check_equal_dbs(masterpath, replica2path);
428 
429  // We need this inner scope to we close the replicas before we remove
430  // the temporary directory on Windows.
431  }
432 
433  rmtmpdir(tempdir);
434 #endif
435 }
436 
437 #ifdef XAPIAN_HAS_REMOTE_BACKEND
438 static void
440  Xapian::DatabaseReplica & replica,
441  const string & tempdir,
442  int expected_changesets,
443  int expected_fullcopies,
444  bool expected_changed)
445 {
446  string changesetpath = tempdir + "/changeset";
447  get_changeset(changesetpath, master, replica,
448  1, 0, 1);
449 
450  // Try applying truncated changesets of various different lengths.
451  string brokenchangesetpath = tempdir + "/changeset_broken";
452  off_t filesize = get_file_size(changesetpath);
453  off_t len = 10;
454  off_t copylen;
455  while (len < filesize) {
456  copylen = truncated_copy(changesetpath, brokenchangesetpath, len);
457  TEST_EQUAL(copylen, len);
458  tout << "Trying replication with a changeset truncated to " << len <<
459  " bytes, from " << filesize << " bytes\n";
461  apply_changeset(brokenchangesetpath, replica,
462  expected_changesets, expected_fullcopies,
463  expected_changed));
464  if (len < 30 || len >= filesize - 10) {
465  // For lengths near the beginning and end, increment size by 1
466  ++len;
467  } else {
468  // Don't bother incrementing by small amounts in the middle of
469  // the changeset.
470  len += 1000;
471  if (len >= filesize - 10) {
472  len = filesize - 10;
473  }
474  }
475  }
476 }
477 #endif
478 
479 // Test changesets which are truncated (and therefore invalid).
480 DEFINE_TESTCASE(replicate3, replicas) {
481 #ifdef XAPIAN_HAS_REMOTE_BACKEND
483  string tempdir = ".replicatmp";
484  mktmpdir(tempdir);
485  string masterpath = get_named_writable_database_path("master");
486 
487  set_max_changesets(10);
488 
489  {
491  Xapian::DatabaseMaster master(masterpath);
492  string replicapath = tempdir + "/replica";
493  Xapian::DatabaseReplica replica(replicapath);
494 
495  // Add a document to the original database.
496  Xapian::Document doc1;
497  doc1.set_data(string("doc1"));
498  doc1.add_posting("doc", 1);
499  doc1.add_posting("one", 1);
500  orig.add_document(doc1);
501  orig.commit();
502 
503  TEST_EQUAL(replicate(master, replica, tempdir, 0, 1, true), 1);
504  check_equal_dbs(masterpath, replicapath);
505 
506  // Make a changeset.
507  orig.add_document(doc1);
508  orig.commit();
509 
510  replicate_with_brokenness(master, replica, tempdir, 1, 0, true);
511  // Although it throws an error, the final replication in
512  // replicate_with_brokenness() updates the database, since it's just
513  // the end-of-replication message which is missing its body.
514  check_equal_dbs(masterpath, replicapath);
515 
516  // Check that the earlier broken replications didn't cause any problems
517  // for the next replication.
518  orig.add_document(doc1);
519  orig.commit();
520  TEST_EQUAL(replicate(master, replica, tempdir, 1, 0, true), 2);
521 
522  // We need this inner scope to we close the replica before we remove
523  // the temporary directory on Windows.
524  }
525 
526  rmtmpdir(tempdir);
527 #endif
528 }
529 
530 // Tests for max_changesets
531 DEFINE_TESTCASE(replicate4, replicas) {
532 #ifdef XAPIAN_HAS_REMOTE_BACKEND
534  string tempdir = ".replicatmp";
535  mktmpdir(tempdir);
536  string masterpath = get_named_writable_database_path("master");
537 
539 
540  {
542  Xapian::DatabaseMaster master(masterpath);
543  string replicapath = tempdir + "/replica";
544  Xapian::DatabaseReplica replica(replicapath);
545 
546  // Add a document with no positions to the original database.
547  Xapian::Document doc1;
548  doc1.set_data(string("doc1"));
549  doc1.add_term("nopos");
550  orig.add_document(doc1);
551  orig.commit();
552 
553  // Apply the replication - we don't have changesets stored, so this
554  // should just do a database copy, and return a count of 1.
555  int count = replicate(master, replica, tempdir, 0, 1, true);
556  TEST_EQUAL(count, 1);
557  {
558  Xapian::Database dbcopy(replicapath);
559  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
560  }
561 
562  // Add a document with positional information to the original database.
563  doc1.add_posting("pos", 1);
564  orig.add_document(doc1);
565  orig.commit();
566 
567  // Replicate, and check that we have the positional information.
568  count = replicate(master, replica, tempdir, 1, 0, true);
569  TEST_EQUAL(count, 2);
570  {
571  Xapian::Database dbcopy(replicapath);
572  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
573  }
574  check_equal_dbs(masterpath, replicapath);
575 
576  // Add a document with no positions to the original database.
577  Xapian::Document doc2;
578  doc2.set_data(string("doc2"));
579  doc2.add_term("nopos");
580  orig.add_document(doc2);
581  if (get_dbtype() != "chert") {
582  // FIXME: Needs to be pre-commit for new-glass
584  }
585  orig.commit();
586 
587  // Replicate, and check that we have the positional information.
588  count = replicate(master, replica, tempdir, 1, 0, true);
589  TEST_EQUAL(count, 2);
590  {
591  Xapian::Database dbcopy(replicapath);
592  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
593  }
594  check_equal_dbs(masterpath, replicapath);
595  TEST(!file_exists(masterpath + "/changes1"));
596 
597  // Turn off replication, make sure we don't write anything.
598  if (get_dbtype() == "chert") {
600  }
601 
602  // Add a document with no positions to the original database.
603  Xapian::Document doc3;
604  doc3.set_data(string("doc3"));
605  doc3.add_term("nonopos");
606  orig.add_document(doc3);
607  orig.commit();
608 
609  // Replicate, and check that we have the positional information.
610  count = replicate(master, replica, tempdir, 0, 1, true);
611  TEST_EQUAL(count, 1);
612  {
613  Xapian::Database dbcopy(replicapath);
614  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
615  }
616  // Should have pulled a full copy
617  check_equal_dbs(masterpath, replicapath);
618  TEST(!file_exists(masterpath + "/changes3"));
619 
620  // We need this inner scope to we close the replica before we remove
621  // the temporary directory on Windows.
622  }
623 
624  rmtmpdir(tempdir);
625 #endif
626 }
627 
628 // Tests for max_changesets
629 DEFINE_TESTCASE(replicate5, replicas) {
630 #ifdef XAPIAN_HAS_REMOTE_BACKEND
631  SKIP_TEST_FOR_BACKEND("chert");
633  string tempdir = ".replicatmp";
634  mktmpdir(tempdir);
635  string masterpath = get_named_writable_database_path("master");
636 
638 
639  {
641  Xapian::DatabaseMaster master(masterpath);
642  string replicapath = tempdir + "/replica";
643  Xapian::DatabaseReplica replica(replicapath);
644 
645  // Add a document with no positions to the original database.
646  Xapian::Document doc1;
647  doc1.set_data(string("doc1"));
648  doc1.add_term("nopos");
649  orig.add_document(doc1);
650  orig.commit();
651 
652  // Apply the replication - we don't have changesets stored, so this
653  // should just do a database copy, and return a count of 1.
654  int count = replicate(master, replica, tempdir, 0, 1, true);
655  TEST_EQUAL(count, 1);
656  {
657  Xapian::Database dbcopy(replicapath);
658  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
659  }
660 
661  // Add a document with positional information to the original database.
662  doc1.add_posting("pos", 1);
663  orig.add_document(doc1);
664  orig.commit();
665 
666  // Replicate, and check that we have the positional information.
667  count = replicate(master, replica, tempdir, 1, 0, true);
668  TEST_EQUAL(count, 2);
669  {
670  Xapian::Database dbcopy(replicapath);
671  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
672  }
673  check_equal_dbs(masterpath, replicapath);
674 
675  // Add a document with no positions to the original database.
676  Xapian::Document doc2;
677  doc2.set_data(string("doc2"));
678  doc2.add_term("nopos");
679  orig.add_document(doc2);
680  orig.commit();
681 
682  // Replicate, and check that we have the positional information.
683  count = replicate(master, replica, tempdir, 1, 0, true);
684  TEST_EQUAL(count, 2);
685  {
686  Xapian::Database dbcopy(replicapath);
687  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
688  }
689  check_equal_dbs(masterpath, replicapath);
690 
691  // Add a document with no positions to the original database.
692  Xapian::Document doc3;
693  doc3.set_data(string("doc3"));
694  doc3.add_term("nonopos");
695  orig.add_document(doc3);
696  orig.commit();
697 
698  // Replicate, and check that we have the positional information.
699  count = replicate(master, replica, tempdir, 1, 0, true);
700  TEST_EQUAL(count, 2);
701  {
702  Xapian::Database dbcopy(replicapath);
703  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
704  }
705  check_equal_dbs(masterpath, replicapath);
706 
707  // Ensure that only these changesets exists
708  TEST(!file_exists(masterpath + "/changes1"));
709  TEST(file_exists(masterpath + "/changes2"));
710  TEST(file_exists(masterpath + "/changes3"));
711 
713  masterpath = get_named_writable_database_path("master");
714 
715  // Add a document with no positions to the original database.
716  Xapian::Document doc4;
717  doc4.set_data(string("doc4"));
718  doc4.add_term("nononopos");
719  orig.add_document(doc4);
720  orig.commit();
721 
722  // Replicate, and check that we have the positional information.
723  count = replicate(master, replica, tempdir, 1, 0, true);
724  TEST_EQUAL(count, 2);
725  {
726  Xapian::Database dbcopy(replicapath);
727  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
728  }
729  check_equal_dbs(masterpath, replicapath);
730 
731  // Add a document with no positions to the original database.
732  Xapian::Document doc5;
733  doc5.set_data(string("doc5"));
734  doc5.add_term("nonononopos");
735  orig.add_document(doc5);
736  orig.commit();
737 
738  // Replicate, and check that we have the positional information.
739  count = replicate(master, replica, tempdir, 1, 0, true);
740  TEST_EQUAL(count, 2);
741  {
742  Xapian::Database dbcopy(replicapath);
743  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
744  }
745  check_equal_dbs(masterpath, replicapath);
746 
747  TEST(!file_exists(masterpath + "/changes2"));
748  TEST(file_exists(masterpath + "/changes3"));
749  TEST(file_exists(masterpath + "/changes4"));
750  TEST(file_exists(masterpath + "/changes5"));
751 
752  // We need this inner scope to we close the replica before we remove
753  // the temporary directory on Windows.
754  }
755 
756  rmtmpdir(tempdir);
757 #endif
758 }
759 
761 DEFINE_TESTCASE(replicate6, replicas) {
762 #ifdef XAPIAN_HAS_REMOTE_BACKEND
764  string tempdir = ".replicatmp";
765  mktmpdir(tempdir);
766  string masterpath = get_named_writable_database_path("master");
767 
768  set_max_changesets(10);
769 
770  {
772  Xapian::DatabaseMaster master(masterpath);
773  string replicapath = tempdir + "/replica";
774  Xapian::DatabaseReplica replica(replicapath);
775 
776  // Add a document to the original database.
777  Xapian::Document doc1;
778  doc1.set_data(string("doc1"));
779  doc1.add_posting("doc", 1);
780  doc1.add_posting("one", 1);
781  orig.add_document(doc1);
782  orig.commit();
783 
784  rm_rf(masterpath + "1");
785  cp_R(masterpath, masterpath + "1");
786 
787  orig.add_document(doc1);
788  orig.commit();
789 
790  // Apply the replication - we don't have changesets stored, so this
791  // should just do a database copy, and return a count of 1.
792  int count = replicate(master, replica, tempdir, 0, 1, true);
793  TEST_EQUAL(count, 1);
794  {
795  Xapian::Database dbcopy(replicapath);
796  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
797  }
798 
799  Xapian::DatabaseMaster master1(masterpath + "1");
800 
801  // Try to replicate an older version of the master.
802  count = replicate(master1, replica, tempdir, 0, 0, false);
803  TEST_EQUAL(count, 1);
804 
805  // Force a full copy.
806  count = replicate(master1, replica, tempdir, 0, 1, true, true);
807  TEST_EQUAL(count, 1);
808 
809  // Test we can still replicate.
810  orig.add_document(doc1);
811  orig.commit();
812 
813  count = replicate(master, replica, tempdir, 2, 0, true);
814  TEST_EQUAL(count, 3);
815 
816  check_equal_dbs(masterpath, replicapath);
817 
818  // We need this inner scope to we close the replica before we remove
819  // the temporary directory on Windows.
820  }
821 
822  rmtmpdir(tempdir);
823 #endif
824 }
825 
827 DEFINE_TESTCASE(replicate7, replicas) {
828 #ifdef XAPIAN_HAS_REMOTE_BACKEND
830  string tempdir = ".replicatmp";
831  mktmpdir(tempdir);
832  string masterpath = get_named_writable_database_path("master");
833 
834  set_max_changesets(10);
835 
837  Xapian::DatabaseMaster master(masterpath);
838  string replicapath = tempdir + "/replica";
839  {
840  Xapian::DatabaseReplica replica(replicapath);
841 
842  // Add a document to the original database.
843  Xapian::Document doc1;
844  doc1.set_data(string("doc1"));
845  doc1.add_posting("doc", 1);
846  doc1.add_posting("one", 1);
847  orig.add_document(doc1);
848  orig.commit();
849 
850  orig.add_document(doc1);
851  orig.commit();
852 
853  // Apply the replication - we don't have changesets stored, so this
854  // should just do a database copy, and return a count of 1.
855  int count = replicate(master, replica, tempdir, 0, 1, true);
856  TEST_EQUAL(count, 1);
857  {
858  Xapian::Database dbcopy(replicapath);
859  TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
860  }
861  }
862 
863  {
864  // Corrupt replica by truncating all the files to size 0.
865  string d = replicapath;
866  d += "/replica_1";
867  DIR * dir = opendir(d.c_str());
868  TEST(dir != NULL);
869  while (true) {
870  errno = 0;
871  struct dirent * entry = readdir(dir);
872  if (!entry) {
873  if (errno == 0)
874  break;
875  FAIL_TEST("readdir failed: " << errno_to_string(errno));
876  }
877 
878  // Skip '.' and '..'.
879  if (entry->d_name[0] == '.') continue;
880 
881  string file = d;
882  file += '/';
883  file += entry->d_name;
884  int fd = open(file.c_str(), O_WRONLY|O_TRUNC);
885  TEST(fd != -1);
886  TEST(close(fd) == 0);
887  }
888  closedir(dir);
889  }
890 
891  {
892  Xapian::DatabaseReplica replica(replicapath);
893 
894  // Replication should succeed and perform a full copy.
895  int count = replicate(master, replica, tempdir, 0, 1, true);
896  TEST_EQUAL(count, 1);
897 
898  check_equal_dbs(masterpath, replicapath);
899 
900  // We need this inner scope to we close the replica before we remove
901  // the temporary directory on Windows.
902  }
903 
904  rmtmpdir(tempdir);
905 #endif
906 }
int close(FD &fd)
Definition: fd.h:63
Xapian::docid add_document(const Xapian::Document &document)
Add a new document to the database.
Definition: omdatabase.cc:902
int closedir(DIR *)
static size_t check(const std::string &path, int opts=0, std::ostream *out=NULL)
Check the integrity of a database or database table.
Definition: database.h:564
void dbcheck(const Xapian::Database &db, Xapian::doccount expected_doccount, Xapian::docid expected_lastdocid)
Check consistency of database and statistics.
Definition: dbcheck.cc:126
#define TEST(a)
Test a condition, without an additional explanation for failure.
Definition: testsuite.h:275
This class is used to access a database, or a group of databases.
Definition: database.h:68
void set_read_fd(int fd)
Set the file descriptor to read changesets from.
Definition: replication.cc:254
int fullcopy_count
Number of times a full database copy was performed.
Definition: replication.h:38
test database contents and consistency.
Access to a master database for replication.
Definition: replication.h:61
static int apply_changeset(const string &changesetpath, Xapian::DatabaseReplica &replica, int expected_changesets, int expected_fullcopies, bool expected_changed)
Access to a database replica, for applying replication to it.
Definition: replication.h:114
TermIterator allterms_end(const std::string &=std::string()) const
Corresponding end iterator to allterms_begin(prefix).
Definition: database.h:269
Xapian::docid get_lastdocid() const
Get the highest document id which has been used in the database.
Definition: omdatabase.cc:279
Convert errno value to std::string, thread-safe if possible.
#define O_BINARY
Definition: safefcntl.h:81
a generic test suite engine
struct dirent * readdir(DIR *)
C++ function versions of useful Unix commands.
WritableDatabase open()
Construct a WritableDatabase object for a new, empty InMemory database.
Definition: dbfactory.h:104
#define set_max_changesets(N)
Xapian::WritableDatabase get_writable_database_again()
Definition: apitest.cc:125
STL namespace.
include <dirent.h>, with alternative implementation for windows.
Utility functions for testing files.
Xapian::doccount get_doccount() const
Get the number of documents in the database.
Definition: omdatabase.cc:267
std::string get_revision_info() const
Get a string describing the current revision of the replica.
Definition: replication.cc:247
include <sys/stat.h> with portability enhancements
static void check_equal_dbs(const string &masterpath, const string &replicapath)
Xapian::WritableDatabase get_named_writable_database(const std::string &name, const std::string &source)
Definition: apitest.cc:93
test functionality of the Xapian API
void rm_rf(const string &filename)
Remove a directory and contents, just like the Unix "rm -rf" command.
Definition: unixcmds.cc:111
void cp_R(const std::string &src, const std::string &dest)
Recursively copy a directory.
Definition: unixcmds.cc:67
static int replicate(Xapian::DatabaseMaster &master, Xapian::DatabaseReplica &replica, const string &tempdir, int expected_changesets, int expected_fullcopies, bool expected_changed, bool full_copy=false)
Class for iterating over a list of terms.
Definition: termiterator.h:41
static void do_write(int fd, const char *p, size_t n)
static size_t do_read(int fd, char *p, size_t desired)
string postlist_to_string(const Xapian::Database &db, const string &tname)
Convert the list of postings in a postlist to a string.
Definition: dbcheck.cc:55
static off_t get_file_size(const string &path)
std::string get_named_writable_database_path(const std::string &name)
Definition: apitest.cc:99
DIR * opendir(const char *)
This class provides read/write access to a database.
Definition: database.h:789
Definition: fd.h:30
std::ostringstream tout
The debug printing stream.
Definition: testsuite.cc:103
void errno_to_string(int e, string &s)
char * d_name
Definition: msvc_dirent.h:36
Public interfaces for the Xapian library.
bool apply_next_changeset(ReplicationInfo *info, double reader_close_time)
Read and apply the next changeset.
Definition: replication.cc:261
#define TEST_EXCEPTION(TYPE, CODE)
Check that CODE throws exactly Xapian exception TYPE.
Definition: testutils.h:109
std::string get_dbtype()
Definition: apitest.cc:42
Wrapper class around a file descriptor to avoid leaks.
Information about the steps involved in performing a replication.
Definition: replication.h:33
int changeset_count
Number of changesets applied.
Definition: replication.h:35
void commit()
Commit any pending modifications made to the database.
Definition: omdatabase.cc:857
static void rmtmpdir(const string &path)
struct DIR DIR
Definition: msvc_dirent.h:32
static void replicate_with_brokenness(Xapian::DatabaseMaster &master, Xapian::DatabaseReplica &replica, const string &tempdir, int expected_changesets, int expected_fullcopies, bool expected_changed)
Provide setenv() with compatibility versions.
TermIterator allterms_begin(const std::string &prefix=std::string()) const
An iterator which runs across all terms with a given prefix.
Definition: omdatabase.cc:223
static void get_changeset(const string &changesetpath, Xapian::DatabaseMaster &master, Xapian::DatabaseReplica &replica, int expected_changesets, int expected_fullcopies, bool expected_changed, bool full_copy=false)
#define SKIP_TEST_FOR_BACKEND(B)
Definition: apitest.h:75
bool changed
True if and only if the replication corresponds to a change in the live version of the database...
Definition: replication.h:45
void add_posting(const std::string &tname, Xapian::termpos tpos, Xapian::termcount wdfinc=1)
Add an occurrence of a term at a particular position.
Definition: omdocument.cc:128
#define FAIL_TEST(MSG)
Fail the current testcase with message MSG.
Definition: testsuite.h:68
Replication support for Xapian databases.
Indicates a problem communicating with a remote database.
Definition: error.h:803
static void mktmpdir(const string &path)
Xapian-specific test helper functions and macros.
<unistd.h>, but with compat.
off_t file_size(const char *path)
Returns the size of a file.
Definition: filetests.h:71
void write_changesets_to_fd(int fd, const std::string &start_revision, ReplicationInfo *info) const
Write a set of changesets for upgrading the database to a file.
Definition: replication.cc:70
#define UNSET_MAX_CHANGESETS_AFTERWARDS
#define TEST_EQUAL(a, b)
Test for equality of two things.
Definition: testsuite.h:278
void set_data(const std::string &data)
Set data stored in the document.
Definition: omdocument.cc:78
bool file_exists(const char *path)
Test if a file exists.
Definition: filetests.h:39
virtual void close()
Close the database.
Definition: omdatabase.cc:138
include <fcntl.h>, but working around broken platforms.
DEFINE_TESTCASE(replicate1, replicas)
static off_t truncated_copy(const string &srcpath, const string &destpath, off_t tocopy)
A handle representing a document in a Xapian database.
Definition: document.h:61
std::string get_uuid() const
Get a UUID for the database.
Definition: omdatabase.cc:776
void add_term(const std::string &tname, Xapian::termcount wdfinc=1)
Add a term to the document, without positional information.
Definition: omdocument.cc:140