xapian-core  1.4.19
glass_database.cc
Go to the documentation of this file.
1 
4 /* Copyright 1999,2000,2001 BrightStation PLC
5  * Copyright 2001 Hein Ragas
6  * Copyright 2002 Ananova Ltd
7  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2019 Olly Betts
8  * Copyright 2006,2008 Lemur Consulting Ltd
9  * Copyright 2009 Richard Boulton
10  * Copyright 2009 Kan-Ru Chen
11  * Copyright 2011 Dan Colish
12  *
13  * This program is free software; you can redistribute it and/or
14  * modify it under the terms of the GNU General Public License as
15  * published by the Free Software Foundation; either version 2 of the
16  * License, or (at your option) any later version.
17  *
18  * This program is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21  * GNU General Public License for more details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with this program; if not, write to the Free Software
25  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
26  * USA
27  */
28 
29 #include <config.h>
30 
31 #include "glass_database.h"
32 
33 #include "xapian/constants.h"
34 #include "xapian/error.h"
35 #include "xapian/valueiterator.h"
36 
38 #include "glass_alldocspostlist.h"
39 #include "glass_alltermslist.h"
40 #include "glass_defs.h"
41 #include "glass_docdata.h"
42 #include "glass_document.h"
43 #include "../flint_lock.h"
44 #include "glass_metadata.h"
45 #include "glass_positionlist.h"
46 #include "glass_postlist.h"
49 #include "glass_termlist.h"
50 #include "glass_valuelist.h"
51 #include "glass_values.h"
52 #include "debuglog.h"
53 #include "fd.h"
54 #include "filetests.h"
55 #include "io_utils.h"
56 #include "pack.h"
57 #include "net/remoteconnection.h"
58 #include "api/replication.h"
59 #include "replicationprotocol.h"
60 #include "net/length.h"
61 #include "posixy_wrapper.h"
62 #include "str.h"
63 #include "stringutils.h"
64 #include "backends/valuestats.h"
65 
66 #include "safesysstat.h"
67 #include <sys/types.h>
68 
69 #include <algorithm>
70 #include "autoptr.h"
71 #include <cerrno>
72 #include <cstdlib>
73 #include <string>
74 
75 using namespace std;
76 using namespace Xapian;
78 
79 // The maximum safe term length is determined by the postlist. There we
80 // store the term using pack_string_preserving_sort() which takes the
81 // length of the string plus an extra byte (assuming the string doesn't
82 // contain any zero bytes), followed by the docid with encoded with
83 // pack_uint_preserving_sort() which takes up to 5 bytes (for a 32-bit
84 // docid).
85 //
86 // The Btree manager's key length limit is 255 bytes so the maximum safe term
87 // length is 255 - 1 - 5 = 249 bytes. We actually set the limit at 245 for
88 // consistency with flint and chert, and also because this allows for 64-bit
89 // docids.
90 //
91 // If the term contains zero bytes, the limit is lower (by one for each zero
92 // byte in the term).
93 #define MAX_SAFE_TERM_LENGTH 245
94 
95 /* This opens the tables, determining the current and next revision numbers,
96  * and stores handles to the tables.
97  */
98 GlassDatabase::GlassDatabase(const string &glass_dir, int flags,
99  unsigned int block_size)
100  : db_dir(glass_dir),
101  readonly(flags == Xapian::DB_READONLY_),
102  version_file(db_dir),
103  postlist_table(db_dir, readonly),
104  position_table(db_dir, readonly),
105  // Note: (Xapian::DB_READONLY_ & Xapian::DB_NO_TERMLIST) is true,
106  // so opening to read we always permit the termlist to be missing.
107  termlist_table(db_dir, readonly, (flags & Xapian::DB_NO_TERMLIST)),
108  value_manager(&postlist_table, &termlist_table),
109  synonym_table(db_dir, readonly),
110  spelling_table(db_dir, readonly),
111  docdata_table(db_dir, readonly),
112  lock(db_dir),
113  changes(db_dir)
114 {
115  LOGCALL_CTOR(DB, "GlassDatabase", glass_dir | flags | block_size);
116 
117  if (readonly) {
118  open_tables(flags);
119  return;
120  }
121 
122  // Block size must in the range 2048..65536, and a power of two.
123  if (block_size < 2048 || block_size > 65536 ||
124  (block_size & (block_size - 1)) != 0) {
125  block_size = GLASS_DEFAULT_BLOCKSIZE;
126  }
127 
128  int action = flags & Xapian::DB_ACTION_MASK_;
129  if (action != Xapian::DB_OPEN && !database_exists()) {
130  // Create the directory for the database, if it doesn't exist
131  // already.
132  if (mkdir(db_dir.c_str(), 0755) < 0) {
133  int mkdir_errno = errno;
134  if (mkdir_errno != EEXIST || !dir_exists(db_dir)) {
135  throw Xapian::DatabaseCreateError(db_dir + ": mkdir failed",
136  mkdir_errno);
137  }
138  }
139 
140  get_database_write_lock(flags, true);
141 
142  create_and_open_tables(flags, block_size);
143  return;
144  }
145 
146  if (action == Xapian::DB_CREATE) {
147  throw Xapian::DatabaseCreateError("Can't create new database at '" +
148  db_dir + "': a database already exists and I was told "
149  "not to overwrite it");
150  }
151 
152  get_database_write_lock(flags, false);
153  // if we're overwriting, pretend the db doesn't exist
154  if (action == Xapian::DB_CREATE_OR_OVERWRITE) {
155  create_and_open_tables(flags, block_size);
156  return;
157  }
158 
159  // Open the latest version of each table.
160  open_tables(flags);
161 }
162 
164  : db_dir(),
165  readonly(true),
166  version_file(fd),
167  postlist_table(fd, version_file.get_offset(), readonly),
168  position_table(fd, version_file.get_offset(), readonly),
169  termlist_table(fd, version_file.get_offset(), readonly, true),
171  synonym_table(fd, version_file.get_offset(), readonly),
172  spelling_table(fd, version_file.get_offset(), readonly),
173  docdata_table(fd, version_file.get_offset(), readonly),
174  lock(),
175  changes(string())
176 {
177  LOGCALL_CTOR(DB, "GlassDatabase", fd);
178  open_tables(Xapian::DB_READONLY_);
179 }
180 
182 {
183  LOGCALL_DTOR(DB, "GlassDatabase");
184 }
185 
186 bool
188  LOGCALL(DB, bool, "GlassDatabase::database_exists", NO_ARGS);
189  // The postlist table is the only non-optional one.
191 }
192 
193 void
194 GlassDatabase::create_and_open_tables(int flags, unsigned int block_size)
195 {
196  LOGCALL_VOID(DB, "GlassDatabase::create_and_open_tables", flags|block_size);
197  // The caller is expected to create the database directory if it doesn't
198  // already exist.
199 
201  v.create(block_size);
202 
204  const string& tmpfile = v.write(rev, flags);
205 
212 
213  if (!v.sync(tmpfile, rev, flags)) {
214  throw Xapian::DatabaseCreateError("Failed to create iamglass file");
215  }
216 
218 }
219 
220 bool
222 {
223  LOGCALL(DB, bool, "GlassDatabase::open_tables", flags);
224 
226 
227  if (cur_rev != 0) {
228  // We're reopening, so ensure that we throw DatabaseError if close()
229  // was called. It could be argued that reopen() can be a no-op in this
230  // case, but that's confusing if someone expects that reopen() can undo
231  // the effects of close() - in fact reopen() is closer to
232  // update_reader_to_latest_revision().
233  if (!postlist_table.is_open())
235  }
236 
237  version_file.read();
239  if (cur_rev && cur_rev == rev) {
240  // We're reopening a database and the revision hasn't changed so we
241  // don't need to do anything.
242  RETURN(false);
243  }
244 
251 
254 
256 
257  if (!readonly) {
260  GlassChanges * p = changes.start(revision, revision + 1, flags);
268  }
269  return true;
270 }
271 
274 {
275  LOGCALL(DB, glass_revision_number_t, "GlassDatabase::get_revision_number", NO_ARGS);
277 }
278 
281 {
282  LOGCALL(DB, glass_revision_number_t, "GlassDatabase::get_next_revision_number", NO_ARGS);
283  // FIXME: If we permit a revision before the latest and then updating it
284  // (to roll back more recent changes) then we (probably) need this to be
285  // one more than the *highest* revision previously committed.
287 }
288 
289 void
291  glass_revision_number_t * startrev,
292  glass_revision_number_t * endrev) const
293 {
294  FD fd(posixy_open(path.c_str(), O_RDONLY | O_CLOEXEC));
295  if (fd < 0) {
296  string message = string("Couldn't open changeset ") +
297  path + " to read";
298  throw Xapian::DatabaseError(message, errno);
299  }
300 
301  char buf[REASONABLE_CHANGESET_SIZE];
302  const char *start = buf;
303  const char *end = buf + io_read(fd, buf, REASONABLE_CHANGESET_SIZE);
304  if (size_t(end - start) < CONST_STRLEN(CHANGES_MAGIC_STRING))
305  throw Xapian::DatabaseError("Changeset too short at " + path);
306  if (memcmp(start, CHANGES_MAGIC_STRING,
308  string message = string("Changeset at ") +
309  path + " does not contain valid magic string";
310  throw Xapian::DatabaseError(message);
311  }
313 
314  unsigned int changes_version;
315  if (!unpack_uint(&start, end, &changes_version))
316  throw Xapian::DatabaseError("Couldn't read a valid version number for "
317  "changeset at " + path);
318  if (changes_version != CHANGES_VERSION)
319  throw Xapian::DatabaseError("Don't support version of changeset at " +
320  path);
321 
322  if (!unpack_uint(&start, end, startrev))
323  throw Xapian::DatabaseError("Couldn't read a valid start revision from "
324  "changeset at " + path);
325 
326  if (!unpack_uint(&start, end, endrev))
327  throw Xapian::DatabaseError("Couldn't read a valid end revision for "
328  "changeset at " + path);
329 }
330 
331 void
333 {
334  LOGCALL_VOID(DB, "GlassDatabase::set_revision_number", flags|new_revision);
335 
337  if (new_revision <= rev && rev != 0) {
338  string m = "New revision ";
339  m += str(new_revision);
340  m += " <= old revision ";
341  m += str(rev);
342  throw Xapian::DatabaseError(m);
343  }
344 
346 
353 
360 
361  const string & tmpfile = version_file.write(new_revision, flags);
362  if (!postlist_table.sync() ||
363  !position_table.sync() ||
364  !termlist_table.sync() ||
365  !synonym_table.sync() ||
366  !spelling_table.sync() ||
367  !docdata_table.sync() ||
368  !version_file.sync(tmpfile, new_revision, flags)) {
369  int saved_errno = errno;
370  (void)unlink(tmpfile.c_str());
371  throw Xapian::DatabaseError("Commit failed", saved_errno);
372  }
373 
374  changes.commit(new_revision, flags);
375 }
376 
377 void
379 {
381 }
382 
383 void
385 {
387  for (t = query.get_unique_terms_begin(); t != Xapian::TermIterator(); ++t) {
388  const string & term = *t;
390  break;
391  }
392 }
393 
394 bool
396 {
397  LOGCALL(DB, bool, "GlassDatabase::reopen", NO_ARGS);
398  if (!readonly) RETURN(false);
400 }
401 
402 void
404 {
405  LOGCALL_VOID(DB, "GlassDatabase::close", NO_ARGS);
406  postlist_table.close(true);
407  position_table.close(true);
408  termlist_table.close(true);
409  synonym_table.close(true);
410  spelling_table.close(true);
411  docdata_table.close(true);
412  lock.release();
413 }
414 
415 void
416 GlassDatabase::get_database_write_lock(int flags, bool creating)
417 {
418  LOGCALL_VOID(DB, "GlassDatabase::get_database_write_lock", flags|creating);
419  // FIXME: Handle Xapian::DB_DANGEROUS here, perhaps by having readers
420  // get a lock on the revision they're reading, and then requiring the
421  // writer get an exclusive lock in this case.
422  string explanation;
423  bool retry = flags & Xapian::DB_RETRY_LOCK;
424  FlintLock::reason why = lock.lock(true, retry, explanation);
425  if (why != FlintLock::SUCCESS) {
426  if (why == FlintLock::UNKNOWN && !creating && !database_exists()) {
427  string msg("No glass database found at path '");
428  msg += db_dir;
429  msg += '\'';
431  }
432  lock.throw_databaselockerror(why, db_dir, explanation);
433  }
434 }
435 
436 void
438 {
439  LOGCALL_VOID(DB, "GlassDatabase::send_whole_database", conn | end_time);
440 #ifdef XAPIAN_HAS_REMOTE_BACKEND
441  // Send the current revision number in the header.
442  string buf;
443  string uuid = get_uuid();
444  buf += encode_length(uuid.size());
445  buf += uuid;
447  conn.send_message(REPL_REPLY_DB_HEADER, buf, end_time);
448 
449  // Send all the tables. The tables which we want to be cached best after
450  // the copy finishes are sent last.
451  static const char filenames[] =
452  "termlist." GLASS_TABLE_EXTENSION "\0"
453  "synonym." GLASS_TABLE_EXTENSION "\0"
454  "spelling." GLASS_TABLE_EXTENSION "\0"
455  "docdata." GLASS_TABLE_EXTENSION "\0"
456  "position." GLASS_TABLE_EXTENSION "\0"
457  "postlist." GLASS_TABLE_EXTENSION "\0"
458  "iamglass\0";
459  string filepath = db_dir;
460  filepath += '/';
461  const char * p = filenames;
462  do {
463  size_t len = strlen(p);
464  filepath.replace(db_dir.size() + 1, string::npos, p, len);
465  FD fd(posixy_open(filepath.c_str(), O_RDONLY | O_CLOEXEC));
466  if (fd >= 0) {
467  conn.send_message(REPL_REPLY_DB_FILENAME, string(p, len), end_time);
468  conn.send_file(REPL_REPLY_DB_FILEDATA, fd, end_time);
469  }
470  p += len + 1;
471  } while (*p);
472 #else
473  (void)conn;
474  (void)end_time;
475 #endif
476 }
477 
478 void
480  const string & revision,
481  bool need_whole_db,
482  ReplicationInfo * info)
483 {
484  LOGCALL_VOID(DB, "GlassDatabase::write_changesets_to_fd", fd | revision | need_whole_db | info);
485 #ifdef XAPIAN_HAS_REMOTE_BACKEND
486  int whole_db_copies_left = MAX_DB_COPIES_PER_CONVERSATION;
487  glass_revision_number_t start_rev_num = 0;
488  string start_uuid = get_uuid();
489 
490  glass_revision_number_t needed_rev_num = 0;
491 
492  const char * rev_ptr = revision.data();
493  const char * rev_end = rev_ptr + revision.size();
494  if (!unpack_uint(&rev_ptr, rev_end, &start_rev_num)) {
495  need_whole_db = true;
496  }
497 
498  RemoteConnection conn(-1, fd, string());
499 
500  // While the starting revision number is less than the latest revision
501  // number, look for a changeset, and write it.
502  //
503  // FIXME - perhaps we should make hardlinks for all the changesets we're
504  // likely to need, first, and then start sending them, so that there's no
505  // risk of them disappearing while we're sending earlier ones.
506  while (true) {
507  if (need_whole_db) {
508  // Decrease the counter of copies left to be sent, and fail
509  // if we've already copied the database enough. This ensures that
510  // synchronisation attempts always terminate eventually.
511  if (whole_db_copies_left == 0) {
513  "Database changing too fast",
514  0.0);
515  return;
516  }
517  whole_db_copies_left--;
518 
519  // Send the whole database across.
520  start_rev_num = get_revision_number();
521  start_uuid = get_uuid();
522 
523  send_whole_database(conn, 0.0);
524  if (info != NULL)
525  ++(info->fullcopy_count);
526 
527  need_whole_db = false;
528 
529  reopen();
530  if (start_uuid == get_uuid()) {
531  // Send the latest revision number after sending the tables.
532  // The update must proceed to that revision number before the
533  // copy is safe to make live.
534 
535  string buf;
536  needed_rev_num = get_revision_number();
537  pack_uint(buf, needed_rev_num);
538  conn.send_message(REPL_REPLY_DB_FOOTER, buf, 0.0);
539  if (info != NULL && start_rev_num == needed_rev_num)
540  info->changed = true;
541  } else {
542  // Database has been replaced since we did the copy. Send a
543  // higher revision number than the revision we've just copied,
544  // so that the client doesn't make the copy we've just done
545  // live, and then mark that we need to do a copy again.
546  // The client will never actually get the required revision,
547  // because the next message is going to be the start of a new
548  // database transfer.
549 
550  string buf;
551  pack_uint(buf, start_rev_num + 1);
552  conn.send_message(REPL_REPLY_DB_FOOTER, buf, 0.0);
553  need_whole_db = true;
554  }
555  } else {
556  // Check if we've sent all the updates.
557  if (start_rev_num >= get_revision_number()) {
558  reopen();
559  if (start_uuid != get_uuid()) {
560  need_whole_db = true;
561  continue;
562  }
563  if (start_rev_num >= get_revision_number()) {
564  break;
565  }
566  }
567 
568  // Look for the changeset for revision start_rev_num.
569  string changes_name = db_dir + "/changes" + str(start_rev_num);
570  FD fd_changes(posixy_open(changes_name.c_str(), O_RDONLY | O_CLOEXEC));
571  if (fd_changes >= 0) {
572  // Send it, and also update start_rev_num to the new value
573  // specified in the changeset.
574  glass_revision_number_t changeset_start_rev_num;
575  glass_revision_number_t changeset_end_rev_num;
576  get_changeset_revisions(changes_name,
577  &changeset_start_rev_num,
578  &changeset_end_rev_num);
579  if (changeset_start_rev_num != start_rev_num) {
580  throw Xapian::DatabaseError("Changeset start revision does not match changeset filename");
581  }
582  if (changeset_start_rev_num >= changeset_end_rev_num) {
583  throw Xapian::DatabaseError("Changeset start revision is not less than end revision");
584  }
585 
586  conn.send_file(REPL_REPLY_CHANGESET, fd_changes, 0.0);
587  start_rev_num = changeset_end_rev_num;
588  if (info != NULL) {
589  ++(info->changeset_count);
590  if (start_rev_num >= needed_rev_num)
591  info->changed = true;
592  }
593  } else {
594  // The changeset doesn't exist: leave the revision number as it
595  // is, and mark for doing a full database copy.
596  need_whole_db = true;
597  }
598  }
599  }
600  conn.send_message(REPL_REPLY_END_OF_CHANGES, string(), 0.0);
601 #else
602  (void)fd;
603  (void)revision;
604  (void)need_whole_db;
605  (void)info;
606 #endif
607 }
608 
609 void
611  const std::string & msg)
612 {
613  // Modifications failed. Wipe all the modifications from memory.
614  int flags = postlist_table.get_flags();
616  try {
617  // Discard any buffered changes and reinitialised cached values
618  // from the table.
619  cancel();
620 
621  // Reopen tables with old revision number.
623  docdata_table.open(flags, version_file.get_root(Glass::DOCDATA), old_revision);
625  synonym_table.open(flags, version_file.get_root(Glass::SYNONYM), old_revision);
629 
632 
634 
635  // Increase revision numbers to new revision number plus one,
636  // writing increased numbers to all tables.
637  ++new_revision;
638  set_revision_number(flags, new_revision);
639  } catch (const Xapian::Error &e) {
640  // We failed to roll-back so close the database to avoid the risk of
641  // database corruption.
643  throw Xapian::DatabaseError("Modifications failed (" + msg + "), "
644  "and couldn't open at the old revision: " +
645  e.get_msg());
646  }
647 
648  GlassChanges * p;
649  p = changes.start(old_revision, new_revision, flags);
657 }
658 
659 void
661 {
662  LOGCALL_VOID(DB, "GlassDatabase::apply", NO_ARGS);
663  if (!postlist_table.is_modified() &&
670  return;
671  }
672 
674 
675  int flags = postlist_table.get_flags();
676  try {
677  set_revision_number(flags, new_revision);
678  } catch (const Xapian::Error &e) {
679  modifications_failed(new_revision, e.get_description());
680  throw;
681  } catch (...) {
682  modifications_failed(new_revision, "Unknown error");
683  throw;
684  }
685 
686  GlassChanges * p;
687  p = changes.start(new_revision, new_revision + 1, flags);
695 }
696 
697 void
699 {
700  LOGCALL_VOID(DB, "GlassDatabase::cancel", NO_ARGS);
710 
713 }
714 
717 {
718  LOGCALL(DB, Xapian::doccount, "GlassDatabase::get_doccount", NO_ARGS);
720 }
721 
724 {
725  LOGCALL(DB, Xapian::docid, "GlassDatabase::get_lastdocid", NO_ARGS);
727 }
728 
731 {
732  LOGCALL(DB, Xapian::totallength, "GlassDatabase::get_total_length", NO_ARGS);
734 }
735 
738 {
739  LOGCALL(DB, Xapian::termcount, "GlassDatabase::get_doclength", did);
740  Assert(did != 0);
741  intrusive_ptr<const GlassDatabase> ptrtothis(this);
742  RETURN(postlist_table.get_doclength(did, ptrtothis));
743 }
744 
747 {
748  LOGCALL(DB, Xapian::termcount, "GlassDatabase::get_unique_terms", did);
749  Assert(did != 0);
750  intrusive_ptr<const GlassDatabase> ptrtothis(this);
751  RETURN(GlassTermList(ptrtothis, did).get_unique_terms());
752 }
753 
754 void
755 GlassDatabase::get_freqs(const string & term,
756  Xapian::doccount * termfreq_ptr,
757  Xapian::termcount * collfreq_ptr) const
758 {
759  LOGCALL_VOID(DB, "GlassDatabase::get_freqs", term | termfreq_ptr | collfreq_ptr);
760  Assert(!term.empty());
761  postlist_table.get_freqs(term, termfreq_ptr, collfreq_ptr);
762 }
763 
766 {
767  LOGCALL(DB, Xapian::doccount, "GlassDatabase::get_value_freq", slot);
769 }
770 
771 std::string
773 {
774  LOGCALL(DB, std::string, "GlassDatabase::get_value_lower_bound", slot);
776 }
777 
778 std::string
780 {
781  LOGCALL(DB, std::string, "GlassDatabase::get_value_upper_bound", slot);
783 }
784 
787 {
789 }
790 
793 {
795 }
796 
798 GlassDatabase::get_wdf_upper_bound(const string & term) const
799 {
800  Assert(!term.empty());
801  Xapian::termcount wdfub;
802  postlist_table.get_freqs(term, NULL, NULL, &wdfub);
803  return min(wdfub, version_file.get_wdf_upper_bound());
804 }
805 
806 bool
807 GlassDatabase::term_exists(const string & term) const
808 {
809  LOGCALL(DB, bool, "GlassDatabase::term_exists", term);
810  Assert(!term.empty());
812 }
813 
814 bool
816 {
817  return !position_table.empty();
818 }
819 
820 LeafPostList *
821 GlassDatabase::open_post_list(const string& term) const
822 {
823  LOGCALL(DB, LeafPostList *, "GlassDatabase::open_post_list", term);
824  intrusive_ptr<const GlassDatabase> ptrtothis(this);
825 
826  if (term.empty()) {
829  RETURN(new ContiguousAllDocsPostList(ptrtothis, doccount));
830  }
831  RETURN(new GlassAllDocsPostList(ptrtothis, doccount));
832  }
833 
834  RETURN(new GlassPostList(ptrtothis, term, true));
835 }
836 
837 ValueList *
839 {
840  LOGCALL(DB, ValueList *, "GlassDatabase::open_value_list", slot);
841  intrusive_ptr<const GlassDatabase> ptrtothis(this);
842  RETURN(new GlassValueList(slot, ptrtothis));
843 }
844 
845 TermList *
847 {
848  LOGCALL(DB, TermList *, "GlassDatabase::open_term_list", did);
849  Assert(did != 0);
850  if (!termlist_table.is_open())
852  intrusive_ptr<const GlassDatabase> ptrtothis(this);
853  RETURN(new GlassTermList(ptrtothis, did));
854 }
855 
858 {
859  LOGCALL(DB, Xapian::Document::Internal *, "GlassDatabase::open_document", did | lazy);
860  Assert(did != 0);
861  if (!lazy) {
862  // This will throw DocNotFoundError if the document doesn't exist.
863  (void)get_doclength(did);
864  }
865 
867  RETURN(new GlassDocument(ptrtothis, did, &value_manager, &docdata_table));
868 }
869 
870 void
872  Xapian::docid did,
873  const string& term) const
874 {
875  Assert(did != 0);
876  pos_list->read_data(&position_table, did, term);
877 }
878 
880 GlassDatabase::positionlist_count(Xapian::docid did, const string& term) const
881 {
882  return position_table.positionlist_count(did, term);
883 }
884 
885 PositionList *
886 GlassDatabase::open_position_list(Xapian::docid did, const string & term) const
887 {
888  Assert(did != 0);
889 
890  AutoPtr<GlassPositionList> poslist(new GlassPositionList);
891  if (!poslist->read_data(&position_table, did, term)) {
892  // As of 1.1.0, we don't check if the did and term exist - we just
893  // return an empty positionlist. If the user really needs to know,
894  // they can check for themselves.
895  }
896 
897  return poslist.release();
898 }
899 
900 TermList *
901 GlassDatabase::open_allterms(const string & prefix) const
902 {
903  LOGCALL(DB, TermList *, "GlassDatabase::open_allterms", NO_ARGS);
905  prefix));
906 }
907 
908 TermList *
909 GlassDatabase::open_spelling_termlist(const string & word) const
910 {
911  return spelling_table.open_termlist(word);
912 }
913 
914 TermList *
916 {
918  if (!cursor) return NULL;
920  cursor);
921 }
922 
924 GlassDatabase::get_spelling_frequency(const string & word) const
925 {
926  return spelling_table.get_word_frequency(word);
927 }
928 
929 TermList *
930 GlassDatabase::open_synonym_termlist(const string & term) const
931 {
932  return synonym_table.open_termlist(term);
933 }
934 
935 TermList *
936 GlassDatabase::open_synonym_keylist(const string & prefix) const
937 {
938  GlassCursor * cursor = synonym_table.cursor_get();
939  if (!cursor) return NULL;
941  cursor, prefix);
942 }
943 
944 string
945 GlassDatabase::get_metadata(const string & key) const
946 {
947  LOGCALL(DB, string, "GlassDatabase::get_metadata", key);
948  string btree_key("\x00\xc0", 2);
949  btree_key += key;
950  string tag;
951  (void)postlist_table.get_exact_entry(btree_key, tag);
952  RETURN(tag);
953 }
954 
955 TermList *
956 GlassDatabase::open_metadata_keylist(const std::string &prefix) const
957 {
958  LOGCALL(DB, TermList *, "GlassDatabase::open_metadata_keylist", NO_ARGS);
960  if (!cursor) RETURN(NULL);
962  cursor, prefix));
963 }
964 
965 string
967 {
968  LOGCALL(DB, string, "GlassDatabase::get_revision_info", NO_ARGS);
969  string buf;
971  RETURN(buf);
972 }
973 
974 string
976 {
977  LOGCALL(DB, string, "GlassDatabase::get_uuid", NO_ARGS);
979 }
980 
981 void
983 {
984  // Either the database has been closed, or else there's no termlist table.
985  // Check if the postlist table is open to determine which is the case.
986  if (!postlist_table.is_open())
988  throw Xapian::FeatureUnavailableError("Database has no termlist");
989 }
990 
991 void
993  Xapian::docid & last) const
994 {
995  last = version_file.get_last_docid();
996  if (last == version_file.get_doccount()) {
997  // Contiguous range starting at 1.
998  first = 1;
999  return;
1000  }
1001  postlist_table.get_used_docid_range(first, last);
1002 }
1003 
1004 bool
1006 {
1007  return false;
1008 }
1009 
1010 bool
1012 {
1013  return lock.test();
1014 }
1015 
1017 
1019  int block_size)
1020  : GlassDatabase(dir, flags, block_size),
1021  change_count(0),
1022  flush_threshold(0),
1023  modify_shortcut_document(NULL),
1024  modify_shortcut_docid(0)
1025 {
1026  LOGCALL_CTOR(DB, "GlassWritableDatabase", dir | flags | block_size);
1027 
1028  const char *p = getenv("XAPIAN_FLUSH_THRESHOLD");
1029  if (p)
1030  flush_threshold = atoi(p);
1031  if (flush_threshold == 0)
1032  flush_threshold = 10000;
1033 }
1034 
1036 {
1037  LOGCALL_DTOR(DB, "GlassWritableDatabase");
1038  dtor_called();
1039 }
1040 
1041 void
1043 {
1044  if (transaction_active())
1045  throw Xapian::InvalidOperationError("Can't commit during a transaction");
1047  apply();
1048 }
1049 
1050 void
1052 {
1053  // FIXME: this should be done by checking memory usage, not the number of
1054  // changes. We could also look at the amount of data the inverter object
1055  // currently holds.
1056  if (++change_count >= flush_threshold) {
1058  if (!transaction_active()) apply();
1059  }
1060 }
1061 
1062 void
1064 {
1065  try {
1069 
1070  change_count = 0;
1071  } catch (...) {
1072  try {
1074  } catch (...) {
1075  }
1076  throw;
1077  }
1078 }
1079 
1080 void
1082 {
1083  LOGCALL_VOID(DB, "GlassWritableDatabase::close", NO_ARGS);
1084  if (!transaction_active()) {
1085  commit();
1086  // FIXME: if commit() throws, should we still close?
1087  }
1089 }
1090 
1091 void
1093 {
1096 }
1097 
1100 {
1101  LOGCALL(DB, Xapian::docid, "GlassWritableDatabase::add_document", document);
1102  // Make sure the docid counter doesn't overflow.
1104  throw Xapian::DatabaseError("Run out of docids - you'll have to use copydatabase to eliminate any gaps before you can add more documents");
1105  // Use the next unused document ID.
1107 }
1108 
1111  const Xapian::Document & document)
1112 {
1113  LOGCALL(DB, Xapian::docid, "GlassWritableDatabase::add_document_", did | document);
1114  Assert(did != 0);
1115  try {
1116  // Set the document data.
1117  docdata_table.replace_document_data(did, document.get_data());
1118 
1119  // Set the values.
1120  value_manager.add_document(did, document, value_stats);
1121 
1122  Xapian::termcount new_doclen = 0;
1123  {
1124  Xapian::TermIterator term = document.termlist_begin();
1125  for ( ; term != document.termlist_end(); ++term) {
1126  termcount wdf = term.get_wdf();
1127  // Calculate the new document length
1128  new_doclen += wdf;
1129  version_file.check_wdf(wdf);
1130 
1131  string tname = *term;
1132  if (tname.size() > MAX_SAFE_TERM_LENGTH)
1133  throw Xapian::InvalidArgumentError("Term too long (> " STRINGIZE(MAX_SAFE_TERM_LENGTH) "): " + tname);
1134 
1135  inverter.add_posting(did, tname, wdf);
1136  inverter.set_positionlist(position_table, did, tname, term);
1137  }
1138  }
1139  LOGLINE(DB, "Calculated doclen for new document " << did << " as " << new_doclen);
1140 
1141  // Set the termlist.
1142  if (termlist_table.is_open())
1143  termlist_table.set_termlist(did, document, new_doclen);
1144 
1145  // Set the new document length
1146  inverter.set_doclength(did, new_doclen, true);
1147  version_file.add_document(new_doclen);
1148  } catch (...) {
1149  // If an error occurs while adding a document, or doing any other
1150  // transaction, the modifications so far must be cleared before
1151  // returning control to the user - otherwise partial modifications will
1152  // persist in memory, and eventually get written to disk.
1153  cancel();
1154  throw;
1155  }
1156 
1158 
1159  RETURN(did);
1160 }
1161 
1162 void
1164 {
1165  LOGCALL_VOID(DB, "GlassWritableDatabase::delete_document", did);
1166  Assert(did != 0);
1167 
1168  if (!termlist_table.is_open())
1170 
1171  // Remove the document data. If this fails, just propagate the exception since
1172  // the state should still be consistent.
1173  bool doc_really_existed = docdata_table.delete_document_data(did);
1174 
1175  if (rare(modify_shortcut_docid == did)) {
1176  // The modify_shortcut document can't be used for a modification
1177  // shortcut now, because it's been deleted!
1178  modify_shortcut_document = NULL;
1180  doc_really_existed = true;
1181  }
1182 
1183  if (!doc_really_existed) {
1184  // Ensure we throw DocumentNotFound if the document doesn't exist.
1185  (void)get_doclength(did);
1186  }
1187 
1188  try {
1189  // Remove the values.
1191 
1192  // OK, now add entries to remove the postings in the underlying record.
1194  GlassTermList termlist(ptrtothis, did);
1195 
1197 
1198  termlist.next();
1199  while (!termlist.at_end()) {
1200  string tname = termlist.get_termname();
1201  inverter.delete_positionlist(did, tname);
1202 
1203  inverter.remove_posting(did, tname, termlist.get_wdf());
1204 
1205  termlist.next();
1206  }
1207 
1208  // Remove the termlist.
1209  if (termlist_table.is_open())
1211 
1212  // Mark this document as removed.
1214  } catch (...) {
1215  // If an error occurs while deleting a document, or doing any other
1216  // transaction, the modifications so far must be cleared before
1217  // returning control to the user - otherwise partial modifications will
1218  // persist in memory, and eventually get written to disk.
1219  cancel();
1220  throw;
1221  }
1222 
1224 }
1225 
1226 void
1228  const Xapian::Document & document)
1229 {
1230  LOGCALL_VOID(DB, "GlassWritableDatabase::replace_document", did | document);
1231  Assert(did != 0);
1232 
1233  try {
1234  if (did > version_file.get_last_docid()) {
1236  // If this docid is above the highwatermark, then we can't be
1237  // replacing an existing document.
1238  (void)add_document_(did, document);
1239  return;
1240  }
1241 
1242  if (!termlist_table.is_open()) {
1243  // We can replace an *unused* docid <= last_docid too.
1244  intrusive_ptr<const GlassDatabase> ptrtothis(this);
1245  if (!postlist_table.document_exists(did, ptrtothis)) {
1246  (void)add_document_(did, document);
1247  return;
1248  }
1250  }
1251 
1252  // Check for a document read from this database being replaced - ie, a
1253  // modification operation.
1254  bool modifying = false;
1255  if (modify_shortcut_docid &&
1256  document.internal->get_docid() == modify_shortcut_docid) {
1257  if (document.internal.get() == modify_shortcut_document) {
1258  // We have a docid, it matches, and the pointer matches, so we
1259  // can skip modification of any data which hasn't been modified
1260  // in the document.
1261  if (!document.internal->modified()) {
1262  // If the document is unchanged, we've nothing to do.
1263  return;
1264  }
1265  modifying = true;
1266  LOGLINE(DB, "Detected potential document modification shortcut.");
1267  } else {
1268  // The modify_shortcut document can't be used for a
1269  // modification shortcut now, because it's about to be
1270  // modified.
1271  modify_shortcut_document = NULL;
1273  }
1274  }
1275 
1276  if (!modifying || document.internal->terms_modified()) {
1277  bool pos_modified = !modifying ||
1278  document.internal->term_positions_modified();
1280  GlassTermList termlist(ptrtothis, did, false);
1281  // We passed false for throw_if_not_present so check at_end()
1282  // before next() to see if the document isn't present at all.
1283  if (termlist.at_end()) {
1284  (void)add_document_(did, document);
1285  return;
1286  }
1287  Xapian::TermIterator term = document.termlist_begin();
1288  Xapian::termcount old_doclen = termlist.get_doclength();
1289  version_file.delete_document(old_doclen);
1290  Xapian::termcount new_doclen = old_doclen;
1291 
1292  string old_tname, new_tname;
1293 
1294  termlist.next();
1295  while (!termlist.at_end() || term != document.termlist_end()) {
1296  int cmp;
1297  if (termlist.at_end()) {
1298  cmp = 1;
1299  new_tname = *term;
1300  } else {
1301  old_tname = termlist.get_termname();
1302  if (term != document.termlist_end()) {
1303  new_tname = *term;
1304  cmp = old_tname.compare(new_tname);
1305  } else {
1306  cmp = -1;
1307  }
1308  }
1309 
1310  if (cmp < 0) {
1311  // Term old_tname has been deleted.
1312  termcount old_wdf = termlist.get_wdf();
1313  new_doclen -= old_wdf;
1314  inverter.remove_posting(did, old_tname, old_wdf);
1315  if (pos_modified)
1316  inverter.delete_positionlist(did, old_tname);
1317  termlist.next();
1318  } else if (cmp > 0) {
1319  // Term new_tname as been added.
1320  termcount new_wdf = term.get_wdf();
1321  new_doclen += new_wdf;
1322  version_file.check_wdf(new_wdf);
1323  if (new_tname.size() > MAX_SAFE_TERM_LENGTH)
1324  throw Xapian::InvalidArgumentError("Term too long (> " STRINGIZE(MAX_SAFE_TERM_LENGTH) "): " + new_tname);
1325  inverter.add_posting(did, new_tname, new_wdf);
1326  if (pos_modified) {
1327  inverter.set_positionlist(position_table, did, new_tname, term);
1328  }
1329  ++term;
1330  } else if (cmp == 0) {
1331  // Term already exists: look for wdf and positionlist changes.
1332  termcount old_wdf = termlist.get_wdf();
1333  termcount new_wdf = term.get_wdf();
1334 
1335  // Check the stats even if wdf hasn't changed, because if
1336  // this is the only document, the stats will have been
1337  // zeroed.
1338  version_file.check_wdf(new_wdf);
1339 
1340  if (old_wdf != new_wdf) {
1341  new_doclen += new_wdf - old_wdf;
1342  inverter.update_posting(did, new_tname, old_wdf, new_wdf);
1343  }
1344 
1345  if (pos_modified) {
1346  inverter.set_positionlist(position_table, did, new_tname, term, true);
1347  }
1348 
1349  ++term;
1350  termlist.next();
1351  }
1352  }
1353  LOGLINE(DB, "Calculated doclen for replacement document " << did << " as " << new_doclen);
1354 
1355  // Set the termlist.
1356  if (termlist_table.is_open())
1357  termlist_table.set_termlist(did, document, new_doclen);
1358 
1359  // Set the new document length
1360  if (new_doclen != old_doclen)
1361  inverter.set_doclength(did, new_doclen, false);
1362  version_file.add_document(new_doclen);
1363  }
1364 
1365  if (!modifying || document.internal->data_modified()) {
1366  // Update the document data.
1367  docdata_table.replace_document_data(did, document.get_data());
1368  }
1369 
1370  if (!modifying || document.internal->values_modified()) {
1371  // Replace the values.
1372  value_manager.replace_document(did, document, value_stats);
1373  }
1374  } catch (...) {
1375  // If an error occurs while replacing a document, or doing any other
1376  // transaction, the modifications so far must be cleared before
1377  // returning control to the user - otherwise partial modifications will
1378  // persist in memory, and eventually get written to disk.
1379  cancel();
1380  throw;
1381  }
1382 
1384 }
1385 
1388 {
1389  LOGCALL(DB, Xapian::Document::Internal *, "GlassWritableDatabase::open_document", did | lazy);
1391  // Store the docid only after open_document() successfully returns, so an
1392  // attempt to open a missing document doesn't overwrite this.
1393  modify_shortcut_docid = did;
1395 }
1396 
1399 {
1400  LOGCALL(DB, Xapian::termcount, "GlassWritableDatabase::get_doclength", did);
1401  Xapian::termcount doclen;
1402  if (inverter.get_doclength(did, doclen))
1403  RETURN(doclen);
1405 }
1406 
1409 {
1410  LOGCALL(DB, Xapian::termcount, "GlassWritableDatabase::get_unique_terms", did);
1411  Assert(did != 0);
1412  // Note that the "approximate" size should be exact in this case.
1413  //
1414  // get_unique_terms() really ought to only count terms with wdf > 0, but
1415  // that's expensive to calculate on demand, so for now let's just ensure
1416  // unique_terms <= doclen.
1417  Xapian::termcount doclen;
1418  if (inverter.get_doclength(did, doclen)) {
1419  intrusive_ptr<const GlassDatabase> ptrtothis(this);
1420  GlassTermList termlist(ptrtothis, did);
1421  RETURN(min(doclen, termlist.get_approx_size()));
1422  }
1424 }
1425 
1426 void
1428  Xapian::doccount * termfreq_ptr,
1429  Xapian::termcount * collfreq_ptr) const
1430 {
1431  LOGCALL_VOID(DB, "GlassWritableDatabase::get_freqs", term | termfreq_ptr | collfreq_ptr);
1432  Assert(!term.empty());
1433  GlassDatabase::get_freqs(term, termfreq_ptr, collfreq_ptr);
1434  Xapian::termcount_diff tf_delta, cf_delta;
1435  if (inverter.get_deltas(term, tf_delta, cf_delta)) {
1436  if (termfreq_ptr)
1437  *termfreq_ptr += tf_delta;
1438  if (collfreq_ptr)
1439  *collfreq_ptr += cf_delta;
1440  }
1441 }
1442 
1445 {
1446  LOGCALL(DB, Xapian::doccount, "GlassWritableDatabase::get_value_freq", slot);
1447  map<Xapian::valueno, ValueStats>::const_iterator i;
1448  i = value_stats.find(slot);
1449  if (i != value_stats.end()) RETURN(i->second.freq);
1451 }
1452 
1453 std::string
1455 {
1456  LOGCALL(DB, std::string, "GlassWritableDatabase::get_value_lower_bound", slot);
1457  map<Xapian::valueno, ValueStats>::const_iterator i;
1458  i = value_stats.find(slot);
1459  if (i != value_stats.end()) RETURN(i->second.lower_bound);
1461 }
1462 
1463 std::string
1465 {
1466  LOGCALL(DB, std::string, "GlassWritableDatabase::get_value_upper_bound", slot);
1467  map<Xapian::valueno, ValueStats>::const_iterator i;
1468  i = value_stats.find(slot);
1469  if (i != value_stats.end()) RETURN(i->second.upper_bound);
1471 }
1472 
1473 bool
1474 GlassWritableDatabase::term_exists(const string & tname) const
1475 {
1476  LOGCALL(DB, bool, "GlassWritableDatabase::term_exists", tname);
1477  Xapian::doccount tf;
1478  get_freqs(tname, &tf, NULL);
1479  RETURN(tf != 0);
1480 }
1481 
1482 bool
1484 {
1486 }
1487 
1488 LeafPostList *
1489 GlassWritableDatabase::open_post_list(const string& tname) const
1490 {
1491  LOGCALL(DB, LeafPostList *, "GlassWritableDatabase::open_post_list", tname);
1493 
1494  if (tname.empty()) {
1497  RETURN(new ContiguousAllDocsPostList(ptrtothis, doccount));
1498  }
1500  RETURN(new GlassAllDocsPostList(ptrtothis, doccount));
1501  }
1502 
1503  // Flush any buffered changes for this term's postlist so we can just
1504  // iterate from the flushed state.
1506  RETURN(new GlassPostList(ptrtothis, tname, true));
1507 }
1508 
1509 ValueList *
1511 {
1512  LOGCALL(DB, ValueList *, "GlassWritableDatabase::open_value_list", slot);
1513  // If there are changes, we don't have code to iterate the modified value
1514  // list so we need to flush (but don't commit - there may be a transaction
1515  // in progress).
1518 }
1519 
1520 void
1522  Xapian::docid did,
1523  const string& term) const
1524 {
1525  Assert(did != 0);
1526  string data;
1527  if (inverter.get_positionlist(did, term, data)) {
1528  pos_list->read_data(data);
1529  return;
1530  }
1531  GlassDatabase::read_position_list(pos_list, did, term);
1532 }
1533 
1536  const string& term) const
1537 {
1538  Assert(did != 0);
1539  string data;
1540  if (inverter.get_positionlist(did, term, data)) {
1541  if (data.empty())
1542  return 0;
1543  return position_table.positionlist_count(data);
1544  }
1545  return GlassDatabase::positionlist_count(did, term);
1546 }
1547 
1548 PositionList *
1550 {
1551  Assert(did != 0);
1552 
1553  AutoPtr<GlassPositionList> poslist(new GlassPositionList);
1554 
1555  string data;
1556  if (inverter.get_positionlist(did, term, data)) {
1557  poslist->read_data(data);
1558  } else if (!poslist->read_data(&position_table, did, term)) {
1559  // As of 1.1.0, we don't check if the did and term exist - we just
1560  // return an empty positionlist. If the user really needs to know,
1561  // they can check for themselves.
1562  }
1563 
1564  return poslist.release();
1565 }
1566 
1567 TermList *
1568 GlassWritableDatabase::open_allterms(const string & prefix) const
1569 {
1570  LOGCALL(DB, TermList *, "GlassWritableDatabase::open_allterms", NO_ARGS);
1571  if (change_count) {
1572  // There are changes, and terms may have been added or removed, and so
1573  // we need to flush changes for terms with the specified prefix (but
1574  // don't commit - there may be a transaction in progress).
1576  if (prefix.empty()) {
1577  // We've flushed all the posting list changes, but the positions,
1578  // document lengths and stats haven't been written, so set
1579  // change_count to 1.
1580  // FIXME: Can we handle this better?
1581  change_count = 1;
1582  }
1583  }
1585 }
1586 
1587 void
1589 {
1591  inverter.clear();
1592  value_stats.clear();
1593  change_count = 0;
1594 }
1595 
1596 void
1598  Xapian::termcount freqinc) const
1599 {
1600  spelling_table.add_word(word, freqinc);
1601 }
1602 
1603 void
1605  Xapian::termcount freqdec) const
1606 {
1607  spelling_table.remove_word(word, freqdec);
1608 }
1609 
1610 TermList *
1612 {
1615 }
1616 
1617 TermList *
1619 {
1621  return GlassDatabase::open_synonym_keylist(prefix);
1622 }
1623 
1624 void
1626  const string & synonym) const
1627 {
1628  synonym_table.add_synonym(term, synonym);
1629 }
1630 
1631 void
1633  const string & synonym) const
1634 {
1635  synonym_table.remove_synonym(term, synonym);
1636 }
1637 
1638 void
1639 GlassWritableDatabase::clear_synonyms(const string & term) const
1640 {
1642 }
1643 
1644 void
1645 GlassWritableDatabase::set_metadata(const string & key, const string & value)
1646 {
1647  LOGCALL_VOID(DB, "GlassWritableDatabase::set_metadata", key | value);
1648  string btree_key("\x00\xc0", 2);
1649  btree_key += key;
1650  if (value.empty()) {
1651  postlist_table.del(btree_key);
1652  } else {
1653  postlist_table.add(btree_key, value);
1654  }
1655 }
1656 
1657 void
1659 {
1660  if (obj == modify_shortcut_document) {
1661  modify_shortcut_document = NULL;
1663  }
1664 }
1665 
1666 bool
1668 {
1669  return change_count > 0 ||
1677 }
GlassVersion version_file
The file describing the Glass database.
bool reopen()
Re-open tables to recover from an overwritten condition, or just get most up-to-date version...
const RootInfo & get_root(Glass::table_type tbl) const
The Xapian namespace contains public interfaces for the Xapian library.
Definition: compactor.cc:80
#define RETURN(A)
Definition: debuglog.h:482
void delete_termlist(Xapian::docid did)
Delete the termlist data for document did.
void throw_databaselockerror(FlintLock::reason why, const std::string &db_dir, const std::string &explanation) const
Throw Xapian::DatabaseLockError.
Definition: flint_lock.cc:499
#define Assert(COND)
Definition: omassert.h:122
void cancel()
Cancel pending modifications to the database.
void release()
Release the lock.
Definition: flint_lock.cc:463
void commit()
Implementation of virtual methods: see Database::Internal for details.
Xapian::termcount get_doclength_upper_bound() const
A PostList which iterates over all documents in a GlassDatabase.
std::string get_termname() const
Return the termname at the current position.
Xapian::docid get_next_docid()
FlintLock lock
Lock object.
void set_value_stats(std::map< Xapian::valueno, ValueStats > &value_stats)
Write the updated statistics to the table.
void create(unsigned blocksize)
Create the version file.
void invalidate_doc_object(Xapian::Document::Internal *obj) const
Virtual methods of Database::Internal.
Xapian::termcount get_unique_terms(Xapian::docid did) const
Virtual methods of Database::Internal.
bool sync(const std::string &tmpfile, glass_revision_number_t new_rev, int flags)
A RemoteConnection object provides a bidirectional connection to another RemoteConnection object on a...
RemoteConnection class used by the remote backend.
Xapian::termcount positionlist_count(const string &data) const
Return the number of entries in specified position list data.
Xapian::termcount get_doclength() const
Return the length of this document.
Statistics about values.
void add_synonym(const std::string &term, const std::string &synonym)
Add a synonym for term.
GlassPositionListTable position_table
Table storing position lists.
length encoded as a string
void create_and_open(int flags_, const RootInfo &root_info)
Create a new empty btree structure on disk and open it at the initial revision.
void throw_termlist_table_close_exception() const
XAPIAN_REVISION_TYPE rev
Revision number of a database.
Definition: types.h:133
bool delete_document_data(Xapian::docid did)
Delete the document data for document did.
A position list in a glass database.
void close()
Close all the tables permanently.
TermList * open_synonym_keylist(const string &prefix) const
Virtual methods of Database::Internal.
string get_uuid() const
Virtual methods of Database::Internal.
Xapian::docid get_lastdocid() const
Virtual methods of Database::Internal.
A PostList iterating all docids when they form a contiguous range.
friend class GlassAllTermsList
std::string db_dir
Directory to store databases in.
int fullcopy_count
Number of times a full database copy was performed.
Definition: replication.h:38
bool get_positionlist(Xapian::docid did, const std::string &term, std::string &s) const
#define GLASS_MAX_DOCID
The largest docid value supported by glass.
Definition: glass_defs.h:43
bool has_positions(const GlassPositionListTable &position_table) const
#define true
Definition: header.h:8
TermList * open_metadata_keylist(const std::string &prefix) const
Virtual methods of Database::Internal.
const int DB_CREATE
Create a new database.
Definition: constants.h:44
void flush_postlist_changes()
Flush any unflushed postlist changes, but don&#39;t commit them.
InvalidOperationError indicates the API was used in an invalid way.
Definition: error.h:283
void create_and_open_tables(int flags, unsigned int blocksize)
Create new tables, and open them.
bool transaction_active() const
Definition: database.h:73
std::string get_value_lower_bound(Xapian::valueno slot) const
Virtual methods of Database::Internal.
Xapian::docid add_document_(Xapian::docid did, const Xapian::Document &document)
Implementation of virtual methods: see Database::Internal for details.
void apply()
Apply any outstanding changes to the tables.
bool empty() const
Return true if there are no entries in the table.
Definition: glass_table.h:686
GlassChanges changes
Replication changesets.
bool readahead_key(const string &key) const
void send_whole_database(RemoteConnection &conn, double end_time)
Send a set of messages which transfer the whole database.
Xapian::doccount get_doccount() const
Virtual methods of Database::Internal.
TermList * open_spelling_wordlist() const
Virtual methods of Database::Internal.
void merge_changes()
Merge in batched-up changes.
Internal definitions for glass database replication.
void delete_document(Xapian::docid did)
Implementation of virtual methods: see Database::Internal for details.
uint4 glass_revision_number_t
The revision number of a glass database.
Definition: glass_defs.h:61
Iterate all document ids when they form a contiguous range.
GlassSynonymTable synonym_table
Table storing synonym data.
std::string get_value_lower_bound(Xapian::valueno slot) const
Virtual methods of Database::Internal.
Glass class for value streams.
void set_metadata(const string &key, const string &value)
Virtual methods of Database::Internal.
XAPIAN_TOTALLENGTH_TYPE totallength
The total length of all documents in a database.
Definition: types.h:139
void write_changesets_to_fd(int fd, const string &start_revision, bool need_whole_db, Xapian::ReplicationInfo *info)
Virtual methods of Database::Internal.
Provides wrappers with POSIXy semantics.
LeafPostList * open_post_list(const string &tname) const
Virtual methods of Database::Internal.
void modifications_failed(glass_revision_number_t new_revision, const std::string &msg)
Called if a modifications fail.
friend class GlassPostList
Constants in the Xapian namespace.
The GlassVersion class manages the revision files.
Definition: glass_version.h:94
Subclass of GlassTable which holds document data.
void send_file(char type, int fd, double end_time)
Send the contents of a file as a message.
const std::string & get_msg() const
Message giving details of the error, intended for human consumption.
Definition: error.h:122
#define LOGCALL_DTOR(CATEGORY, CLASS)
Definition: debuglog.h:479
double end_time(double timeout)
Return the end time for a timeout in timeout seconds.
Definition: realtime.h:95
A termlist containing all terms in a glass database.
void delete_doclength(Xapian::docid did)
Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy) const
Implementation of virtual methods: see Database::Internal for details.
A document in the database, possibly plus modifications.
Definition: document.h:41
#define STRINGIZE(X)
The STRINGIZE macro converts its parameter into a string constant.
Definition: stringutils.h:36
#define LOGCALL_VOID(CATEGORY, FUNC, PARAMS)
Definition: debuglog.h:477
Abstract base class for termlists.
Definition: termlist.h:39
Postlists in glass databases.
STL namespace.
Xapian::Document::Internal * modify_shortcut_document
A pointer to the last document which was returned by open_document(), or NULL if there is no such val...
Definitions, types, etc for use inside glass.
GlassWritableDatabase(const string &dir, int flags, int block_size)
Create and open a writable glass database.
Convert types to std::string.
TermList * open_termlist(const std::string &term)
Open synonym termlist for a term.
Xapian::doccount change_count
The number of documents added, deleted, or replaced since the last flush.
int revision()
Report the revision of the library which the program is linked with.
Definition: xapian.h:142
glass_revision_number_t get_oldest_changeset() const
Definition: glass_changes.h:71
Abstract base class for leaf postlists.
Definition: leafpostlist.h:38
Utility functions for testing files.
glass_revision_number_t get_next_revision_number() const
Get an object holding the next revision number which should be used in the tables.
A position list in a glass database.
#define O_CLOEXEC
Definition: safefcntl.h:90
std::string encode_length(T len)
Encode a length as a variable-length string.
Definition: length.h:36
A document read from a GlassDatabase.
Xapian::termcount get_doclength(Xapian::docid did, Xapian::Internal::intrusive_ptr< const GlassDatabase > db) const
Returns the length of document did.
TermIterator termlist_end() const
Equivalent end iterator for termlist_begin().
Definition: document.h:260
void check_wdf(Xapian::termcount wdf)
void get_database_write_lock(int flags, bool creating)
Get a write lock on the database, or throw an Xapian::DatabaseLockError if failure.
GlassDocDataTable docdata_table
Table storing document data.
friend class GlassAllDocsPostList
bool read_data(const string &data)
Fill list with data, and move the position to the start.
bool test() const
Test if the lock is held.
Definition: flint_lock.cc:75
void commit(glass_revision_number_t new_rev, int flags)
#define rare(COND)
Definition: config.h:543
include <sys/stat.h> with portability enhancements
const TermIterator get_unique_terms_begin() const
Begin iterator for unique terms in the query object.
Definition: query.cc:160
bool locked() const
Return true if the database is open for writing.
void close()
Close all the tables permanently.
#define GLASS_TABLE_EXTENSION
Glass table extension.
Definition: glass_defs.h:27
Xapian::docid get_last_docid() const
void delete_positionlist(Xapian::docid did, const std::string &term)
Xapian::termcount get_approx_size() const
Return approximate size of this termlist.
Access to metadata for a glass database.
#define REASONABLE_CHANGESET_SIZE
void remove_spelling(const string &word, Xapian::termcount freqdec) const
Virtual methods of Database::Internal.
#define CHANGES_MAGIC_STRING
virtual void read_position_list(GlassPositionList *pos_list, Xapian::docid did, const string &term) const
Virtual methods of Database::Internal.
void replace_document_data(Xapian::docid did, const std::string &data)
Replace the document data for document did.
Definition: glass_docdata.h:87
Hierarchy of classes which Xapian can throw as exceptions.
Class for iterating over a list of terms.
Definition: termiterator.h:41
map< Xapian::valueno, ValueStats > value_stats
Glass class for value streams.
unsigned XAPIAN_TERMCOUNT_BASE_TYPE termcount
A counts of terms.
Definition: types.h:72
Xapian::termcount get_wdf_upper_bound() const
void get_freqs(const string &term, Xapian::doccount *termfreq_ptr, Xapian::termcount *collfreq_ptr) const
Virtual methods of Database::Internal.
void clear()
Xapian::doccount get_spelling_frequency(const string &word) const
Virtual methods of Database::Internal.
void flush_db()
Flush any outstanding changes to the DB file of the table.
InvalidArgumentError indicates an invalid parameter value was passed to the API.
Definition: error.h:241
A TermList in a glass database.
bool document_exists(Xapian::docid did, Xapian::Internal::intrusive_ptr< const GlassDatabase > db) const
Check if document did exists.
RootInfo * root_to_set(Glass::table_type tbl)
void readahead_for_document(Xapian::docid did) const
A termlist containing all words which are spelling targets.
void add_synonym(const string &word, const string &synonym) const
Virtual methods of Database::Internal.
Xapian::docid add_document(const Xapian::Document &document)
Implementation of virtual methods: see Database::Internal for details.
void flush_post_list(GlassPostListTable &table, const std::string &term)
Flush postlist changes for term.
static string make_key(const string &term, Xapian::docid did)
Compose a key from a termname and docid.
string get_metadata(const string &key) const
Virtual methods of Database::Internal.
bool sync()
Definition: glass_table.h:540
void add_word(const std::string &word, Xapian::termcount freqinc)
void remove_synonym(const string &word, const string &synonym) const
Virtual methods of Database::Internal.
bool is_modified() const
Definition: glass_values.h:166
const int DB_OPEN
Open an existing database.
Definition: constants.h:50
Definition: fd.h:30
void close(bool permanent=false)
Close the Btree.
GlassPostListTable postlist_table
Table storing posting lists.
void readahead_for_query(const Xapian::Query &query)
Virtual methods of Database::Internal.
TermList * open_synonym_termlist(const string &term) const
Virtual methods of Database::Internal.
Xapian::termcount get_doclength(Xapian::docid did) const
Virtual methods of Database::Internal.
void set_changes(GlassChanges *changes_)
Indicates an attempt to use a feature which is unavailable.
Definition: error.h:719
bool is_modified() const
Determine whether the object contains uncommitted modifications.
Definition: glass_table.h:702
ValueList * open_value_list(Xapian::valueno slot) const
Virtual methods of Database::Internal.
DatabaseCreateError indicates a failure to create a database.
Definition: error.h:451
void get_used_docid_range(Xapian::docid &first, Xapian::docid &last) const
Find lowest and highest docids actually in use.
void commit(glass_revision_number_t revision, RootInfo *root_info)
Commit any outstanding changes to the table.
void add(const std::string &key, std::string tag, bool already_compressed=false)
Add a key/tag pair to the table, replacing any existing pair with the same key.
PositionList * open_position_list(Xapian::docid did, const string &term) const
Virtual methods of Database::Internal.
bool get_doclength(Xapian::docid did, Xapian::termcount &doclen) const
void add_posting(Xapian::docid did, const std::string &term, Xapian::doccount wdf)
void read_position_list(GlassPositionList *pos_list, Xapian::docid did, const string &term) const
Virtual methods of Database::Internal.
void get_used_docid_range(Xapian::docid &first, Xapian::docid &last) const
void flush(GlassPostListTable &table)
Flush all postlist table changes.
int get_flags() const
Definition: glass_table.h:631
LeafPostList * open_post_list(const string &tname) const
Virtual methods of Database::Internal.
void add_document(Xapian::termcount doclen)
Xapian::termcount flush_db()
Returns updated wordfreq upper bound.
void set_oldest_changeset(glass_revision_number_t rev)
Definition: glass_changes.h:67
Xapian::termcount get_doclength_lower_bound() const
bool has_positions() const
Virtual methods of Database::Internal.
string str(int value)
Convert int to std::string.
Definition: str.cc:90
GlassTermListTable termlist_table
Table storing term lists.
Wrapper class around a file descriptor to avoid leaks.
TermList * open_allterms(const string &prefix) const
Virtual methods of Database::Internal.
Information about the steps involved in performing a replication.
Definition: replication.h:33
int changeset_count
Number of changesets applied.
Definition: replication.h:35
#define GLASS_DEFAULT_BLOCKSIZE
Default B-tree block size.
Definition: glass_defs.h:30
C++ class definition for glass database.
ValueList * open_value_list(Xapian::valueno slot) const
Virtual methods of Database::Internal.
Class for iterating over document values.
Xapian::doccount flush_threshold
If change_count reaches this threshold we automatically flush.
#define CONST_STRLEN(S)
Returns the length of a string constant.
Definition: stringutils.h:43
void set_termlist(Xapian::docid did, const Xapian::Document &doc, Xapian::termcount doclen)
Set the termlist data for document did.
void set_positionlist(Xapian::docid did, const std::string &term, const std::string &s)
glass_revision_number_t get_revision_number() const
Get an object holding the revision number which the tables are opened at.
size_t io_read(int fd, char *p, size_t n, size_t min)
Read n bytes (or until EOF) into block pointed to by p from file descriptor fd.
Definition: io_utils.cc:123
bool open_tables(int flags)
Open all tables at most recent revision.
void set_wordfreq_upper_bound(Xapian::termcount ub)
GlassChanges * start(glass_revision_number_t old_rev, glass_revision_number_t rev, int flags)
void set_oldest_changeset(glass_revision_number_t changeset) const
static void throw_database_closed()
Throw an exception indicating that the database is closed.
void delete_document(Xapian::termcount doclen)
Indicates an attempt to access a database not present.
Definition: error.h:1055
Xapian::termcount get_doclength_lower_bound() const
Virtual methods of Database::Internal.
bool at_end() const
Return true if the current position is past the last term in this list.
Xapian::termcount get_wdf() const
Return the wdf for the term at the current position.
TermList * open_termlist(const std::string &word)
void replace_document(Xapian::docid did, const Xapian::Document &document)
Implementation of virtual methods: see Database::Internal for details.
void set_spelling_wordfreq_upper_bound(Xapian::termcount ub)
Xapian::Internal::intrusive_ptr< Internal > internal
Definition: document.h:63
#define LOGCALL_CTOR(CATEGORY, CLASS, PARAMS)
Definition: debuglog.h:478
void cancel()
Cancel any outstanding changes to the tables.
void read()
Read the version file and check it&#39;s a version we understand.
virtual bool has_uncommitted_changes() const
Return true if there are uncommitted changes.
void set_doclength(Xapian::docid did, Xapian::termcount doclen, bool add)
A backend designed for efficient indexing and retrieval, using compressed posting lists and a btree s...
bool is_open() const
Return true if this table is open.
Definition: glass_table.h:511
bool dir_exists(const char *path)
Test if a directory exists.
Definition: filetests.h:136
bool changed
True if and only if the replication corresponds to a change in the live version of the database...
Definition: replication.h:45
Xapian::docid modify_shortcut_docid
The document ID for the last document returned by open_document().
A cursor pointing to a position in a Btree table, for reading several entries in order, or finding approximate matches.
Definition: glass_cursor.h:147
Xapian::termcount get_wdf() const
Return the wdf for the term at the current position.
void flush_pos_lists(GlassPositionListTable &table)
Flush position changes.
std::string get_description() const
Return a string describing this object.
Definition: error.cc:93
void remove_word(const std::string &word, Xapian::termcount freqdec)
void get_freqs(const string &term, Xapian::doccount *termfreq_ptr, Xapian::termcount *collfreq_ptr) const
Virtual methods of Database::Internal.
void set_changes(GlassChanges *changes)
Set the GlassChanges object to write changed blocks to.
Definition: glass_table.h:726
void open(int flags_, const RootInfo &root_info, glass_revision_number_t rev)
Open the btree.
bool has_positions() const
Virtual methods of Database::Internal.
bool is_modified() const
Override methods of GlassTable.
TermList * open_spelling_termlist(const string &word) const
Virtual methods of Database::Internal.
Abstract base class for value streams.
Definition: valuelist.h:31
static Xapian::Query query(Xapian::Query::op op, const string &t1=string(), const string &t2=string(), const string &t3=string(), const string &t4=string(), const string &t5=string(), const string &t6=string(), const string &t7=string(), const string &t8=string(), const string &t9=string(), const string &t10=string())
Definition: api_anydb.cc:63
Xapian::termcount get_wdf_upper_bound(const string &term) const
Virtual methods of Database::Internal.
void open(int flags_, const RootInfo &root_info, glass_revision_number_t rev)
void pack_uint(std::string &s, U value)
Append an encoded unsigned integer to a string.
Definition: pack.h:382
friend class GlassTermList
TermList * open_spelling_wordlist() const
Virtual methods of Database::Internal.
bool term_exists(const string &tname) const
Virtual methods of Database::Internal.
XAPIAN_TERMCOUNT_BASE_TYPE termcount_diff
A signed difference between two counts of terms.
Definition: types.h:79
bool term_exists(const string &tname) const
Virtual methods of Database::Internal.
void request_document(Xapian::docid) const
Virtual methods of Database::Internal.
void set_revision_number(int flags, glass_revision_number_t new_revision)
Set the revision number in the tables.
void check_flush_threshold()
Check if we should autoflush.
Xapian::termcount positionlist_count(Xapian::docid did, const string &term) const
Virtual methods of Database::Internal.
void flush_db()
Override methods of GlassTable.
Replication protocol version and message numbers.
Xapian::doccount get_value_freq(Xapian::valueno slot) const
Virtual methods of Database::Internal.
bool get_exact_entry(const std::string &key, std::string &tag) const
Read an entry from the table, if and only if it is exactly that being asked for.
bool term_exists(const string &term) const
unsigned XAPIAN_DOCID_BASE_TYPE doccount
A count of documents.
Definition: types.h:38
std::string get_value_upper_bound(Xapian::valueno slot) const
Virtual methods of Database::Internal.
Xapian::doccount get_word_frequency(const std::string &word) const
void remove_synonym(const std::string &term, const std::string &synonym)
Remove a synonym for term.
glass_revision_number_t get_oldest_changeset() const
Replication support for Xapian databases.
GlassValueManager class.
Xapian::totallength get_total_doclen() const
void cancel(const RootInfo &root_info, glass_revision_number_t rev)
Override methods of GlassTable.
GlassDatabase(const string &db_dir_, int flags=Xapian::DB_READONLY_, unsigned int block_size=0u)
Create and open a glass database.
TermList * open_synonym_keylist(const string &prefix) const
Virtual methods of Database::Internal.
Xapian::termcount get_spelling_wordfreq_upper_bound() const
All exceptions thrown by Xapian are subclasses of Xapian::Error.
Definition: error.h:43
GlassCursor * cursor_get() const
Get a cursor for reading from the table.
void update_posting(Xapian::docid did, const std::string &term, Xapian::termcount old_wdf, Xapian::termcount new_wdf)
Pack types into strings and unpack them again.
unsigned valueno
The number for a value slot in a document.
Definition: types.h:108
void remove_posting(Xapian::docid did, const std::string &term, Xapian::doccount wdf)
Wrappers for low-level POSIX I/O routines.
Xapian::doccount get_value_freq(Xapian::valueno slot) const
Virtual methods of Database::Internal.
Various handy helpers which std::string really should provide.
bool unpack_uint(const char **p, const char *end, U *result)
Decode an unsigned integer from a string.
Definition: pack.h:413
void dtor_called()
Internal method to perform cleanup when a writable database is destroyed with uncommitted changes...
Definition: database.cc:87
PositionList * open_position_list(Xapian::docid did, const string &term) const
Virtual methods of Database::Internal.
#define MAX_DB_COPIES_PER_CONVERSATION
void add_spelling(const string &word, Xapian::termcount freqinc) const
Virtual methods of Database::Internal.
std::string get_value_upper_bound(Xapian::valueno slot) const
Virtual methods of Database::Internal.
const int DB_NO_TERMLIST
When creating a database, don&#39;t create a termlist table.
Definition: constants.h:136
Xapian::termcount get_doclength(Xapian::docid did) const
Virtual methods of Database::Internal.
void send_message(char type, const std::string &s, double end_time)
Send a message.
#define posixy_open
bool database_exists()
Return true if a database exists at the path specified for this database.
void delete_document(Xapian::docid did, std::map< Xapian::valueno, ValueStats > &value_stats)
reason lock(bool exclusive, bool wait, std::string &explanation)
Attempt to obtain the lock.
Definition: flint_lock.cc:125
Xapian::doccount get_value_freq(Xapian::valueno slot) const
Definition: glass_values.h:137
TermList * next()
Advance the current position to the next term in the termlist.
#define CHANGES_VERSION
const int DB_RETRY_LOCK
If the database is already locked, retry the lock.
Definition: constants.h:145
void cancel(const RootInfo &root_info, glass_revision_number_t rev)
Cancel any outstanding changes.
void get_freqs(const std::string &term, Xapian::doccount *termfreq_ptr, Xapian::termcount *collfreq_ptr, Xapian::termcount *wdfub_ptr=NULL) const
Returns frequencies for a term.
#define LOGLINE(a, b)
Definition: debuglog.h:483
const int DB_CREATE_OR_OVERWRITE
Create database if it doesn&#39;t already exist, or overwrite if it does.
Definition: constants.h:38
void flush_post_lists(GlassPostListTable &table, const std::string &pfx)
Flush postlist changes for all terms which start with pfx.
A TermList in a glass database.
unsigned XAPIAN_DOCID_BASE_TYPE docid
A unique identifier for a document.
Definition: types.h:52
Class representing a query.
Definition: query.h:46
DatabaseError indicates some sort of database related error.
Definition: error.h:367
std::string get_data() const
Get data stored in the document.
Definition: omdocument.cc:71
const std::string write(glass_revision_number_t new_rev, int flags)
Abstract base class for iterating term positions in a document.
Definition: positionlist.h:31
TermList * open_term_list(Xapian::docid did) const
Virtual methods of Database::Internal.
void cancel(const RootInfo &root_info, glass_revision_number_t rev)
Override methods of GlassTable.
Xapian::totallength get_total_length() const
Virtual methods of Database::Internal.
glass_revision_number_t get_revision() const
void clear_synonyms(const string &word) const
Virtual methods of Database::Internal.
A smart pointer that uses intrusive reference counting.
Definition: intrusive_ptr.h:81
bool exists() const
Determine whether the btree exists on disk.
Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy) const
Virtual methods of Database::Internal.
std::string get_uuid_string() const
Return UUID in the standard 36 character string format.
TermList * open_allterms(const string &prefix) const
Virtual methods of Database::Internal.
bool has_uncommitted_changes() const
Return true if there are uncommitted changes.
void add_document(Xapian::docid did, const Xapian::Document &doc, std::map< Xapian::valueno, ValueStats > &value_stats)
A document read from a GlassDatabase.
void flush_doclengths(GlassPostListTable &table)
Flush document length changes.
void set_last_docid(Xapian::docid did)
virtual Xapian::termcount positionlist_count(Xapian::docid did, const string &term) const
Virtual methods of Database::Internal.
TermIterator termlist_begin() const
Iterator for the terms in this document.
Definition: omdocument.cc:197
GlassValueManager value_manager
Value manager.
std::string get_value_upper_bound(Xapian::valueno slot) const
Definition: glass_values.h:147
GlassSpellingTable spelling_table
Table storing spelling correction data.
A handle representing a document in a Xapian database.
Definition: document.h:61
bool del(const std::string &key)
Delete an entry from the table.
std::string get_value_lower_bound(Xapian::valueno slot) const
Definition: glass_values.h:142
Wrapper around standard unique_ptr template.
void replace_document(Xapian::docid did, const Xapian::Document &doc, std::map< Xapian::valueno, ValueStats > &value_stats)
Debug logging macros.
#define LOGCALL(CATEGORY, TYPE, FUNC, PARAMS)
Definition: debuglog.h:476
bool readonly
Whether the database is readonly.
bool get_deltas(const std::string &term, Xapian::termcount_diff &tf_delta, Xapian::termcount_diff &cf_delta) const
Xapian::doccount get_doccount() const
Xapian::termcount get_doclength_upper_bound() const
Virtual methods of Database::Internal.
string get_revision_info() const
Virtual methods of Database::Internal.
bool is_modified() const
Override methods of GlassTable.
void apply()
Apply changes.
#define MAX_SAFE_TERM_LENGTH
Xapian::termcount get_unique_terms(Xapian::docid did) const
Virtual methods of Database::Internal.
void clear_synonyms(const std::string &term)
Remove all synonyms for term.
void get_changeset_revisions(const string &path, glass_revision_number_t *startrev, glass_revision_number_t *endrev) const
Get the revision stored in a changeset.