xapian-core  1.4.19
glass_database.h
Go to the documentation of this file.
1 
4 /* Copyright 1999,2000,2001 BrightStation PLC
5  * Copyright 2002 Ananova Ltd
6  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2019 Olly Betts
7  * Copyright 2008 Lemur Consulting Ltd
8  *
9  * This program is free software; you can redistribute it and/or
10  * modify it under the terms of the GNU General Public License as
11  * published by the Free Software Foundation; either version 2 of the
12  * License, or (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
22  * USA
23  */
24 
25 #ifndef OM_HGUARD_GLASS_DATABASE_H
26 #define OM_HGUARD_GLASS_DATABASE_H
27 
28 #include "backends/backends.h"
29 #include "backends/database.h"
30 #include "glass_changes.h"
31 #include "glass_docdata.h"
32 #include "glass_inverter.h"
33 #include "glass_positionlist.h"
34 #include "glass_postlist.h"
35 #include "glass_spelling.h"
36 #include "glass_synonym.h"
37 #include "glass_termlisttable.h"
38 #include "glass_values.h"
39 #include "glass_version.h"
40 #include "../flint_lock.h"
41 #include "glass_defs.h"
42 #include "backends/valuestats.h"
43 
44 #include "noreturn.h"
45 
46 #include "xapian/compactor.h"
47 #include "xapian/constants.h"
48 
49 #include <map>
50 
51 class GlassTermList;
53 class RemoteConnection;
54 
59  friend class GlassWritableDatabase;
60  friend class GlassTermList;
61  friend class GlassPostList;
62  friend class GlassAllTermsList;
63  friend class GlassAllDocsPostList;
64 
65  private:
68  std::string db_dir;
69 
72  bool readonly;
73 
79 
87 
91 
95 
98 
102 
106 
110 
113 
116 
120  bool database_exists();
121 
125  void create_and_open_tables(int flags, unsigned int blocksize);
126 
135  bool open_tables(int flags);
136 
146  void get_database_write_lock(int flags, bool creating);
147 
154 
165  void set_revision_number(int flags, glass_revision_number_t new_revision);
166 
170  bool reopen();
171 
174  void close();
175 
182  const std::string & msg);
183 
192  void apply();
193 
196  void cancel();
197 
200  void send_whole_database(RemoteConnection & conn, double end_time);
201 
204  void get_changeset_revisions(const string & path,
205  glass_revision_number_t * startrev,
206  glass_revision_number_t * endrev) const;
207 
208  public:
228  explicit GlassDatabase(const string& db_dir_,
229  int flags = Xapian::DB_READONLY_,
230  unsigned int block_size = 0u);
231 
232  explicit GlassDatabase(int fd);
233 
234  ~GlassDatabase();
235 
238  return postlist_table.cursor_get();
239  }
240 
247 
255  void get_freqs(const string & term,
256  Xapian::doccount * termfreq_ptr,
257  Xapian::termcount * collfreq_ptr) const;
259  std::string get_value_lower_bound(Xapian::valueno slot) const;
260  std::string get_value_upper_bound(Xapian::valueno slot) const;
263  Xapian::termcount get_wdf_upper_bound(const string & term) const;
264  bool term_exists(const string & tname) const;
265  bool has_positions() const;
266 
267  LeafPostList * open_post_list(const string & tname) const;
270  bool lazy) const;
271 
272  virtual void read_position_list(GlassPositionList* pos_list,
273  Xapian::docid did,
274  const string& term) const;
276  const string& term) const;
278  const string& term) const;
280  TermList * open_allterms(const string & prefix) const;
281 
282  TermList * open_spelling_termlist(const string & word) const;
284  Xapian::doccount get_spelling_frequency(const string & word) const;
285 
286  TermList * open_synonym_termlist(const string & term) const;
287  TermList * open_synonym_keylist(const string & prefix) const;
288 
289  string get_metadata(const string & key) const;
290  TermList * open_metadata_keylist(const std::string &prefix) const;
291  void write_changesets_to_fd(int fd,
292  const string & start_revision,
293  bool need_whole_db,
294  Xapian::ReplicationInfo * info);
295  string get_revision_info() const;
296  string get_uuid() const;
297 
298  void request_document(Xapian::docid /*did*/) const;
301 
302  XAPIAN_NORETURN(void throw_termlist_table_close_exception() const);
303 
304  int get_backend_info(string * path) const {
305  if (path) *path = db_dir;
306  return BACKEND_GLASS;
307  }
308 
309  bool single_file() const { return version_file.single_file(); }
310 
311  void get_used_docid_range(Xapian::docid & first,
312  Xapian::docid & last) const;
313 
315  virtual bool has_uncommitted_changes() const;
316 
317  bool locked() const;
318 
319  static void compact(Xapian::Compactor * compactor,
320  const char * destdir,
321  int fd,
322  const std::vector<Xapian::Database::Internal*>& sources,
323  const std::vector<Xapian::docid> & offset,
324  size_t block_size,
326  unsigned flags,
327  Xapian::docid last_docid);
328 };
329 
334 
335  mutable map<Xapian::valueno, ValueStats> value_stats;
336 
341 
344 
352 
356 
361  void check_flush_threshold();
362 
364  void flush_postlist_changes();
365 
367  void close();
368 
370  void apply();
371 
373 
376  void commit();
377 
379  void cancel();
380 
382  Xapian::docid add_document_(Xapian::docid did,
383  const Xapian::Document& document);
384  // Stop the default implementation of delete_document(term) and
385  // replace_document(term) from being hidden. This isn't really
386  // a problem as we only try to call them through the base class
387  // (where they aren't hidden) but some compilers generate a warning
388  // about the hiding.
391  void delete_document(Xapian::docid did);
392  void replace_document(Xapian::docid did, const Xapian::Document & document);
393 
395  bool lazy) const;
396 
398 
399  public:
411  GlassWritableDatabase(const string &dir, int flags, int block_size);
412 
414 
419  void get_freqs(const string & term,
420  Xapian::doccount * termfreq_ptr,
421  Xapian::termcount * collfreq_ptr) const;
423  std::string get_value_lower_bound(Xapian::valueno slot) const;
424  std::string get_value_upper_bound(Xapian::valueno slot) const;
425  bool term_exists(const string & tname) const;
426  bool has_positions() const;
427 
428  LeafPostList * open_post_list(const string & tname) const;
430 
431  void read_position_list(GlassPositionList* pos_list,
432  Xapian::docid did,
433  const string& term) const;
435  const string& term) const;
437  const string& term) const;
438  TermList * open_allterms(const string & prefix) const;
439 
440  void add_spelling(const string & word, Xapian::termcount freqinc) const;
441  void remove_spelling(const string & word,
442  Xapian::termcount freqdec) const;
444 
445  TermList * open_synonym_keylist(const string & prefix) const;
446  void add_synonym(const string & word, const string & synonym) const;
447  void remove_synonym(const string & word, const string & synonym) const;
448  void clear_synonyms(const string & word) const;
449 
450  void set_metadata(const string & key, const string & value);
453 
455  bool has_uncommitted_changes() const;
456 };
457 
458 #endif /* OM_HGUARD_GLASS_DATABASE_H */
GlassVersion version_file
The file describing the Glass database.
bool reopen()
Re-open tables to recover from an overwritten condition, or just get most up-to-date version...
Define the XAPIAN_NORETURN macro.
GlassVersion class.
FlintLock lock
Lock object.
virtual void replace_document(Xapian::docid did, const Xapian::Document &document)
Replace a given document in the database.
Definition: database.cc:186
Xapian::termcount get_unique_terms(Xapian::docid did) const
Virtual methods of Database::Internal.
A RemoteConnection object provides a bidirectional connection to another RemoteConnection object on a...
Statistics about values.
GlassPositionListTable position_table
Table storing position lists.
void throw_termlist_table_close_exception() const
A position list in a glass database.
void close()
Close all the tables permanently.
TermList * open_synonym_keylist(const string &prefix) const
Virtual methods of Database::Internal.
string get_uuid() const
Virtual methods of Database::Internal.
Xapian::docid get_lastdocid() const
Virtual methods of Database::Internal.
std::string db_dir
Directory to store databases in.
TermList * open_metadata_keylist(const std::string &prefix) const
Virtual methods of Database::Internal.
void create_and_open_tables(int flags, unsigned int blocksize)
Create new tables, and open them.
void apply()
Apply any outstanding changes to the tables.
GlassChanges changes
Replication changesets.
void send_whole_database(RemoteConnection &conn, double end_time)
Send a set of messages which transfer the whole database.
Xapian::doccount get_doccount() const
Virtual methods of Database::Internal.
virtual Xapian::docid add_document(const Xapian::Document &document)
Add a new document to the database.
Definition: database.cc:161
virtual void commit()
Commit pending modifications to the database.
Definition: database.cc:102
Class which "inverts the file".
Base class for databases.
Definition: database.h:56
uint4 glass_revision_number_t
The revision number of a glass database.
Definition: glass_defs.h:61
GlassSynonymTable synonym_table
Table storing synonym data.
std::string get_value_lower_bound(Xapian::valueno slot) const
Virtual methods of Database::Internal.
XAPIAN_TOTALLENGTH_TYPE totallength
The total length of all documents in a database.
Definition: types.h:139
void write_changesets_to_fd(int fd, const string &start_revision, bool need_whole_db, Xapian::ReplicationInfo *info)
Virtual methods of Database::Internal.
void modifications_failed(glass_revision_number_t new_revision, const std::string &msg)
Called if a modifications fail.
Constants in the Xapian namespace.
The GlassVersion class manages the revision files.
Definition: glass_version.h:94
Subclass of GlassTable which holds document data.
static void compact(Xapian::Compactor *compactor, const char *destdir, int fd, const std::vector< Xapian::Database::Internal *> &sources, const std::vector< Xapian::docid > &offset, size_t block_size, Xapian::Compactor::compaction_level compaction, unsigned flags, Xapian::docid last_docid)
double end_time(double timeout)
Return the end time for a timeout in timeout seconds.
Definition: realtime.h:95
virtual void clear_synonyms(const string &term) const
Clear all synonyms for a term.
Definition: database.cc:286
A document in the database, possibly plus modifications.
Definition: document.h:41
Compact a database, or merge and compact several.
Abstract base class for termlists.
Definition: termlist.h:39
Postlists in glass databases.
Xapian::Document::Internal * modify_shortcut_document
A pointer to the last document which was returned by open_document(), or NULL if there is no such val...
Definitions, types, etc for use inside glass.
GlassCursor * get_postlist_cursor() const
Get a postlist table cursor (used by GlassValueList).
virtual void set_metadata(const string &key, const string &value)
Set the metadata associated with a given key.
Definition: database.cc:306
Subclass of GlassTable which holds termlists.
Xapian::doccount change_count
The number of documents added, deleted, or replaced since the last flush.
Abstract base class for leaf postlists.
Definition: leafpostlist.h:38
glass_revision_number_t get_next_revision_number() const
Get an object holding the next revision number which should be used in the tables.
std::string term
The term name for this postlist (empty for an alldocs postlist).
Definition: leafpostlist.h:51
A position list in a glass database.
friend class GlassWritableDatabase
void get_database_write_lock(int flags, bool creating)
Get a write lock on the database, or throw an Xapian::DatabaseLockError if failure.
GlassDocDataTable docdata_table
Table storing document data.
bool locked() const
Return true if the database is open for writing.
A writable glass database.
virtual void delete_document(Xapian::docid did)
Delete a document in the database.
Definition: database.cc:169
virtual void read_position_list(GlassPositionList *pos_list, Xapian::docid did, const string &term) const
Virtual methods of Database::Internal.
map< Xapian::valueno, ValueStats > value_stats
virtual void add_synonym(const string &term, const string &synonym) const
Add a synonym for a term.
Definition: database.cc:274
unsigned XAPIAN_TERMCOUNT_BASE_TYPE termcount
A counts of terms.
Definition: types.h:72
Xapian::doccount get_spelling_frequency(const string &word) const
Virtual methods of Database::Internal.
A TermList in a glass database.
BACKEND_* constants.
string get_metadata(const string &key) const
Virtual methods of Database::Internal.
GlassPostListTable postlist_table
Table storing posting lists.
void readahead_for_query(const Xapian::Query &query)
Virtual methods of Database::Internal.
TermList * open_synonym_termlist(const string &term) const
Virtual methods of Database::Internal.
ValueList * open_value_list(Xapian::valueno slot) const
Virtual methods of Database::Internal.
void get_used_docid_range(Xapian::docid &first, Xapian::docid &last) const
Find lowest and highest docids actually in use.
Glass changesets.
PositionList * open_position_list(Xapian::docid did, const string &term) const
Virtual methods of Database::Internal.
Compact a database, or merge and compact several.
Definition: compactor.h:42
LeafPostList * open_post_list(const string &tname) const
Virtual methods of Database::Internal.
virtual void remove_synonym(const string &term, const string &synonym) const
Remove a synonym for a term.
Definition: database.cc:280
bool has_positions() const
Virtual methods of Database::Internal.
GlassTermListTable termlist_table
Table storing term lists.
Information about the steps involved in performing a replication.
Definition: replication.h:33
Xapian::doccount flush_threshold
If change_count reaches this threshold we automatically flush.
virtual void remove_spelling(const string &word, Xapian::termcount freqdec) const
Remove a word from the spelling dictionary.
Definition: database.cc:250
A postlist in a glass database.
glass_revision_number_t get_revision_number() const
Get an object holding the revision number which the tables are opened at.
bool open_tables(int flags)
Open all tables at most recent revision.
Xapian::termcount get_doclength_lower_bound() const
Virtual methods of Database::Internal.
void cancel()
Cancel any outstanding changes to the tables.
virtual bool has_uncommitted_changes() const
Return true if there are uncommitted changes.
A backend designed for efficient indexing and retrieval, using compressed posting lists and a btree s...
Xapian::docid modify_shortcut_docid
The document ID for the last document returned by open_document().
A cursor pointing to a position in a Btree table, for reading several entries in order, or finding approximate matches.
Definition: glass_cursor.h:147
void get_freqs(const string &term, Xapian::doccount *termfreq_ptr, Xapian::termcount *collfreq_ptr) const
Virtual methods of Database::Internal.
TermList * open_spelling_termlist(const string &word) const
Virtual methods of Database::Internal.
Abstract base class for value streams.
Definition: valuelist.h:31
static Xapian::Query query(Xapian::Query::op op, const string &t1=string(), const string &t2=string(), const string &t3=string(), const string &t4=string(), const string &t5=string(), const string &t6=string(), const string &t7=string(), const string &t8=string(), const string &t9=string(), const string &t10=string())
Definition: api_anydb.cc:63
Xapian::termcount get_wdf_upper_bound(const string &term) const
Virtual methods of Database::Internal.
TermList * open_spelling_wordlist() const
Virtual methods of Database::Internal.
bool term_exists(const string &tname) const
Virtual methods of Database::Internal.
void request_document(Xapian::docid) const
Virtual methods of Database::Internal.
void set_revision_number(int flags, glass_revision_number_t new_revision)
Set the revision number in the tables.
bool single_file() const
unsigned XAPIAN_DOCID_BASE_TYPE doccount
A count of documents.
Definition: types.h:38
GlassValueManager class.
GlassDatabase(const string &db_dir_, int flags=Xapian::DB_READONLY_, unsigned int block_size=0u)
Create and open a glass database.
virtual void add_spelling(const string &word, Xapian::termcount freqinc) const
Add a word to the spelling dictionary.
Definition: database.cc:244
Xapian::docid did
Document id we&#39;re currently at.
GlassCursor * cursor_get() const
Get a cursor for reading from the table.
compaction_level
Compaction level.
Definition: compactor.h:48
unsigned valueno
The number for a value slot in a document.
Definition: types.h:108
Xapian::doccount get_value_freq(Xapian::valueno slot) const
Virtual methods of Database::Internal.
virtual void invalidate_doc_object(Xapian::Document::Internal *obj) const
Notify the database that document is no longer valid.
Definition: database.cc:353
std::string get_value_upper_bound(Xapian::valueno slot) const
Virtual methods of Database::Internal.
Xapian::termcount get_doclength(Xapian::docid did) const
Virtual methods of Database::Internal.
bool database_exists()
Return true if a database exists at the path specified for this database.
bool single_file() const
unsigned XAPIAN_DOCID_BASE_TYPE docid
A unique identifier for a document.
Definition: types.h:52
Class representing a query.
Definition: query.h:46
Inverter class which "inverts the file".
int get_backend_info(string *path) const
Get backend information about this database.
Abstract base class for iterating term positions in a document.
Definition: positionlist.h:31
TermList * open_term_list(Xapian::docid did) const
Virtual methods of Database::Internal.
Xapian::totallength get_total_length() const
Virtual methods of Database::Internal.
Synonym data for a glass database.
Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy) const
Virtual methods of Database::Internal.
Spelling correction data for a glass database.
TermList * open_allterms(const string &prefix) const
Virtual methods of Database::Internal.
virtual Xapian::termcount positionlist_count(Xapian::docid did, const string &term) const
Virtual methods of Database::Internal.
GlassValueManager value_manager
Value manager.
GlassSpellingTable spelling_table
Table storing spelling correction data.
A handle representing a document in a Xapian database.
Definition: document.h:61
bool readonly
Whether the database is readonly.
Xapian::termcount get_doclength_upper_bound() const
Virtual methods of Database::Internal.
string get_revision_info() const
Virtual methods of Database::Internal.
void get_changeset_revisions(const string &path, glass_revision_number_t *startrev, glass_revision_number_t *endrev) const
Get the revision stored in a changeset.