xapian-core  1.4.19
backendmanager_multi.cc
Go to the documentation of this file.
1 
4 /* Copyright (C) 2007,2008,2009,2011,2012,2013,2015,2017,2018,2019,2020 Olly Betts
5  * Copyright (C) 2008 Lemur Consulting Ltd
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License as
9  * published by the Free Software Foundation; either version 2 of the
10  * License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include <config.h>
23 
24 #include "backendmanager_multi.h"
25 
26 #include "errno_to_string.h"
27 #include "filetests.h"
28 #include "index_utils.h"
29 #include "str.h"
30 
31 #include "safeunistd.h"
32 
33 #include <cerrno>
34 #include <cstdio> // For rename().
35 #include <cstring>
36 
37 using namespace std;
38 
39 BackendManagerMulti::BackendManagerMulti(const std::string& datadir_,
40  vector<BackendManager*> sub_managers_)
41  : BackendManager(datadir_),
42  sub_managers(sub_managers_)
43 {
44  cachedir = ".multi";
45  if (sub_managers.size() == 2 &&
46  sub_managers[0]->get_dbtype() == sub_managers[1]->get_dbtype()) {
47  cachedir += sub_managers[0]->get_dbtype();
48  } else {
49  for (auto sub_manager : sub_managers) {
50  cachedir += sub_manager->get_dbtype();
51  }
52  }
53  // Ensure the directory we store cached test databases in exists.
55 }
56 
57 std::string
59 {
60  string dbtype = "multi";
61  if (sub_managers.size() == 2 &&
62  sub_managers[0]->get_dbtype() == sub_managers[1]->get_dbtype()) {
63  dbtype += "_" + sub_managers[0]->get_dbtype();
64  } else {
65  for (auto sub_manager : sub_managers) {
66  dbtype += "_" + sub_manager->get_dbtype();
67  }
68  }
69  return dbtype;
70 }
71 
72 #define NUMBER_OF_SUB_DBS 2
73 
74 string
76  const vector<string>& files)
77 {
78  string dbname;
79  if (!name.empty()) {
80  dbname = name;
81  } else {
82  dbname = "db";
83  for (const string& file : files) {
84  dbname += "__";
85  dbname += file;
86  }
87  }
88 
89  string db_path = cachedir;
90  db_path += '/';
91  db_path += dbname;
92 
93  if (!name.empty()) {
94  if (unlink(db_path.c_str()) < 0 && errno != ENOENT) {
95  string msg = "Couldn't unlink file '";
96  msg += db_path;
97  msg += "' (";
98  errno_to_string(errno, msg);
99  msg += ')';
100  throw msg;
101  }
102  } else {
103  if (file_exists(db_path)) return db_path;
104  }
105 
106  string tmpfile = db_path + ".tmp";
107  ofstream out(tmpfile.c_str());
108  if (!out.is_open()) {
109  string msg = "Couldn't create file '";
110  msg += tmpfile;
111  msg += "' (";
112  errno_to_string(errno, msg);
113  msg += ')';
114  throw msg;
115  }
116 
117  // Open NUMBER_OF_SUB_DBS databases and index files to them alternately so
118  // a multi-db combining them contains the documents in the expected order.
120 
121  string dbbase = db_path;
122  dbbase += "___";
123  size_t dbbase_len = dbbase.size();
124 
125  for (size_t n = 0; n < NUMBER_OF_SUB_DBS; ++n) {
126  const string& subtype = sub_managers[n]->get_dbtype();
127  int flags = Xapian::DB_CREATE_OR_OVERWRITE;
128  if (subtype == "glass") {
129  flags |= Xapian::DB_BACKEND_GLASS;
130  dbbase += str(n);
131  dbs.add_database(Xapian::WritableDatabase(dbbase, flags));
132  out << subtype << ' ' << dbname << "___" << n << '\n';
133  } else if (subtype == "chert") {
134  flags |= Xapian::DB_BACKEND_CHERT;
135  dbbase += str(n);
136  dbs.add_database(Xapian::WritableDatabase(dbbase, flags));
137  out << subtype << ' ' << dbname << "___" << n << '\n';
138  } else if (subtype == "remoteprog_glass") {
139  flags |= Xapian::DB_BACKEND_GLASS;
140  dbbase += str(n);
141  Xapian::WritableDatabase remote_db(dbbase, flags);
142  remote_db.close();
143  string args = sub_managers[n]->get_writable_database_args(dbbase,
144  300000);
145 
146  dbs.add_database(
148 
149  out << "remote :" << BackendManager::get_xapian_progsrv_command()
150  << " " << args << '\n';
151  } else {
152  string msg = "Unknown multidb subtype: ";
153  msg += subtype;
154  throw msg;
155  }
156  dbbase.resize(dbbase_len);
157  }
158 
159  out.close();
160 
161  FileIndexer(get_datadir(), files).index_to(dbs);
162  dbs.close();
163 
164 retry:
165  if (rename(tmpfile.c_str(), db_path.c_str()) < 0) {
166  if (errno == EACCES) {
167  // At least when run under appveyor, sometimes this rename fails
168  // with EACCES. The destination file doesn't exist (and from
169  // debugging it shouldn't), which suggests that tmpfile is still
170  // open, but it shouldn't be, and a sleep+retry makes it work.
171  // Perhaps some AV is kicking in and opening newly created files
172  // to inspect them or something?
173  //
174  // FIXME: It would be good to get to the bottom of this!
175  sleep(1);
176  goto retry;
177  }
178  throw Xapian::DatabaseError("rename failed", errno);
179  }
180 
181  last_wdb_path = db_path;
182  return db_path;
183 }
184 
185 string
186 BackendManagerMulti::do_get_database_path(const vector<string> & files)
187 {
188  return createdb_multi(string(), files);
189 }
190 
192 BackendManagerMulti::get_writable_database(const string& name, const string& file)
193 {
194  vector<string> files;
195  if (!file.empty()) files.push_back(file);
196  return Xapian::WritableDatabase(createdb_multi(name, files));
197 }
198 
199 string
201 {
202  return cachedir + "/" + name;
203 }
204 
206 BackendManagerMulti::get_remote_database(const std::vector<std::string>& files,
207  unsigned int timeout)
208 {
209  Xapian::Database db;
210  size_t remotes = 0;
211  for (auto sub_manager : sub_managers) {
212  if (sub_manager->get_dbtype().find("remote") == string::npos) {
213  db.add_database(sub_manager->get_database(files));
214  continue;
215  }
216 
217  ++remotes;
218  db.add_database(sub_manager->get_remote_database(files, timeout));
219  }
220 
221  if (remotes == 0) {
222  // It's useful to support mixed local/remote multi databases with a
223  // custom timeout so we can test timeout and keepalive handling for
224  // this case, but this method shouldn't be called on an all-local
225  // multi database.
226  const char* m = "BackendManager::get_remote_database() called for "
227  "multi with no remote shards";
229  }
230  return db;
231 }
232 
233 string
235 {
236  return cachedir + "/" + name;
237 }
238 
239 string
241 {
243 }
244 
247 {
249 }
250 
251 string
253 {
254  return last_wdb_path;
255 }
static const char * get_xapian_progsrv_command()
Get the command line required to run xapian-progsrv.
const std::string & get_datadir() const
Get the directory to store data in.
This class is used to access a database, or a group of databases.
Definition: database.h:68
unsigned timeout
A timeout value in milliseconds.
Definition: types.h:100
InvalidOperationError indicates the API was used in an invalid way.
Definition: error.h:283
Convert errno value to std::string, thread-safe if possible.
void sleep(double t)
Sleep until the time represented by this object.
Definition: realtime.h:127
STL namespace.
Convert types to std::string.
Xapian::WritableDatabase get_writable_database_again()
Create a WritableDatabase object for the last opened WritableDatabase.
Utility functions for testing files.
std::string last_wdb_path
The path of the last writable database used.
BackendManager subclass for multi databases.
std::string get_writable_database_path(const std::string &name)
Get the path of Xapian::WritableDatabase instance.
std::string get_generated_database_path(const std::string &name)
Get the path to use for generating a database, if supported.
const int DB_BACKEND_GLASS
Use the glass backend.
Definition: constants.h:158
bool create_dir_if_needed(const std::string &dirname)
Create the directory dirname if needed.
std::string createdb_multi(const std::string &name, const std::vector< std::string > &files)
This class provides read/write access to a database.
Definition: database.h:785
void errno_to_string(int e, string &s)
virtual Xapian::WritableDatabase get_remote_writable_database(std::string args)
Get a remote Xapian::WritableDatabase instance with specified args.
std::string get_compaction_output_path(const std::string &name)
Get a path to compact a database to.
std::string get_writable_database_path_again()
Get the path of the last opened WritableDatabase.
string str(int value)
Convert int to std::string.
Definition: str.cc:90
std::vector< BackendManager * > sub_managers
void add_database(const Database &database)
Add an existing database (or group of databases) to those accessed by this object.
Definition: omdatabase.cc:148
size_t size() const
Return number of shards in this Database object.
Definition: database.h:93
Xapian::Database get_remote_database(const std::vector< std::string > &files, unsigned int timeout)
Get a remote database instance with the specified timeout.
std::string get_dbtype() const
Return a string representing the current database type.
Xapian::WritableDatabase get_writable_database(const std::string &name, const std::string &file)
Create a Multi Xapian::WritableDatabase object indexing a single file.
std::string do_get_database_path(const std::vector< std::string > &files)
Get the path of the Xapian::Database instance.
char name[9]
Definition: dbcheck.cc:55
void add_database(const WritableDatabase &other)
Add shards from another WritableDatabase.
Definition: database.h:891
void index_to(Xapian::WritableDatabase &db)
Definition: index_utils.cc:52
<unistd.h>, but with compat.
Definition: header.h:151
BackendManagerMulti(const BackendManagerMulti &)
Don&#39;t allow copying.
utility functions for indexing testcase data
const int DB_CREATE_OR_OVERWRITE
Create database if it doesn&#39;t already exist, or overwrite if it does.
Definition: constants.h:38
DatabaseError indicates some sort of database related error.
Definition: error.h:367
bool file_exists(const char *path)
Test if a file exists.
Definition: filetests.h:39
#define NUMBER_OF_SUB_DBS
virtual void close()
Close the database.
Definition: omdatabase.cc:138
const int DB_BACKEND_CHERT
Use the chert backend.
Definition: constants.h:170