xapian-core  1.4.21
xapian-replicate.cc
Go to the documentation of this file.
1 
4 /* Copyright (C) 2008,2011,2012,2015 Olly Betts
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License as
8  * published by the Free Software Foundation; either version 2 of the
9  * License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
19  * USA
20  */
21 
22 #include <config.h>
23 
24 #include "net/replicatetcpclient.h"
25 
26 #include <xapian.h>
27 
28 #include "gnu_getopt.h"
29 #include "stringutils.h"
30 #include "safeunistd.h"
31 
32 #include <iostream>
33 
34 using namespace std;
35 
36 #define PROG_NAME "xapian-replicate"
37 #define PROG_DESC "Replicate a database from a master server to a local copy"
38 
39 #define OPT_HELP 1
40 #define OPT_VERSION 2
41 
42 // Wait DEFAULT_INTERVAL seconds between updates unless --interval is passed.
43 #define DEFAULT_INTERVAL 60
44 
45 // Number of seconds before we assume that a reader will be closed.
46 #define READER_CLOSE_TIME 30
47 
48 // Socket level timeout (in seconds).
49 #define DEFAULT_TIMEOUT 0
50 
51 static void show_usage() {
52  cout << "Usage: " PROG_NAME " [OPTIONS] DATABASE\n\n"
53 "Options:\n"
54 " -h, --host=HOST host to connect to (required)\n"
55 " -p, --port=PORT port to connect to (required)\n"
56 " -m, --master=DB replicate database DB from the master (default: DATABASE)\n"
57 " -i, --interval=N wait N seconds between each connection to the master\n"
58 " (default: " STRINGIZE(DEFAULT_INTERVAL) ")\n"
59 " -r, --reader-time=N wait N seconds to allow readers time to close before\n"
60 " applying repeated changesets (default: " STRINGIZE(READER_CLOSE_TIME) ")\n"
61 " -t, --timeout=N set socket timeouts (if supported) to N seconds; N=0 for\n"
62 " no timeout (default: " STRINGIZE(DEFAULT_TIMEOUT) ")\n"
63 " -f, --force-copy force a full copy of the database to be sent (and then\n"
64 " replicate as normal)\n"
65 " -o, --one-shot replicate only once and then exit\n"
66 " -q, --quiet only report errors\n"
67 " -v, --verbose be more verbose\n"
68 " --help display this help and exit\n"
69 " --version output version information and exit" << endl;
70 }
71 
72 int
73 main(int argc, char **argv)
74 {
75  const char * opts = "h:p:m:i:r:t:ofqv";
76  static const struct option long_opts[] = {
77  {"host", required_argument, 0, 'h'},
78  {"port", required_argument, 0, 'p'},
79  {"master", required_argument, 0, 'm'},
80  {"interval", required_argument, 0, 'i'},
81  {"reader-time", required_argument, 0, 'r'},
82  {"timeout", required_argument, 0, 't'},
83  {"one-shot", no_argument, 0, 'o'},
84  {"force-copy", no_argument, 0, 'f'},
85  {"quiet", no_argument, 0, 'q'},
86  {"verbose", no_argument, 0, 'v'},
87  {"help", no_argument, 0, OPT_HELP},
88  {"version", no_argument, 0, OPT_VERSION},
89  {NULL, 0, 0, 0}
90  };
91 
92  string host;
93  int port = 0;
94  string masterdb;
95  int interval = DEFAULT_INTERVAL;
96  bool one_shot = false;
97  enum { NORMAL, VERBOSE, QUIET } verbosity = NORMAL;
98  bool force_copy = false;
99  int reader_close_time = READER_CLOSE_TIME;
100  int timeout = DEFAULT_TIMEOUT;
101 
102  int c;
103  while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
104  switch (c) {
105  case 'h':
106  host.assign(optarg);
107  break;
108  case 'p':
109  port = atoi(optarg);
110  break;
111  case 'm':
112  masterdb.assign(optarg);
113  break;
114  case 'i':
115  interval = atoi(optarg);
116  break;
117  case 'r':
118  reader_close_time = atoi(optarg);
119  break;
120  case 't':
121  timeout = atoi(optarg);
122  break;
123  case 'f':
124  force_copy = true;
125  break;
126  case 'o':
127  one_shot = true;
128  break;
129  case 'q':
130  verbosity = QUIET;
131  break;
132  case 'v':
133  verbosity = VERBOSE;
134  break;
135  case OPT_HELP:
136  cout << PROG_NAME " - " PROG_DESC "\n\n";
137  show_usage();
138  exit(0);
139  case OPT_VERSION:
140  cout << PROG_NAME " - " PACKAGE_STRING << endl;
141  exit(0);
142  default:
143  show_usage();
144  exit(1);
145  }
146  }
147 
148  if (argc - optind != 1) {
149  show_usage();
150  exit(1);
151  }
152 
153  if (host.empty()) {
154  cout << "Host required - specify with --host=HOST\n\n";
155  show_usage();
156  exit(1);
157  }
158 
159  if (port == 0) {
160  cout << "Port required - specify with --port=PORT\n\n";
161  show_usage();
162  exit(1);
163  }
164 
165  // Path to the database to create/update.
166  string dbpath(argv[optind]);
167 
168  if (masterdb.empty())
169  masterdb = dbpath;
170 
171  while (true) {
172  try {
173  if (verbosity == VERBOSE) {
174  cout << "Connecting to " << host << ":" << port << endl;
175  }
176  ReplicateTcpClient client(host, port, 10.0, timeout);
177  if (verbosity == VERBOSE) {
178  cout << "Getting update for " << dbpath << " from "
179  << masterdb << endl;
180  }
182  client.update_from_master(dbpath, masterdb, info,
183  reader_close_time, force_copy);
184  if (verbosity == VERBOSE) {
185  cout << "Update complete: "
186  << info.fullcopy_count << " copies, "
187  << info.changeset_count << " changesets, "
188  << (info.changed ? "new live database"
189  : "no changes to live database")
190  << endl;
191  }
192  if (verbosity != QUIET) {
193  if (info.fullcopy_count > 0 && !info.changed) {
194  cout <<
195 "Replication using a full copy failed. This usually means that the master\n"
196 "database is changing too frequently. Ensure that sufficient changesets are\n"
197 "present by setting XAPIAN_MAX_CHANGESETS on the master." << endl;
198  }
199  }
200  force_copy = false;
201  } catch (const Xapian::NetworkError &error) {
202  // Don't stop running if there's a network error - just log to
203  // stderr and retry at next timeout. This should make the client
204  // robust against temporary network failures.
205  cerr << argv[0] << ": " << error.get_description() << endl;
206 
207  // If we were running as a one-shot client though, we're going to
208  // exit anyway, so let's make the return value reflect that there
209  // was a failure.
210  if (one_shot)
211  exit(1);
212  } catch (const Xapian::Error &error) {
213  cerr << argv[0] << ": " << error.get_description() << endl;
214  exit(1);
215  } catch (const exception &e) {
216  cerr << "Caught standard exception: " << e.what() << endl;
217  exit(1);
218  } catch (...) {
219  cerr << "Caught unknown exception" << endl;
220  exit(1);
221  }
222  if (one_shot) break;
223  sleep(interval);
224  }
225 }
Wrappers to allow GNU getopt to be used cleanly from C++ code.
int optind
Definition: getopt.cc:94
#define PROG_NAME
unsigned timeout
A timeout value in milliseconds.
Definition: types.h:100
int gnu_getopt_long(int argc_, char *const *argv_, const char *shortopts_, const struct option *longopts_, int *optind_)
Definition: gnu_getopt.h:97
int fullcopy_count
Number of times a full database copy was performed.
Definition: replication.h:38
int main(int argc, char **argv)
#define OPT_HELP
#define DEFAULT_INTERVAL
static const char * opts
void sleep(double t)
Sleep until the time represented by this object.
Definition: realtime.h:127
#define STRINGIZE(X)
The STRINGIZE macro converts its parameter into a string constant.
Definition: stringutils.h:36
STL namespace.
TCP/IP replication client class.
#define no_argument
Definition: gnu_getopt.h:79
#define DEFAULT_TIMEOUT
Public interfaces for the Xapian library.
char * optarg
Definition: getopt.cc:79
static void show_usage()
Information about the steps involved in performing a replication.
Definition: replication.h:33
int changeset_count
Number of changesets applied.
Definition: replication.h:35
#define required_argument
Definition: gnu_getopt.h:80
bool changed
True if and only if the replication corresponds to a change in the live version of the database...
Definition: replication.h:45
std::string get_description() const
Return a string describing this object.
Definition: error.cc:93
#define OPT_VERSION
#define READER_CLOSE_TIME
All exceptions thrown by Xapian are subclasses of Xapian::Error.
Definition: error.h:43
Indicates a problem communicating with a remote database.
Definition: error.h:803
Various handy helpers which std::string really should provide.
#define PACKAGE_STRING
Definition: config.h:337
<unistd.h>, but with compat.
#define PROG_DESC