xapian-core  1.4.25
backendmanager_remotetcp.cc
Go to the documentation of this file.
1 
4 /* Copyright (C) 2006,2007,2008,2009,2013,2015 Olly Betts
5  * Copyright (C) 2008 Lemur Consulting Ltd
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License as
9  * published by the Free Software Foundation; either version 2 of the
10  * License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include <config.h>
23 
25 
26 #include <xapian.h>
27 
28 #include <stdio.h> // For fdopen().
29 #include <cerrno>
30 #include <cstring>
31 
32 #include "safesysexits.h"
33 
34 #ifdef HAVE_FORK
35 # include <signal.h>
36 # include <sys/types.h>
37 # include "safesyssocket.h"
38 # include <sys/wait.h>
39 # include <unistd.h>
40 // Some older systems had SIGCLD rather than SIGCHLD.
41 # if !defined SIGCHLD && defined SIGCLD
42 # define SIGCHLD SIGCLD
43 # endif
44 #endif
45 
46 #ifdef __WIN32__
47 # include <io.h> // For _open_osfhandle().
48 # include "safefcntl.h"
49 # include "safewindows.h"
50 # include <cstdlib> // For free().
51 #endif
52 
53 #include "errno_to_string.h"
54 #include "noreturn.h"
55 #include "str.h"
56 
57 #include <string>
58 #include <vector>
59 
60 #ifdef HAVE_VALGRIND
61 # include <valgrind/memcheck.h>
62 #endif
63 
64 using namespace std;
65 
66 // We've had problems on some hosts which run tinderbox tests with "localhost"
67 // not being set in /etc/hosts - using the IP address equivalent seems more
68 // reliable.
69 #define LOCALHOST "127.0.0.1"
70 
71 // Start at DEFAULT port and try higher ports until one isn't already in use.
72 #define DEFAULT_PORT 1239
73 
74 #ifdef HAVE_FORK
75 
76 // We can't dynamically allocate memory for this because it confuses the leak
77 // detector. We only have 1-3 child fds open at once anyway, so a fixed size
78 // array isn't a problem, and linear scanning isn't a problem either.
79 struct pid_fd {
80  pid_t pid;
81  int fd;
82 };
83 
84 static pid_fd pid_to_fd[16];
85 
86 extern "C" {
87 
88 static void
89 on_SIGCHLD(int /*sig*/)
90 {
91  int status;
92  pid_t child;
93  while ((child = waitpid(-1, &status, WNOHANG)) > 0) {
94  for (unsigned i = 0; i < sizeof(pid_to_fd) / sizeof(pid_fd); ++i) {
95  if (pid_to_fd[i].pid == child) {
96  int fd = pid_to_fd[i].fd;
97  pid_to_fd[i].fd = 0;
98  pid_to_fd[i].pid = 0;
99  // NB close() *is* safe to use in a signal handler.
100  close(fd);
101  break;
102  }
103  }
104  }
105 }
106 
107 }
108 
109 static int
110 launch_xapian_tcpsrv(const string & args)
111 {
112  int port = DEFAULT_PORT;
113 
114  // We want to be able to get the exit status of the child process we fork
115  // if xapian-tcpsrv doesn't start listening successfully.
116  signal(SIGCHLD, SIG_DFL);
117 try_next_port:
118  string cmd = XAPIAN_TCPSRV " --one-shot --interface " LOCALHOST " --port ";
119  cmd += str(port);
120  cmd += " ";
121  cmd += args;
122 #ifdef HAVE_VALGRIND
123  if (RUNNING_ON_VALGRIND) cmd = "./runsrv " + cmd;
124 #endif
125  int fds[2];
126  if (socketpair(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, PF_UNSPEC, fds) < 0) {
127  string msg("Couldn't create socketpair: ");
128  errno_to_string(errno, msg);
129  throw msg;
130  }
131 
132  pid_t child = fork();
133  if (child == 0) {
134  // Child process.
135  close(fds[0]);
136  // Connect stdout and stderr to the socket.
137  //
138  // Make sure the socket isn't fd 1 or 2. We need to ensure that
139  // FD_CLOEXEC isn't set for stdout or stderr (which creating them with
140  // dup2() achieves), and that we close fds[1]. The cleanest way to
141  // address this seems to be to turn the unusual situation into the
142  // usual one.
143  if (fds[1] == 1 || fds[1] == 2) {
144  dup2(fds[1], 3);
145  fds[1] = 3;
146  }
147  dup2(fds[1], 1);
148  dup2(fds[1], 2);
149  close(fds[1]);
150  execl("/bin/sh", "/bin/sh", "-c", cmd.c_str(), static_cast<void*>(0));
151  _exit(-1);
152  }
153 
154  close(fds[1]);
155  if (child == -1) {
156  // Couldn't fork.
157  int fork_errno = errno;
158  close(fds[0]);
159  string msg("Couldn't fork: ");
160  errno_to_string(fork_errno, msg);
161  throw msg;
162  }
163 
164  // Parent process.
165 
166  // Wrap the file descriptor in a FILE * so we can read lines using fgets().
167  FILE * fh = fdopen(fds[0], "r");
168  if (fh == NULL) {
169  string msg("Failed to run command '");
170  msg += cmd;
171  msg += "': ";
172  errno_to_string(errno, msg);
173  throw msg;
174  }
175 
176  string output;
177  while (true) {
178  char buf[256];
179  if (fgets(buf, sizeof(buf), fh) == NULL) {
180  fclose(fh);
181  // Wait for the child to exit.
182  int status;
183  if (waitpid(child, &status, 0) == -1) {
184  string msg("waitpid failed: ");
185  errno_to_string(errno, msg);
186  throw msg;
187  }
188  if (++port < 65536 && status != 0) {
189  if (WIFEXITED(status) &&
190  WEXITSTATUS(status) == EX_UNAVAILABLE) {
191  // Exit code EX_UNAVAILABLE from xapian-tcpsrv means the
192  // specified port was in use.
193  goto try_next_port;
194  }
195  }
196  string msg("Failed to get 'Listening...' from command '");
197  msg += cmd;
198  msg += "' (output: ";
199  msg += output;
200  msg += ")";
201  throw msg;
202  }
203  if (strcmp(buf, "Listening...\n") == 0) break;
204  output += buf;
205  }
206 
207  // dup() the fd we wrapped with fdopen() so we can keep it open so the
208  // xapian-tcpsrv keeps running.
209  int tracked_fd = dup(fds[0]);
210 
211  // We must fclose() the FILE* to avoid valgrind detecting memory leaks from
212  // its buffers.
213  fclose(fh);
214 
215  // Find a slot to track the pid->fd mapping in. If we can't find a slot
216  // it just means we'll leak the fd, so don't worry about that too much.
217  for (unsigned i = 0; i < sizeof(pid_to_fd) / sizeof(pid_fd); ++i) {
218  if (pid_to_fd[i].pid == 0) {
219  pid_to_fd[i].fd = tracked_fd;
220  pid_to_fd[i].pid = child;
221  break;
222  }
223  }
224 
225  // Set a signal handler to clean up the xapian-tcpsrv child process when it
226  // finally exits.
227  signal(SIGCHLD, on_SIGCHLD);
228 
229  return port;
230 }
231 
232 #elif defined __WIN32__
233 
234 static HANDLE tcpsrv_handles[16];
235 static unsigned tcpsrv_handles_index = 0;
236 
237 static constexpr auto TCPSRV_HANDLES_INDEX_MAX =
238  sizeof(tcpsrv_handles) / sizeof(tcpsrv_handles[0]);
239 
240 XAPIAN_NORETURN(static void win32_throw_error_string(const char * str));
241 static void win32_throw_error_string(const char * str)
242 {
243  string msg(str);
244  char * error = 0;
245  DWORD len;
246  len = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_ALLOCATE_BUFFER,
247  0, GetLastError(), 0, (CHAR*)&error, 0, 0);
248  if (error) {
249  // Remove any trailing \r\n from output of FormatMessage.
250  if (len >= 2 && error[len - 2] == '\r' && error[len - 1] == '\n')
251  len -= 2;
252  if (len) {
253  msg += ": ";
254  msg.append(error, len);
255  }
256  LocalFree(error);
257  }
258  throw msg;
259 }
260 
261 // This implementation uses the WIN32 API to start xapian-tcpsrv as a child
262 // process and read its output using a pipe.
263 static int
264 launch_xapian_tcpsrv(const string & args)
265 {
266  int port = DEFAULT_PORT;
267 
268 try_next_port:
269  string cmd = XAPIAN_TCPSRV " --one-shot --interface " LOCALHOST " --port ";
270  cmd += str(port);
271  cmd += " ";
272  cmd += args;
273 
274  // Create a pipe so we can read stdout/stderr from the child process.
275  HANDLE hRead, hWrite;
276  if (!CreatePipe(&hRead, &hWrite, 0, 0))
277  win32_throw_error_string("Couldn't create pipe");
278 
279  // Set the write handle to be inherited by the child process.
280  SetHandleInformation(hWrite, HANDLE_FLAG_INHERIT, 1);
281 
282  // Create the child process.
283  PROCESS_INFORMATION procinfo;
284  memset(&procinfo, 0, sizeof(PROCESS_INFORMATION));
285 
286  STARTUPINFO startupinfo;
287  memset(&startupinfo, 0, sizeof(STARTUPINFO));
288  startupinfo.cb = sizeof(STARTUPINFO);
289  startupinfo.hStdError = hWrite;
290  startupinfo.hStdOutput = hWrite;
291  startupinfo.hStdInput = INVALID_HANDLE_VALUE;
292  startupinfo.dwFlags |= STARTF_USESTDHANDLES;
293 
294  // For some reason Windows wants a modifiable command line string
295  // so pass a pointer to the first character rather than using c_str().
296  if (!CreateProcess(XAPIAN_TCPSRV, &cmd[0], 0, 0, TRUE, 0, 0, 0,
297  &startupinfo, &procinfo)) {
298  win32_throw_error_string("Couldn't create child process");
299  }
300 
301  CloseHandle(hWrite);
302  CloseHandle(procinfo.hThread);
303 
304  string output;
305  FILE *fh = fdopen(_open_osfhandle(intptr_t(hRead), O_RDONLY), "r");
306  while (true) {
307  char buf[256];
308  if (fgets(buf, sizeof(buf), fh) == NULL) {
309  fclose(fh);
310  DWORD rc;
311  // This doesn't seem to be necessary on the machine I tested on,
312  // but I guess it could be on a slow machine...
313  while (GetExitCodeProcess(procinfo.hProcess, &rc) && rc == STILL_ACTIVE) {
314  Sleep(100);
315  }
316  CloseHandle(procinfo.hProcess);
317  if (++port < 65536 && rc == EX_UNAVAILABLE) {
318  // Exit code EX_UNAVAILABLE from xapian-tcpsrv means the
319  // specified port was in use.
320  goto try_next_port;
321  }
322  string msg("Failed to get 'Listening...' from command '");
323  msg += cmd;
324  msg += "' (output: ";
325  msg += output;
326  msg += ")";
327  throw msg;
328  }
329  if (strcmp(buf, "Listening...\r\n") == 0) break;
330  output += buf;
331  }
332  fclose(fh);
333 
334  if (tcpsrv_handles_index < TCPSRV_HANDLES_INDEX_MAX) {
335  tcpsrv_handles[tcpsrv_handles_index++] = procinfo.hProcess;
336  }
337 
338  return port;
339 }
340 
341 #else
342 # error Neither HAVE_FORK nor __WIN32__ is defined
343 #endif
344 
347 }
348 
350 BackendManagerRemoteTcp::do_get_database(const vector<string> & files)
351 {
352  // Default to a long (5 minute) timeout so that tests won't fail just
353  // because the host is slow or busy.
354  return BackendManagerRemoteTcp::get_remote_database(files, 300000);
355 }
356 
359  const string & file)
360 {
361  string args = get_writable_database_args(name, file);
362  int port = launch_xapian_tcpsrv(args);
364 }
365 
367 BackendManagerRemoteTcp::get_remote_database(const vector<string> & files,
368  unsigned int timeout)
369 {
370  string args = get_remote_database_args(files, timeout);
371  int port = launch_xapian_tcpsrv(args);
372  return Xapian::Remote::open(LOCALHOST, port);
373 }
374 
377 {
378  string args = get_remote_database_args(path, 300000);
379  int port = launch_xapian_tcpsrv(args);
380  return Xapian::Remote::open(LOCALHOST, port);
381 }
382 
385 {
386  string args = get_writable_database_as_database_args();
387  int port = launch_xapian_tcpsrv(args);
388  return Xapian::Remote::open(LOCALHOST, port);
389 }
390 
393 {
394  string args = get_writable_database_again_args();
395  int port = launch_xapian_tcpsrv(args);
397 }
398 
399 void
401 {
402 #ifdef HAVE_FORK
403  signal(SIGCHLD, SIG_DFL);
404  for (unsigned i = 0; i < sizeof(pid_to_fd) / sizeof(pid_fd); ++i) {
405  pid_t child = pid_to_fd[i].pid;
406  if (child) {
407  int status;
408  while (waitpid(child, &status, 0) == -1 && errno == EINTR) { }
409  // Other possible error from waitpid is ECHILD, which it seems can
410  // only mean that the child has already exited and SIGCHLD was set
411  // to SIG_IGN. If we did somehow see that, the sanest response
412  // seems to be to close the fd and move on.
413  int fd = pid_to_fd[i].fd;
414  pid_to_fd[i].fd = 0;
415  pid_to_fd[i].pid = 0;
416  close(fd);
417  }
418  }
419 #elif defined __WIN32__
420  for (unsigned i = 0; i != tcpsrv_handles_index; ++i) {
421  WaitForSingleObject(tcpsrv_handles[i], INFINITE);
422  CloseHandle(tcpsrv_handles[i]);
423  }
424  tcpsrv_handles_index = 0;
425 #endif
426 }
int close(FD &fd)
Definition: fd.h:63
Xapian::WritableDatabase get_writable_database(const std::string &name, const std::string &file)
Create a RemoteTcp Xapian::WritableDatabase object indexing a single file.
Define the XAPIAN_NORETURN macro.
This class is used to access a database, or a group of databases.
Definition: database.h:68
Xapian::Database do_get_database(const std::vector< std::string > &files)
Create a Xapian::Database object indexing multiple files.
unsigned timeout
A timeout value in milliseconds.
Definition: types.h:100
include <sys/socket.h> with portability workarounds.
Convert errno value to std::string, thread-safe if possible.
#define XAPIAN_TCPSRV
STL namespace.
Convert types to std::string.
BackendManager subclass for remotetcp databases.
#define SOCK_CLOEXEC
Definition: safesyssocket.h:83
Xapian::Database get_writable_database_as_database()
Create a Database object for the last opened WritableDatabase.
This class provides read/write access to a database.
Definition: database.h:789
include <sysexits.h> with portability workarounds.
void errno_to_string(int e, string &s)
Database open(const std::string &host, unsigned int port, useconds_t timeout=10000, useconds_t connect_timeout=10000)
Construct a Database object for read-only access to a remote database accessed via a TCP connection...
Public interfaces for the Xapian library.
string str(int value)
Convert int to std::string.
Definition: str.cc:90
#define DEFAULT_PORT
Xapian::Database get_remote_database(const std::vector< std::string > &files, unsigned int timeout)
Create a RemoteTcp Xapian::Database with the specified timeout.
Xapian::Database get_database_by_path(const std::string &path)
Get a RemoteTcp Xapian::Database instance of the database at path.
include <windows.h> without all the bloat and damage.
Xapian::WritableDatabase get_writable_database_again()
Create a WritableDatabase object for the last opened WritableDatabase.
#define EX_UNAVAILABLE
Definition: safesysexits.h:33
#define LOCALHOST
Definition: header.h:151
WritableDatabase open_writable(const std::string &host, unsigned int port, useconds_t timeout=0, useconds_t connect_timeout=10000, int flags=0)
Construct a WritableDatabase object for update access to a remote database accessed via a TCP connect...
void clean_up()
Called after each test, to perform any necessary cleanup.
include <fcntl.h>, but working around broken platforms.