xapian-core  1.4.21
glass_changes.cc
Go to the documentation of this file.
1 
4 /* Copyright 2014,2016,2020 Olly Betts
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License as
8  * published by the Free Software Foundation; either version 2 of the
9  * License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
19  * USA
20  */
21 
22 #include <config.h>
23 
24 #include "glass_changes.h"
25 
27 #include "fd.h"
28 #include "io_utils.h"
29 #include "pack.h"
30 #include "posixy_wrapper.h"
31 #include "str.h"
32 #include "stringutils.h"
33 #include "wordaccess.h"
34 #include "xapian/constants.h"
35 #include "xapian/error.h"
36 
37 #include <cerrno>
38 #include <cstdlib>
39 #include <string>
40 
41 using namespace std;
42 
44 {
45  if (changes_fd >= 0) {
46  ::close(changes_fd);
47  string changes_tmp = changes_stem;
48  changes_tmp += "tmp";
49  io_unlink(changes_tmp);
50  }
51 }
52 
56  int flags)
57 {
58  if (rev == 0) {
59  // Don't generate a changeset for the first revision.
60  return NULL;
61  }
62 
63  // Always check max_changesets for modification since last revision.
64  const char *p = getenv("XAPIAN_MAX_CHANGESETS");
65  if (p) {
66  max_changesets = atoi(p);
67  } else {
68  max_changesets = 0;
69  }
70 
71  if (max_changesets == 0)
72  return NULL;
73 
74  string changes_tmp = changes_stem;
75  changes_tmp += "tmp";
76  changes_fd = posixy_open(changes_tmp.c_str(),
77  O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0666);
78  if (changes_fd < 0) {
79  string message = "Couldn't open changeset ";
80  message += changes_tmp;
81  message += " to write";
82  throw Xapian::DatabaseError(message, errno);
83  }
84 
85  // Write header for changeset file.
86  string header = CHANGES_MAGIC_STRING;
87  header += char(CHANGES_VERSION);
88  pack_uint(header, old_rev);
89  pack_uint(header, rev);
90 
91  if (flags & Xapian::DB_DANGEROUS) {
92  header += '\x01'; // Changes can't be applied to a live database.
93  } else {
94  header += '\x00'; // Changes can be applied to a live database.
95  }
96 
97  io_write(changes_fd, header.data(), header.size());
98  // FIXME: save the block stream as a single zlib stream...
99 
100  // bool compressed = CHANGES_VERSION != 1; FIXME: always true for glass, but make optional?
101  return this;
102 }
103 
104 void
105 GlassChanges::write_block(const char * p, size_t len)
106 {
107  io_write(changes_fd, p, len);
108 }
109 
110 void
112 {
113  if (changes_fd < 0)
114  return;
115 
116  io_write(changes_fd, "\xff", 1);
117 
118  string changes_tmp = changes_stem;
119  changes_tmp += "tmp";
120 
121  if (!(flags & Xapian::DB_NO_SYNC) && !io_sync(changes_fd)) {
122  int saved_errno = errno;
123  (void)::close(changes_fd);
124  changes_fd = -1;
125  (void)unlink(changes_tmp.c_str());
126  string m = changes_tmp;
127  m += ": Failed to sync";
128  throw Xapian::DatabaseError(m, saved_errno);
129  }
130 
131  (void)::close(changes_fd);
132  changes_fd = -1;
133 
134  string changes_file = changes_stem;
135  changes_file += str(new_rev - 1); // FIXME: ?
136 
137  if (!io_tmp_rename(changes_tmp, changes_file)) {
138  string m = changes_tmp;
139  m += ": Failed to rename to ";
140  m += changes_file;
141  throw Xapian::DatabaseError(m, errno);
142  }
143 
144  if (new_rev <= max_changesets) {
145  // We can't yet have max_changesets old changesets.
146  return;
147  }
148 
149  // Only remove old changesets if we successfully wrote a new changeset.
150  // Start at the oldest changeset we know about, and stop at max_changesets
151  // before new_rev. If max_changesets is unchanged from the previous
152  // commit and nothing went wrong, exactly one changeset file should be
153  // deleted.
154  glass_revision_number_t stop_changeset = new_rev - max_changesets;
155  while (oldest_changeset < stop_changeset) {
156  changes_file.resize(changes_stem.size());
157  changes_file += str(oldest_changeset);
158  (void)io_unlink(changes_file);
159  ++oldest_changeset;
160  }
161 }
162 
163 void
164 GlassChanges::check(const string & changes_file)
165 {
166  FD fd(posixy_open(changes_file.c_str(), O_RDONLY | O_CLOEXEC, 0666));
167  if (fd < 0) {
168  string message = "Couldn't open changeset ";
169  message += changes_file;
170  throw Xapian::DatabaseError(message, errno);
171  }
172 
173  char buf[10240];
174 
175  size_t n = io_read(fd, buf, sizeof(buf), CONST_STRLEN(CHANGES_MAGIC_STRING) + 4);
176  if (memcmp(buf, CHANGES_MAGIC_STRING,
178  throw Xapian::DatabaseError("Changes file has wrong magic");
179  }
180 
181  const char * p = buf + CONST_STRLEN(CHANGES_MAGIC_STRING);
182  if (*p++ != CHANGES_VERSION) {
183  throw Xapian::DatabaseError("Changes file has unknown version");
184  }
185  const char * end = buf + n;
186 
187  glass_revision_number_t old_rev, rev;
188  if (!unpack_uint(&p, end, &old_rev))
189  throw Xapian::DatabaseError("Changes file has bad old_rev");
190  if (!unpack_uint(&p, end, &rev))
191  throw Xapian::DatabaseError("Changes file has bad rev");
192  if (rev <= old_rev)
193  throw Xapian::DatabaseError("Changes file has rev <= old_rev");
194  if (p == end || (*p != 0 && *p != 1))
195  throw Xapian::DatabaseError("Changes file has bad dangerous flag");
196  ++p;
197 
198  while (true) {
199  n -= (p - buf);
200  memmove(buf, p, n);
201  n += io_read(fd, buf + n, sizeof(buf) - n);
202 
203  if (n == 0)
204  throw Xapian::DatabaseError("Changes file truncated");
205 
206  p = buf;
207  end = buf + n;
208 
209  unsigned char v = *p++;
210  if (v == 0xff) {
211  if (p != end)
212  throw Xapian::DatabaseError("Changes file - junk at end");
213  break;
214  }
215  if (v == 0xfe) {
216  // Version file.
217  glass_revision_number_t version_rev;
218  if (!unpack_uint(&p, end, &version_rev))
219  throw Xapian::DatabaseError("Changes file - bad version file revision");
220  if (rev != version_rev)
221  throw Xapian::DatabaseError("Version file revision != changes file new revision");
222  size_t len;
223  if (!unpack_uint(&p, end, &len))
224  throw Xapian::DatabaseError("Changes file - bad version file length");
225  if (len <= size_t(end - p)) {
226  p += len;
227  } else {
228  if (lseek(fd, len - (end - p), SEEK_CUR) < 0)
229  throw Xapian::DatabaseError("Changes file - version file data truncated");
230  p = end = buf;
231  n = 0;
232  }
233  continue;
234  }
235  unsigned table = (v & 0x7);
236  v >>= 3;
237  if (table > 5)
238  throw Xapian::DatabaseError("Changes file - bad table code");
239  // Changed block.
240  if (v > 5)
241  throw Xapian::DatabaseError("Changes file - bad block size");
242  unsigned block_size = 2048 << v;
243  uint4 block_number;
244  if (!unpack_uint(&p, end, &block_number))
245  throw Xapian::DatabaseError("Changes file - bad block number");
246 
247  // Parse information from the start of the block.
248  //
249  // Although the revision number is aligned within the block, the block
250  // data may not be aligned to a word boundary here.
251  uint4 block_rev = unaligned_read4(reinterpret_cast<const uint8_t*>(p));
252  (void)block_rev; // FIXME: Sanity check value.
253  unsigned level = static_cast<unsigned char>(p[4]);
254  (void)level; // FIXME: Sanity check value.
255 
256  // Skip over the block content.
257  if (block_size <= unsigned(end - p)) {
258  p += block_size;
259  } else {
260  if (lseek(fd, block_size - (end - p), SEEK_CUR) < 0)
261  throw Xapian::DatabaseError("Changes file - block data truncated");
262  p = end = buf;
263  n = 0;
264  }
265  }
266 }
int close(FD &fd)
Definition: fd.h:63
void io_write(int fd, const char *p, size_t n)
Write n bytes from block pointed to by p to file descriptor fd.
Definition: io_utils.cc:145
XAPIAN_REVISION_TYPE rev
Revision number of a database.
Definition: types.h:133
bool io_unlink(const std::string &filename)
Delete a file.
Definition: io_utils.cc:52
Internal definitions for glass database replication.
uint4 glass_revision_number_t
The revision number of a glass database.
Definition: glass_defs.h:61
bool io_sync(int fd)
Ensure all data previously written to file descriptor fd has been written to disk.
Definition: io_utils.h:73
Provides wrappers with POSIXy semantics.
Constants in the Xapian namespace.
STL namespace.
Convert types to std::string.
static void check(const std::string &changes_file)
#define O_CLOEXEC
Definition: safefcntl.h:90
void commit(glass_revision_number_t new_rev, int flags)
#define CHANGES_MAGIC_STRING
Hierarchy of classes which Xapian can throw as exceptions.
uint32_t uint4
Definition: internaltypes.h:32
functions for reading and writing different width words
Definition: fd.h:30
Glass changesets.
string str(int value)
Convert int to std::string.
Definition: str.cc:90
Wrapper class around a file descriptor to avoid leaks.
bool io_tmp_rename(const std::string &tmp_file, const std::string &real_file)
Rename a temporary file to its final position.
Definition: io_utils.cc:271
#define CONST_STRLEN(S)
Returns the length of a string constant.
Definition: stringutils.h:43
size_t io_read(int fd, char *p, size_t n, size_t min)
Read n bytes (or until EOF) into block pointed to by p from file descriptor fd.
Definition: io_utils.cc:123
GlassChanges * start(glass_revision_number_t old_rev, glass_revision_number_t rev, int flags)
const int DB_DANGEROUS
Update the database in-place.
Definition: constants.h:103
const int DB_NO_SYNC
Don&#39;t attempt to ensure changes have hit disk.
Definition: constants.h:66
void pack_uint(std::string &s, U value)
Append an encoded unsigned integer to a string.
Definition: pack.h:382
Pack types into strings and unpack them again.
Wrappers for low-level POSIX I/O routines.
void write_block(const char *p, size_t len)
Various handy helpers which std::string really should provide.
bool unpack_uint(const char **p, const char *end, U *result)
Decode an unsigned integer from a string.
Definition: pack.h:413
#define posixy_open
#define CHANGES_VERSION
DatabaseError indicates some sort of database related error.
Definition: error.h:367
uint32_t unaligned_read4(const unsigned char *ptr)
Definition: wordaccess.h:151