Xapian: Internal Source Documentation: xapian-core: tests/api

00001 
00004 /* Copyright (C) 2009,2010,2011 Olly Betts
00005  * Copyright (C) 2010 Richard Boulton
00006  *
00007  * This program is free software; you can redistribute it and/or
00008  * modify it under the terms of the GNU General Public License as
00009  * published by the Free Software Foundation; either version 2 of the
00010  * License, or (at your option) any later version.
00011  *
00012  * This program is distributed in the hope that it will be useful,
00013  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00014  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00015  * GNU General Public License for more details.
00016  *
00017  * You should have received a copy of the GNU General Public License
00018  * along with this program; if not, write to the Free Software
00019  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
00020  * USA
00021  */
00022 
00023 #include <config.h>
00024 
00025 #include "api_compact.h"
00026 
00027 #include "apitest.h"
00028 #include "dbcheck.h"
00029 #include "testsuite.h"
00030 #include "testutils.h"
00031 
00032 #include <xapian.h>
00033 
00034 #include <cstdlib>
00035 #include <fstream>
00036 
00037 #include "str.h"
00038 #include "utils.h"
00039 #include "unixcmds.h"
00040 
00041 using namespace std;
00042 
00043 static void
00044 make_sparse_db(Xapian::WritableDatabase &db, const string & s)
00045 {
00046     // Need non-const pointer for strtoul(), but data isn't modified.
00047     char * p = const_cast<char *>(s.c_str());
00048 
00049     while (*p) {
00050         bool del = (*p == '!');
00051         if (del) ++p;
00052         Xapian::docid first = strtoul(p, &p, 10);
00053         Xapian::docid last = first;
00054         if (*p == '-') {
00055             last = strtoul(p + 1, &p, 10);
00056         }
00057         if (*p && *p != ' ') {
00058             tout << p - s.c_str() << endl;
00059             FAIL_TEST("Bad sparse db spec (expected space): " << s);
00060         }
00061         if (first > last) {
00062             FAIL_TEST("Bad sparse db spec (first > last): " << s);
00063         }
00064 
00065         do {
00066             if (del) {
00067                 db.delete_document(first);
00068             } else {
00069                 Xapian::Document doc;
00070                 string id = str(first);
00071                 doc.set_data(id);
00072                 doc.add_term("Q" + str(first));
00073                 doc.add_term(string(first % 7 + 1, char((first % 26) + 'a')));
00074                 db.replace_document(first, doc);
00075             }
00076         } while (first++ < last);
00077 
00078         if (*p == '\0') break;
00079         ++p;
00080     }
00081 
00082     db.commit();
00083 }
00084 
00085 static void
00086 check_sparse_uid_terms(const string & path)
00087 {
00088     Xapian::Database db(path);
00089     Xapian::TermIterator t;
00090     for (t = db.allterms_begin("Q"); t != db.allterms_end("Q"); ++t) {
00091         Xapian::docid did = atoi((*t).c_str() + 1);
00092         Xapian::PostingIterator p = db.postlist_begin(*t);
00093         TEST_EQUAL(*p, did);
00094     }
00095 }
00096 
00097 DEFINE_TESTCASE(compactnorenumber1, generated) {
00098     string a = get_database_path("compactnorenumber1a", make_sparse_db,
00099                                  "5-7 24 76 987 1023-1027 9999 !9999");
00100     string a_uuid;
00101     {
00102         Xapian::Database db(a);
00103         a_uuid = db.get_uuid();
00104     }
00105     string b = get_database_path("compactnorenumber1b", make_sparse_db,
00106                                  "1027-1030");
00107     string c = get_database_path("compactnorenumber1c", make_sparse_db,
00108                                  "1028-1040");
00109     string d = get_database_path("compactnorenumber1d", make_sparse_db,
00110                                  "3000 999999 !999999");
00111 
00112     string out = get_named_writable_database_path("compactnorenumber1out");
00113 
00114     rm_rf(out);
00115     {
00116         Xapian::Compactor compact;
00117         compact.set_renumber(false);
00118         compact.set_destdir(out);
00119         compact.add_source(a);
00120         compact.compact();
00121     }
00122 
00123     check_sparse_uid_terms(out);
00124 
00125     {
00126         if (get_dbtype() == "flint") {
00127             // Flint creates its uuid file if it doesn't already exist, so we
00128             // need a white box test to ensure xapian-compact created it.
00129             TEST(file_exists(out + "/uuid"));
00130         }
00131         TEST(!dir_exists(out + "/donor"));
00132         Xapian::Database db(out);
00133         // xapian-compact should change the UUID of the database, but didn't
00134         // prior to 1.0.18/1.1.4.
00135         string out_uuid = db.get_uuid();
00136         TEST_NOT_EQUAL(a_uuid, out_uuid);
00137         TEST_EQUAL(out_uuid.size(), 36);
00138         TEST_NOT_EQUAL(out_uuid, "00000000-0000-0000-0000-000000000000");
00139 
00140         // White box test - ensure that the donor database is removed.
00141         TEST(!dir_exists(out + "/donor"));
00142     }
00143 
00144     rm_rf(out);
00145     {
00146         Xapian::Compactor compact;
00147         compact.set_renumber(false);
00148         compact.set_destdir(out);
00149         compact.add_source(a);
00150         compact.add_source(c);
00151         compact.compact();
00152     }
00153     check_sparse_uid_terms(out);
00154     {
00155         // Check that xapian-compact is producing a consistent database.  Also,
00156         // regression test - xapian 1.1.4 set lastdocid to 0 in the output
00157         // database.
00158         Xapian::Database outdb(out);
00159         dbcheck(outdb, 24, 9999);
00160     }
00161 
00162     rm_rf(out);
00163     {
00164         Xapian::Compactor compact;
00165         compact.set_renumber(false);
00166         compact.set_destdir(out);
00167         compact.add_source(d);
00168         compact.add_source(a);
00169         compact.add_source(c);
00170         compact.compact();
00171     }
00172     check_sparse_uid_terms(out);
00173 
00174     rm_rf(out);
00175     {
00176         Xapian::Compactor compact;
00177         compact.set_renumber(false);
00178         compact.set_destdir(out);
00179         compact.add_source(c);
00180         compact.add_source(a);
00181         compact.add_source(d);
00182         compact.compact();
00183     }
00184     check_sparse_uid_terms(out);
00185 
00186     // Should fail.
00187     rm_rf(out);
00188     {
00189         Xapian::Compactor compact;
00190         compact.set_renumber(false);
00191         compact.set_destdir(out);
00192         compact.add_source(a);
00193         compact.add_source(b);
00194         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
00195     }
00196  
00197     // Should fail.
00198     rm_rf(out);
00199     {
00200         Xapian::Compactor compact;
00201         compact.set_renumber(false);
00202         compact.set_destdir(out);
00203         compact.add_source(b);
00204         compact.add_source(a);
00205         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
00206     }
00207 
00208     // Should fail.
00209     rm_rf(out);
00210     {
00211         Xapian::Compactor compact;
00212         compact.set_renumber(false);
00213         compact.set_destdir(out);
00214         compact.add_source(a);
00215         compact.add_source(b);
00216         compact.add_source(d);
00217         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
00218     }
00219  
00220     // Should fail.
00221     rm_rf(out);
00222     {
00223         Xapian::Compactor compact;
00224         compact.set_renumber(false);
00225         compact.set_destdir(out);
00226         compact.add_source(d);
00227         compact.add_source(b);
00228         compact.add_source(a);
00229         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
00230     }
00231 
00232     // Should fail.
00233     rm_rf(out);
00234     {
00235         Xapian::Compactor compact;
00236         compact.set_renumber(false);
00237         compact.set_destdir(out);
00238         compact.add_source(b);
00239         compact.add_source(a);
00240         compact.add_source(d);
00241         TEST_EXCEPTION(Xapian::InvalidOperationError, compact.compact());
00242     }
00243 
00244     return true;
00245 }
00246 
00247 // Test use of compact to merge two databases.
00248 DEFINE_TESTCASE(compactmerge1, brass || chert || flint) {
00249     string indbpath = get_database_path("apitest_simpledata");
00250     string outdbpath = get_named_writable_database_path("compactmerge1out");
00251     rm_rf(outdbpath);
00252 
00253     Xapian::Compactor compact;
00254     compact.set_destdir(outdbpath);
00255     compact.add_source(indbpath);
00256     compact.add_source(indbpath);
00257     compact.compact();
00258 
00259     Xapian::Database indb(get_database("apitest_simpledata"));
00260     Xapian::Database outdb(outdbpath);
00261 
00262     TEST_EQUAL(indb.get_doccount() * 2, outdb.get_doccount());
00263     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
00264 
00265     return true;
00266 }
00267 
00268 static void
00269 make_multichunk_db(Xapian::WritableDatabase &db, const string &)
00270 {
00271     int count = 10000;
00272 
00273     Xapian::Document doc;
00274     doc.add_term("a");
00275     while (count) {
00276         db.add_document(doc);
00277         --count;
00278     }
00279 
00280     db.commit();
00281 }
00282 
00283 // Test use of compact on a database which has multiple chunks for a term.
00284 // This is a regression test for ticket #427
00285 DEFINE_TESTCASE(compactmultichunks1, generated) {
00286     string indbpath = get_database_path("compactmultichunks1in",
00287                                         make_multichunk_db, "");
00288     string outdbpath = get_named_writable_database_path("compactmultichunks1out");
00289     rm_rf(outdbpath);
00290 
00291     Xapian::Compactor compact;
00292     compact.set_destdir(outdbpath);
00293     compact.add_source(indbpath);
00294     compact.compact();
00295 
00296     Xapian::Database indb(indbpath);
00297     Xapian::Database outdb(outdbpath);
00298 
00299     TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
00300     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
00301 
00302     return true;
00303 }
00304 
00305 // Test compacting from a stub database directory.
00306 DEFINE_TESTCASE(compactstub1, brass || chert || flint) {
00307     const char * stubpath = ".stub/compactstub1";
00308     const char * stubpathfile = ".stub/compactstub1/XAPIANDB";
00309     mkdir(".stub", 0755);
00310     mkdir(stubpath, 0755);
00311     ofstream stub(stubpathfile);
00312     TEST(stub.is_open());
00313     stub << "auto ../../" << get_database_path("apitest_simpledata") << endl;
00314     stub << "auto ../../" << get_database_path("apitest_simpledata2") << endl;
00315     stub.close();
00316 
00317     string outdbpath = get_named_writable_database_path("compactstub1out");
00318     rm_rf(outdbpath);
00319 
00320     Xapian::Compactor compact;
00321     compact.set_destdir(outdbpath);
00322     compact.add_source(stubpath);
00323     compact.compact();
00324 
00325     Xapian::Database indb(stubpath);
00326     Xapian::Database outdb(outdbpath);
00327 
00328     TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
00329     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
00330 
00331     return true;
00332 }
00333 
00334 // Test compacting from a stub database file.
00335 DEFINE_TESTCASE(compactstub2, brass || chert || flint) {
00336     const char * stubpath = ".stub/compactstub2";
00337     mkdir(".stub", 0755);
00338     ofstream stub(stubpath);
00339     TEST(stub.is_open());
00340     stub << "auto ../" << get_database_path("apitest_simpledata") << endl;
00341     stub << "auto ../" << get_database_path("apitest_simpledata2") << endl;
00342     stub.close();
00343 
00344     string outdbpath = get_named_writable_database_path("compactstub2out");
00345     rm_rf(outdbpath);
00346 
00347     Xapian::Compactor compact;
00348     compact.set_destdir(outdbpath);
00349     compact.add_source(stubpath);
00350     compact.compact();
00351 
00352     Xapian::Database indb(stubpath);
00353     Xapian::Database outdb(outdbpath);
00354 
00355     TEST_EQUAL(indb.get_doccount(), outdb.get_doccount());
00356     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
00357 
00358     return true;
00359 }
00360 
00361 // Test compacting a stub database file to itself.
00362 DEFINE_TESTCASE(compactstub3, brass || chert || flint) {
00363     const char * stubpath = ".stub/compactstub3";
00364     mkdir(".stub", 0755);
00365     ofstream stub(stubpath);
00366     TEST(stub.is_open());
00367     stub << "auto ../" << get_database_path("apitest_simpledata") << endl;
00368     stub << "auto ../" << get_database_path("apitest_simpledata2") << endl;
00369     stub.close();
00370 
00371     Xapian::doccount in_docs;
00372     {
00373         Xapian::Database indb(stubpath);
00374         in_docs = indb.get_doccount();
00375     }
00376 
00377     Xapian::Compactor compact;
00378     compact.set_destdir(stubpath);
00379     compact.add_source(stubpath);
00380     compact.compact();
00381 
00382     Xapian::Database outdb(stubpath);
00383 
00384     TEST_EQUAL(in_docs, outdb.get_doccount());
00385     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
00386 
00387     return true;
00388 }
00389 
00390 // Test compacting a stub database directory to itself.
00391 DEFINE_TESTCASE(compactstub4, brass || chert || flint) {
00392     const char * stubpath = ".stub/compactstub4";
00393     const char * stubpathfile = ".stub/compactstub4/XAPIANDB";
00394     mkdir(".stub", 0755);
00395     mkdir(stubpath, 0755);
00396     ofstream stub(stubpathfile);
00397     TEST(stub.is_open());
00398     stub << "auto ../../" << get_database_path("apitest_simpledata") << endl;
00399     stub << "auto ../../" << get_database_path("apitest_simpledata2") << endl;
00400     stub.close();
00401 
00402     Xapian::doccount in_docs;
00403     {
00404         Xapian::Database indb(stubpath);
00405         in_docs = indb.get_doccount();
00406     }
00407 
00408     Xapian::Compactor compact;
00409     compact.set_destdir(stubpath);
00410     compact.add_source(stubpath);
00411     compact.compact();
00412 
00413     Xapian::Database outdb(stubpath);
00414 
00415     TEST_EQUAL(in_docs, outdb.get_doccount());
00416     dbcheck(outdb, outdb.get_doccount(), outdb.get_doccount());
00417 
00418     return true;
00419 }
00420 
00421 static void
00422 make_all_tables(Xapian::WritableDatabase &db, const string &)
00423 {
00424     Xapian::Document doc;
00425     doc.add_term("foo");
00426     db.add_document(doc);
00427     db.add_spelling("foo");
00428     db.add_synonym("foobar", "foo");
00429 
00430     db.commit();
00431 }
00432 
00433 static void
00434 make_missing_tables(Xapian::WritableDatabase &db, const string &)
00435 {
00436     Xapian::Document doc;
00437     doc.add_term("foo");
00438     db.add_document(doc);
00439 
00440     db.commit();
00441 }
00442 
00443 DEFINE_TESTCASE(compactmissingtables1, generated) {
00444     string a = get_database_path("compactmissingtables1a",
00445                                  make_all_tables);
00446     string b = get_database_path("compactmissingtables1b",
00447                                  make_missing_tables);
00448 
00449     string out = get_named_writable_database_path("compactmissingtables1out");
00450     rm_rf(out);
00451 
00452     Xapian::Compactor compact;
00453     compact.set_destdir(out);
00454     compact.add_source(a);
00455     compact.add_source(b);
00456     compact.compact();
00457 
00458     {
00459         Xapian::Database db(out);
00460         TEST_NOT_EQUAL(db.spellings_begin(), db.spellings_end());
00461         TEST_NOT_EQUAL(db.synonym_keys_begin(), db.synonym_keys_end());
00462         // FIXME: arrange for input b to not have a termlist table.
00463 //      TEST_EXCEPTION(Xapian::FeatureUnavailableError, db.termlist_begin(1));
00464     }
00465 
00466     return true;
00467 }
00468
tests/api_compact.cc
