Xapian: Internal Source Documentation: xapian-core: tests/api

00001 /* api_valuestats.cc: tests of the value statistics functions.
00002  *
00003  * Copyright 2008 Lemur Consulting Ltd
00004  * Copyright 2008,2009 Olly Betts
00005  *
00006  * This program is free software; you can redistribute it and/or
00007  * modify it under the terms of the GNU General Public License as
00008  * published by the Free Software Foundation; either version 2 of the
00009  * License, or (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
00019  * USA
00020  */
00021 
00022 #include <config.h>
00023 
00024 #include "api_valuestats.h"
00025 
00026 #include <xapian.h>
00027 #include "testsuite.h"
00028 #include "testutils.h"
00029 
00030 #include "apitest.h"
00031 
00032 using namespace std;
00033 
00034 // #######################################################################
00035 // # Tests start here
00036 
00038 DEFINE_TESTCASE(valuestats1, writable && valuestats) {
00039     Xapian::WritableDatabase db_w = get_writable_database();
00040 
00041     // Check that counts are initially zero.
00042     TEST_EQUAL(db_w.get_value_freq(0), 0);
00043     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
00044     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
00045     TEST_EQUAL(db_w.get_value_freq(1), 0);
00046     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
00047     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
00048 
00049     Xapian::Document doc;
00050     doc.add_value(0, "hello");
00051 
00052     // Check that statistics for the correct value slot increase when document
00053     // is added.  (Check slot 1 first, so that cache invalidation of the last
00054     // slot read also gets checked.)
00055     db_w.add_document(doc);
00056     TEST_EQUAL(db_w.get_value_freq(1), 0);
00057     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
00058     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
00059     TEST_EQUAL(db_w.get_value_freq(0), 1);
00060     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
00061     TEST_EQUAL(db_w.get_value_upper_bound(0), "hello");
00062 
00063     // Check that statistics work correctly when second document is added.
00064     doc = Xapian::Document();
00065     doc.add_value(0, "world");
00066     doc.add_value(1, "cheese");
00067     db_w.replace_document(2, doc);
00068     TEST_EQUAL(db_w.get_value_freq(0), 2);
00069     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
00070     TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
00071     TEST_EQUAL(db_w.get_value_freq(1), 1);
00072     TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
00073     TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
00074 
00075     // Deleting a document affects the count, but not the bounds.
00076     db_w.delete_document(1);
00077     TEST_EQUAL(db_w.get_value_freq(1), 1);
00078     TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
00079     TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
00080     TEST_EQUAL(db_w.get_value_freq(0), 1);
00081     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
00082     TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
00083 
00084     // Deleting all the documents returns the bounds to their original value.
00085     db_w.delete_document(2);
00086     TEST_EQUAL(db_w.get_value_freq(0), 0);
00087     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
00088     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
00089     TEST_EQUAL(db_w.get_value_freq(1), 0);
00090     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
00091     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
00092 
00093     // Adding a document with a value in one of the old slots should still
00094     // end up with tight bounds on it.
00095     doc = Xapian::Document();
00096     doc.add_value(1, "newval");
00097     db_w.replace_document(2, doc);
00098     TEST_EQUAL(db_w.get_value_freq(1), 1);
00099     TEST_EQUAL(db_w.get_value_lower_bound(1), "newval");
00100     TEST_EQUAL(db_w.get_value_upper_bound(1), "newval");
00101     TEST_EQUAL(db_w.get_value_freq(0), 0);
00102     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
00103     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
00104 
00105     return true;
00106 }
00107 
00109 DEFINE_TESTCASE(valuestats2, transactions && valuestats) {
00110     Xapian::WritableDatabase db_w = get_writable_database();
00111     Xapian::Database db = get_writable_database_as_database();
00112 
00113     // Check that counts are initially zero.
00114     TEST_EQUAL(db_w.get_value_freq(0), 0);
00115     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
00116     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
00117     TEST_EQUAL(db_w.get_value_freq(1), 0);
00118     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
00119     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
00120     TEST_EQUAL(db.get_value_freq(0), 0);
00121     TEST_EQUAL(db.get_value_lower_bound(0), "");
00122     TEST_EQUAL(db.get_value_upper_bound(0), "");
00123     TEST_EQUAL(db.get_value_freq(1), 0);
00124     TEST_EQUAL(db.get_value_lower_bound(1), "");
00125     TEST_EQUAL(db.get_value_upper_bound(1), "");
00126 
00127     Xapian::Document doc;
00128     doc.add_value(0, "hello");
00129 
00130     // Check that statistics for the correct value slot increase when document
00131     // is added.  (Check slot 1 first, so that cache invalidation of the last
00132     // slot read also gets checked.)
00133     db_w.add_document(doc);
00134     TEST_EQUAL(db_w.get_value_freq(1), 0);
00135     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
00136     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
00137     TEST_EQUAL(db_w.get_value_freq(0), 1);
00138     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
00139     TEST_EQUAL(db_w.get_value_upper_bound(0), "hello");
00140 
00141     // The readonly database shouldn't change, though.
00142     TEST_EQUAL(db.get_value_freq(1), 0);
00143     TEST_EQUAL(db.get_value_lower_bound(1), "");
00144     TEST_EQUAL(db.get_value_upper_bound(1), "");
00145     TEST_EQUAL(db.get_value_freq(0), 0);
00146     TEST_EQUAL(db.get_value_lower_bound(0), "");
00147     TEST_EQUAL(db.get_value_upper_bound(0), "");
00148 
00149     // Check that statistics work correctly when second document is added.
00150     doc = Xapian::Document();
00151     doc.add_value(0, "world");
00152     doc.add_value(1, "cheese");
00153     db_w.replace_document(2, doc);
00154     TEST_EQUAL(db_w.get_value_freq(0), 2);
00155     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
00156     TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
00157     TEST_EQUAL(db_w.get_value_freq(1), 1);
00158     TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
00159     TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
00160 
00161     // The readonly database shouldn't change, though.
00162     TEST_EQUAL(db.get_value_freq(0), 0);
00163     TEST_EQUAL(db.get_value_lower_bound(0), "");
00164     TEST_EQUAL(db.get_value_upper_bound(0), "");
00165     TEST_EQUAL(db.get_value_freq(1), 0);
00166     TEST_EQUAL(db.get_value_lower_bound(1), "");
00167     TEST_EQUAL(db.get_value_upper_bound(1), "");
00168 
00169     // Check that readonly database catches up when a commit is done.
00170     db_w.commit();
00171     db.reopen();
00172     TEST_EQUAL(db.get_value_freq(1), 1);
00173     TEST_EQUAL(db.get_value_lower_bound(1), "cheese");
00174     TEST_EQUAL(db.get_value_upper_bound(1), "cheese");
00175     TEST_EQUAL(db.get_value_freq(0), 2);
00176     TEST_EQUAL(db.get_value_lower_bound(0), "hello");
00177     TEST_EQUAL(db.get_value_upper_bound(0), "world");
00178 
00179     // Deleting a document affects the count, but not the bounds.
00180     db_w.delete_document(1);
00181     TEST_EQUAL(db_w.get_value_freq(1), 1);
00182     TEST_EQUAL(db_w.get_value_lower_bound(1), "cheese");
00183     TEST_EQUAL(db_w.get_value_upper_bound(1), "cheese");
00184     TEST_EQUAL(db_w.get_value_freq(0), 1);
00185     TEST_EQUAL(db_w.get_value_lower_bound(0), "hello");
00186     TEST_EQUAL(db_w.get_value_upper_bound(0), "world");
00187 
00188     // Deleting all the documents returns the bounds to their original value.
00189     db_w.delete_document(2);
00190     TEST_EQUAL(db_w.get_value_freq(0), 0);
00191     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
00192     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
00193     TEST_EQUAL(db_w.get_value_freq(1), 0);
00194     TEST_EQUAL(db_w.get_value_lower_bound(1), "");
00195     TEST_EQUAL(db_w.get_value_upper_bound(1), "");
00196 
00197     // Adding a document with a value in one of the old slots should still
00198     // end up with tight bounds on it.
00199     doc = Xapian::Document();
00200     doc.add_value(1, "newval");
00201     db_w.replace_document(2, doc);
00202     TEST_EQUAL(db_w.get_value_freq(1), 1);
00203     TEST_EQUAL(db_w.get_value_lower_bound(1), "newval");
00204     TEST_EQUAL(db_w.get_value_upper_bound(1), "newval");
00205     TEST_EQUAL(db_w.get_value_freq(0), 0);
00206     TEST_EQUAL(db_w.get_value_lower_bound(0), "");
00207     TEST_EQUAL(db_w.get_value_upper_bound(0), "");
00208 
00209     // Check that a readonly database gets the right statistics, too.
00210     db_w.commit();
00211     db.reopen();
00212     TEST_EQUAL(db.get_value_freq(0), 0);
00213     TEST_EQUAL(db.get_value_lower_bound(0), "");
00214     TEST_EQUAL(db.get_value_upper_bound(0), "");
00215     TEST_EQUAL(db.get_value_freq(1), 1);
00216     TEST_EQUAL(db.get_value_lower_bound(1), "newval");
00217     TEST_EQUAL(db.get_value_upper_bound(1), "newval");
00218 
00219     return true;
00220 }
00221 
00223 DEFINE_TESTCASE(valuestats3, valuestats) {
00224     Xapian::Database db = get_database("apitest_simpledata");
00225 
00226     TEST_EQUAL(db.get_value_freq(1), 6);
00227     TEST_EQUAL(db.get_value_lower_bound(1), "h");
00228     TEST_EQUAL(db.get_value_upper_bound(1), "n");
00229     TEST_EQUAL(db.get_value_freq(2), 6);
00230     TEST_EQUAL(db.get_value_lower_bound(2), "d");
00231     TEST_EQUAL(db.get_value_upper_bound(2), "i");
00232     TEST_EQUAL(db.get_value_freq(3), 6);
00233     TEST_EQUAL(db.get_value_lower_bound(3), " ");
00234     TEST_EQUAL(db.get_value_upper_bound(3), "s");
00235     TEST_EQUAL(db.get_value_freq(4), 6);
00236     TEST_EQUAL(db.get_value_lower_bound(4), " ");
00237     TEST_EQUAL(db.get_value_upper_bound(4), "y");
00238     TEST_EQUAL(db.get_value_freq(5), 6);
00239     TEST_EQUAL(db.get_value_lower_bound(5), "e");
00240     TEST_EQUAL(db.get_value_upper_bound(5), "p");
00241     TEST_EQUAL(db.get_value_freq(6), 6);
00242     TEST_EQUAL(db.get_value_lower_bound(6), "a");
00243     TEST_EQUAL(db.get_value_upper_bound(6), "t");
00244     TEST_EQUAL(db.get_value_freq(7), 6);
00245     TEST_EQUAL(db.get_value_lower_bound(7), " ");
00246     TEST_EQUAL(db.get_value_upper_bound(7), "r");
00247     TEST_EQUAL(db.get_value_freq(8), 6);
00248     TEST_EQUAL(db.get_value_lower_bound(8), "a");
00249     TEST_EQUAL(db.get_value_upper_bound(8), "t");
00250     TEST_EQUAL(db.get_value_freq(9), 6);
00251     TEST_EQUAL(db.get_value_lower_bound(9), " ");
00252     TEST_EQUAL(db.get_value_upper_bound(9), "n");
00253     TEST_EQUAL(db.get_value_freq(10), 6);
00254     TEST_EQUAL(db.get_value_lower_bound(10), "e");
00255     TEST_EQUAL(db.get_value_upper_bound(10), "w");
00256     TEST_EQUAL(db.get_value_freq(11), 6);
00257     TEST_EQUAL(db.get_value_lower_bound(11), "\xb9P");
00258     TEST_EQUAL(db.get_value_upper_bound(11), "\xc7\x04");
00259 
00260     return true;
00261 }
00262 
00263 DEFINE_TESTCASE(valuestats4, transactions && valuestats) {
00264     const size_t FLUSH_THRESHOLD = 10000;
00265     {
00266         Xapian::WritableDatabase db_w = get_writable_database();
00267         Xapian::Document doc;
00268         doc.add_value(1, "test");
00269         for (size_t i = 0; i < FLUSH_THRESHOLD; ++i) {
00270             db_w.add_document(doc);
00271         }
00272 
00273         Xapian::Database db = get_writable_database_as_database();
00274         // Check that we had an automatic-commit.
00275         TEST_EQUAL(db.get_doccount(), FLUSH_THRESHOLD);
00276         // Check that the value stats are there.
00277         TEST_EQUAL(db.get_value_freq(1), FLUSH_THRESHOLD);
00278         TEST_EQUAL(db.get_value_lower_bound(1), "test");
00279         TEST_EQUAL(db.get_value_upper_bound(1), "test");
00280 
00281         db_w.begin_transaction();
00282         doc.add_value(1, "umbrella");
00283         db_w.cancel_transaction();
00284     }
00285 
00286     {
00287         Xapian::Database db = get_writable_database_as_database();
00288         // Check that we had an automatic-commit.
00289         TEST_EQUAL(db.get_doccount(), FLUSH_THRESHOLD);
00290         // Check that the value stats are there.
00291         TEST_EQUAL(db.get_value_freq(1), FLUSH_THRESHOLD);
00292         TEST_EQUAL(db.get_value_lower_bound(1), "test");
00293         TEST_EQUAL(db.get_value_upper_bound(1), "test");
00294     }
00295  
00296     return true;
00297 }
00298 
00300 DEFINE_TESTCASE(valuestats5, !backend) {
00301     Xapian::Document doc;
00302     doc.add_value(0, "zero");
00303     doc.add_value(1, "one");
00304     doc.add_value(2, "two");
00305     doc.add_value(3, "three");
00306     doc.add_value(4, "");
00307     doc.add_value(5, "five");
00308     doc.remove_value(3);
00309     doc.add_value(1, "");
00310 
00311     // Check that we don't have any empty values reported.
00312     size_t c = 0;
00313     Xapian::ValueIterator v = doc.values_begin();
00314     while (v != doc.values_end()) {
00315         TEST(!(*v).empty());
00316         ++c;
00317         ++v;
00318     }
00319     TEST_EQUAL(c, 3); // 0, 2, 5
00320 
00321     return true;
00322 }
tests/api_valuestats.cc
