Xapian: Internal Source Documentation: xapian-core: tests/api

00001 
00004 /* Copyright (C) 2007,2008,2009,2010,2011 Olly Betts
00005  * Copyright (C) 2007 Lemur Consulting Ltd
00006  *
00007  * This program is free software; you can redistribute it and/or modify
00008  * it under the terms of the GNU General Public License as published by
00009  * the Free Software Foundation; either version 2 of the License, or
00010  * (at your option) any later version.
00011  *
00012  * This program is distributed in the hope that it will be useful,
00013  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00014  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00015  * GNU General Public License for more details.
00016  *
00017  * You should have received a copy of the GNU General Public License
00018  * along with this program; if not, write to the Free Software
00019  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
00020  */
00021 
00022 #include <config.h>
00023 
00024 #include "api_spelling.h"
00025 
00026 #include <xapian.h>
00027 
00028 #include "apitest.h"
00029 #include "testsuite.h"
00030 #include "testutils.h"
00031 
00032 #include <string>
00033 
00034 using namespace std;
00035 
00036 // Test add_spelling() and remove_spelling(), which remote dbs support.
00037 DEFINE_TESTCASE(spell0, spelling || remote) {
00038     Xapian::WritableDatabase db = get_writable_database();
00039 
00040     db.add_spelling("hello");
00041     db.add_spelling("cell", 2);
00042     db.commit();
00043     db.add_spelling("zig");
00044     db.add_spelling("ch");
00045     db.add_spelling("hello", 2);
00046     db.remove_spelling("hello", 2);
00047     db.remove_spelling("cell", 6);
00048     db.commit();
00049     db.remove_spelling("hello");
00050     db.remove_spelling("nonsuch");
00051     db.remove_spelling("zzzzzzzzz", 1000000);
00052     db.remove_spelling("aarvark");
00053     db.remove_spelling("hello");
00054     db.commit();
00055     db.remove_spelling("hello");
00056 
00057     return true;
00058 }
00059 
00060 // Test basic spelling correction features.
00061 DEFINE_TESTCASE(spell1, spelling) {
00062     Xapian::WritableDatabase db = get_writable_database();
00063 
00064     // Check that the more frequent term is chosen.
00065     db.add_spelling("hello");
00066     TEST_EQUAL(db.get_spelling_suggestion("cell"), "hello");
00067     db.add_spelling("cell", 2);
00068     TEST_EQUAL(db.get_spelling_suggestion("hell"), "cell");
00069     db.commit();
00070     Xapian::Database dbr(get_writable_database_as_database());
00071     TEST_EQUAL(db.get_spelling_suggestion("hell"), "cell");
00072     TEST_EQUAL(dbr.get_spelling_suggestion("hell"), "cell");
00073 
00074     // Check suggestions for single edit errors to "zig".
00075     db.add_spelling("zig");
00076     // Transpositions:
00077     TEST_EQUAL(db.get_spelling_suggestion("izg"), "zig");
00078     TEST_EQUAL(db.get_spelling_suggestion("zgi"), "zig");
00079     // Substitutions:
00080     TEST_EQUAL(db.get_spelling_suggestion("sig"), "zig");
00081     TEST_EQUAL(db.get_spelling_suggestion("zog"), "zig");
00082     TEST_EQUAL(db.get_spelling_suggestion("zif"), "zig");
00083     // Deletions:
00084     TEST_EQUAL(db.get_spelling_suggestion("ig"), "zig");
00085     TEST_EQUAL(db.get_spelling_suggestion("zg"), "zig");
00086     TEST_EQUAL(db.get_spelling_suggestion("zi"), "zig");
00087     // Insertions:
00088     TEST_EQUAL(db.get_spelling_suggestion("azig"), "zig");
00089     TEST_EQUAL(db.get_spelling_suggestion("zaig"), "zig");
00090     TEST_EQUAL(db.get_spelling_suggestion("ziag"), "zig");
00091     TEST_EQUAL(db.get_spelling_suggestion("ziga"), "zig");
00092 
00093     // Check suggestions for single edit errors to "ch".
00094     db.add_spelling("ch");
00095     // Transpositions:
00096     TEST_EQUAL(db.get_spelling_suggestion("hc"), "ch");
00097     // Substitutions - we don't handle these for two character words:
00098     TEST_EQUAL(db.get_spelling_suggestion("qh"), "");
00099     TEST_EQUAL(db.get_spelling_suggestion("cq"), "");
00100     // Deletions would leave a single character, and we don't handle those.
00101     TEST_EQUAL(db.get_spelling_suggestion("c"), "");
00102     TEST_EQUAL(db.get_spelling_suggestion("h"), "");
00103     // Insertions:
00104     TEST_EQUAL(db.get_spelling_suggestion("qch"), "ch");
00105     TEST_EQUAL(db.get_spelling_suggestion("cqh"), "ch");
00106     TEST_EQUAL(db.get_spelling_suggestion("chq"), "ch");
00107 
00108     // Check assorted cases:
00109     TEST_EQUAL(db.get_spelling_suggestion("shello"), "hello");
00110     TEST_EQUAL(db.get_spelling_suggestion("hellot"), "hello");
00111     TEST_EQUAL(db.get_spelling_suggestion("acell"), "cell");
00112     TEST_EQUAL(db.get_spelling_suggestion("cella"), "cell");
00113     TEST_EQUAL(db.get_spelling_suggestion("acella"), "cell");
00114     TEST_EQUAL(db.get_spelling_suggestion("helo"), "hello");
00115     TEST_EQUAL(db.get_spelling_suggestion("cll"), "cell");
00116     TEST_EQUAL(db.get_spelling_suggestion("helol"), "hello");
00117     TEST_EQUAL(db.get_spelling_suggestion("clel"), "cell");
00118     TEST_EQUAL(db.get_spelling_suggestion("ecll"), "cell");
00119     TEST_EQUAL(db.get_spelling_suggestion("cll"), "cell");
00120 
00121     // Check that edit distance 3 isn't found by default:
00122     TEST_EQUAL(db.get_spelling_suggestion("shelolx"), "");
00123     TEST_EQUAL(db.get_spelling_suggestion("celling"), "");
00124     TEST_EQUAL(db.get_spelling_suggestion("dellin"), "");
00125 
00126     // Check that edit distance 3 is found if specified:
00127     TEST_EQUAL(db.get_spelling_suggestion("shelolx", 3), "hello");
00128     TEST_EQUAL(db.get_spelling_suggestion("celling", 3), "cell");
00129     TEST_EQUAL(db.get_spelling_suggestion("dellin", 3), "cell");
00130 
00131     // Make "hello" more frequent than "cell" (3 vs 2).
00132     db.add_spelling("hello", 2);
00133     TEST_EQUAL(db.get_spelling_suggestion("hell"), "hello");
00134     db.commit();
00135     TEST_EQUAL(db.get_spelling_suggestion("cello"), "hello");
00136     db.remove_spelling("hello", 2);
00137     TEST_EQUAL(db.get_spelling_suggestion("hell"), "cell");
00138     // Test "over-removing".
00139     db.remove_spelling("cell", 6);
00140     TEST_EQUAL(db.get_spelling_suggestion("cell"), "hello");
00141     db.commit();
00142     TEST_EQUAL(db.get_spelling_suggestion("cell"), "hello");
00143     db.remove_spelling("hello");
00144     TEST_EQUAL(db.get_spelling_suggestion("cell"), "");
00145 
00146     // Test removing words not in the table.
00147     db.remove_spelling("nonsuch");
00148     db.remove_spelling("zzzzzzzzz", 1000000);
00149     db.remove_spelling("aarvark");
00150 
00151     // Try removing word which was present but no longer is.
00152     db.remove_spelling("hello");
00153     db.commit();
00154     db.remove_spelling("hello");
00155 
00156     return true;
00157 }
00158 
00159 // Test spelling correction for Unicode.
00160 DEFINE_TESTCASE(spell2, spelling) {
00161     Xapian::WritableDatabase db = get_writable_database();
00162 
00163     // Check that a UTF-8 sequence counts as a single character.
00164     db.add_spelling("h\xc3\xb6hle");
00165     db.add_spelling("ascii");
00166     TEST_EQUAL(db.get_spelling_suggestion("hohle", 1), "h\xc3\xb6hle");
00167     TEST_EQUAL(db.get_spelling_suggestion("hhle", 1), "h\xc3\xb6hle");
00168     TEST_EQUAL(db.get_spelling_suggestion("\xf0\xa8\xa8\x8f\xc3\xb6le", 2), "h\xc3\xb6hle");
00169     TEST_EQUAL(db.get_spelling_suggestion("hh\xc3\xb6l"), "h\xc3\xb6hle");
00170     TEST_EQUAL(db.get_spelling_suggestion("as\xc3\xb6\xc3\xb7i"), "ascii");
00171     TEST_EQUAL(db.get_spelling_suggestion("asc\xc3\xb6i\xc3\xb7i"), "ascii");
00172     db.commit();
00173     Xapian::Database dbr(get_writable_database_as_database());
00174     TEST_EQUAL(dbr.get_spelling_suggestion("hohle", 1), "h\xc3\xb6hle");
00175     TEST_EQUAL(dbr.get_spelling_suggestion("hhle", 1), "h\xc3\xb6hle");
00176     TEST_EQUAL(dbr.get_spelling_suggestion("\xf0\xa8\xa8\x8f\xc3\xb6le", 2), "h\xc3\xb6hle");
00177     TEST_EQUAL(dbr.get_spelling_suggestion("hh\xc3\xb6l"), "h\xc3\xb6hle");
00178     TEST_EQUAL(dbr.get_spelling_suggestion("as\xc3\xb6\xc3\xb7i"), "ascii");
00179     TEST_EQUAL(dbr.get_spelling_suggestion("asc\xc3\xb6i\xc3\xb7i"), "ascii");
00180 
00181     return true;
00182 }
00183 
00184 // Test spelling correction with multi databases
00185 DEFINE_TESTCASE(spell3, spelling) {
00186     Xapian::WritableDatabase db1 = get_writable_database();
00187     // We can't just call get_writable_database() since it would delete db1
00188     // which doesn't work at all under __WIN32__ and will go wrong elsewhere if
00189     // changes to db1 are committed.
00190     Xapian::WritableDatabase db2 = get_named_writable_database("spell3", "");
00191 
00192     db1.add_spelling("hello");
00193     db1.add_spelling("cell", 2);
00194     db2.add_spelling("hello", 2);
00195     db2.add_spelling("helo");
00196 
00197     Xapian::Database db;
00198     db.add_database(db1);
00199     db.add_database(db2);
00200 
00201     TEST_EQUAL(db.get_spelling_suggestion("hello"), "");
00202     TEST_EQUAL(db.get_spelling_suggestion("hell"), "hello");
00203     TEST_EQUAL(db1.get_spelling_suggestion("hell"), "cell");
00204     TEST_EQUAL(db2.get_spelling_suggestion("hell"), "hello");
00205 
00206 
00207     // Test spelling iterator
00208     Xapian::TermIterator i(db1.spellings_begin());
00209     TEST_EQUAL(*i, "cell");
00210     TEST_EQUAL(i.get_termfreq(), 2);
00211     ++i;
00212     TEST_EQUAL(*i, "hello");
00213     TEST_EQUAL(i.get_termfreq(), 1);
00214     ++i;
00215     TEST(i == db1.spellings_end());
00216 
00217     i = db2.spellings_begin();
00218     TEST_EQUAL(*i, "hello");
00219     TEST_EQUAL(i.get_termfreq(), 2);
00220     ++i;
00221     TEST_EQUAL(*i, "helo");
00222     TEST_EQUAL(i.get_termfreq(), 1);
00223     ++i;
00224     TEST(i == db2.spellings_end());
00225 
00226     i = db.spellings_begin();
00227     TEST_EQUAL(*i, "cell");
00228     TEST_EQUAL(i.get_termfreq(), 2);
00229     ++i;
00230     TEST_EQUAL(*i, "hello");
00231     TEST_EQUAL(i.get_termfreq(), 3);
00232     ++i;
00233     TEST_EQUAL(*i, "helo");
00234     TEST_EQUAL(i.get_termfreq(), 1);
00235     ++i;
00236     TEST(i == db.spellings_end());
00237 
00238     return true;
00239 }
00240 
00241 // Regression test - check that appending works correctly.
00242 DEFINE_TESTCASE(spell4, spelling) {
00243     Xapian::WritableDatabase db = get_writable_database();
00244 
00245     db.add_spelling("check");
00246     db.add_spelling("pecks", 2);
00247     db.commit();
00248     db.add_spelling("becky");
00249     db.commit();
00250 
00251     TEST_EQUAL(db.get_spelling_suggestion("jeck", 2), "pecks");
00252 
00253     return true;
00254 }
00255 
00256 // Regression test - used to segfault with some input values.
00257 DEFINE_TESTCASE(spell5, spelling) {
00258     const char * target = "\xe4\xb8\x80\xe4\xba\x9b";
00259 
00260     Xapian::WritableDatabase db = get_writable_database();
00261     db.add_spelling(target);
00262     db.commit();
00263 
00264     string s = db.get_spelling_suggestion("\xe4\xb8\x8d", 3);
00265     TEST_EQUAL(s, target);
00266 
00267     return true;
00268 }
00269 
00270 // Test basic spelling correction features.
00271 DEFINE_TESTCASE(spell6, spelling) {
00272     Xapian::WritableDatabase db = get_writable_database();
00273 
00274     // Check that the more frequent term is chosen.
00275     db.add_spelling("hello", 2);
00276     db.add_spelling("sell", 3);
00277     TEST_EQUAL(db.get_spelling_suggestion("hell"), "sell");
00278     db.commit();
00279     Xapian::Database dbr(get_writable_database_as_database());
00280     TEST_EQUAL(db.get_spelling_suggestion("hell"), "sell");
00281     TEST_EQUAL(dbr.get_spelling_suggestion("hell"), "sell");
00282 
00283     return true;
00284 }
00285 
00286 // Test suggestions when there's an exact match.
00287 DEFINE_TESTCASE(spell7, spelling) {
00288     Xapian::WritableDatabase db = get_writable_database();
00289 
00290     // Check that the more frequent term is chosen.
00291     db.add_spelling("word", 57);
00292     db.add_spelling("wrod", 3);
00293     db.add_spelling("sword", 56);
00294     db.add_spelling("words", 57);
00295     db.add_spelling("ward", 58);
00296     db.commit();
00297     TEST_EQUAL(db.get_spelling_suggestion("ward"), "");
00298     TEST_EQUAL(db.get_spelling_suggestion("words"), "word");
00299     TEST_EQUAL(db.get_spelling_suggestion("sword"), "word");
00300     TEST_EQUAL(db.get_spelling_suggestion("wrod"), "word");
00301 
00302     return true;
00303 }
00304 
00306 DEFINE_TESTCASE(spell8, spelling) {
00307     Xapian::WritableDatabase db = get_writable_database();
00308 
00309     // kin and kin used to cancel out in "skinking".
00310     db.add_spelling("skinking", 2);
00311     db.add_spelling("stinking", 1);
00312     db.commit();
00313     TEST_EQUAL(db.get_spelling_suggestion("scimkin", 3), "skinking");
00314 
00315     return true;
00316 }
tests/api_spelling.cc
