tests/api_anydb.cc

Go to the documentation of this file.

/* api_anydb.cc: tests which work with any backend
 *
 * Copyright 1999,2000,2001 BrightStation PLC
 * Copyright 2002 Ananova Ltd
 * Copyright 2002,2003,2004,2005,2006,2007,2008,2009 Olly Betts
 * Copyright 2006,2008 Lemur Consulting Ltd
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */

#include <config.h>

#include "api_anydb.h"

#include <algorithm>
#include <string>

#include <xapian.h>
#include "backendmanager_local.h"
#include "testsuite.h"
#include "testutils.h"
#include "utils.h"

#include "apitest.h"

#include <list>

using namespace std;

static void
print_mset_weights(const Xapian::MSet &mset)
{
    Xapian::MSetIterator i = mset.begin();
    for ( ; i != mset.end(); ++i) {
        tout << " " << i.get_weight();
    }
}

static void
print_mset_percentages(const Xapian::MSet &mset)
{
    Xapian::MSetIterator i = mset.begin();
    for ( ; i != mset.end(); ++i) {
        tout << " " << mset.convert_to_percent(i);
    }
}

static Xapian::Query
query(Xapian::Query::op op,
      const string & t1 = string(), const string & t2 = string(),
      const string & t3 = string(), const string & t4 = string(),
      const string & t5 = string(), const string & t6 = string(),
      const string & t7 = string(), const string & t8 = string(),
      const string & t9 = string(), const string & t10 = string())
{
    vector<string> v;
    Xapian::Stem stemmer("english");
    if (!t1.empty()) v.push_back(stemmer(t1));
    if (!t2.empty()) v.push_back(stemmer(t2));
    if (!t3.empty()) v.push_back(stemmer(t3));
    if (!t4.empty()) v.push_back(stemmer(t4));
    if (!t5.empty()) v.push_back(stemmer(t5));
    if (!t6.empty()) v.push_back(stemmer(t6));
    if (!t7.empty()) v.push_back(stemmer(t7));
    if (!t8.empty()) v.push_back(stemmer(t8));
    if (!t9.empty()) v.push_back(stemmer(t9));
    if (!t10.empty()) v.push_back(stemmer(t10));
    return Xapian::Query(op, v.begin(), v.end());
}

static Xapian::Query
query(Xapian::Query::op op, Xapian::termcount parameter,
      const string & t1 = string(), const string & t2 = string(),
      const string & t3 = string(), const string & t4 = string(),
      const string & t5 = string(), const string & t6 = string(),
      const string & t7 = string(), const string & t8 = string(),
      const string & t9 = string(), const string & t10 = string())
{
    vector<string> v;
    Xapian::Stem stemmer("english");
    if (!t1.empty()) v.push_back(stemmer(t1));
    if (!t2.empty()) v.push_back(stemmer(t2));
    if (!t3.empty()) v.push_back(stemmer(t3));
    if (!t4.empty()) v.push_back(stemmer(t4));
    if (!t5.empty()) v.push_back(stemmer(t5));
    if (!t6.empty()) v.push_back(stemmer(t6));
    if (!t7.empty()) v.push_back(stemmer(t7));
    if (!t8.empty()) v.push_back(stemmer(t8));
    if (!t9.empty()) v.push_back(stemmer(t9));
    if (!t10.empty()) v.push_back(stemmer(t10));
    return Xapian::Query(op, v.begin(), v.end(), parameter);
}

static Xapian::Query
query(const string &t)
{
    return Xapian::Query(Xapian::Stem("english")(t));
}

// #######################################################################
// # Tests start here

// tests that the backend doesn't return zero docids
DEFINE_TESTCASE(zerodocid1, backend) {
    // open the database (in this case a simple text file
    // we prepared earlier)

    Xapian::Database mydb(get_database("apitest_onedoc"));

    Xapian::Enquire enquire(mydb);

    // make a simple query, with one word in it - "word".
    enquire.set_query(Xapian::Query("word"));

    // retrieve the top ten results (we only expect one)
    Xapian::MSet mymset = enquire.get_mset(0, 10);

    // We've done the query, now check that the result is what
    // we expect (1 document, with non-zero docid)
    TEST_MSET_SIZE(mymset, 1);

    TEST_AND_EXPLAIN(*(mymset.begin()) != 0,
                     "A query on a database returned a zero docid");

    return true;
}

// tests that an empty query returns no matches
DEFINE_TESTCASE(emptyquery1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));

    enquire.set_query(Xapian::Query());
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 0);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 0);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 0);
    TEST_EQUAL(mymset.get_matches_estimated(), 0);

    vector<Xapian::Query> v;
    enquire.set_query(Xapian::Query(Xapian::Query::OP_AND, v.begin(), v.end()));
    mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 0);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 0);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 0);
    TEST_EQUAL(mymset.get_matches_estimated(), 0);

    return true;
}

// tests the document count for a simple query
DEFINE_TESTCASE(simplequery1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("word"));
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 2);
    return true;
}

// tests for the right documents and weights returned with simple query
DEFINE_TESTCASE(simplequery2, backend) {
    // open the database (in this case a simple text file
    // we prepared earlier)
    Xapian::Database db = get_database("apitest_simpledata");
    Xapian::Enquire enquire(db);
    enquire.set_query(Xapian::Query("word"));

    // retrieve the top results
    Xapian::MSet mymset = enquire.get_mset(0, 10);

    // We've done the query, now check that the result is what
    // we expect (documents 2 and 4)
    mset_expect_order(mymset, 2, 4);

    // Check the weights
    Xapian::MSetIterator i = mymset.begin();
    // These weights are for BM25Weight(1,0,1,0.5,0.5)
    TEST_EQUAL_DOUBLE(i.get_weight(), 1.04648168717725);
    i++;
    TEST_EQUAL_DOUBLE(i.get_weight(), 0.640987686595914);

    return true;
}

// tests for the right document count for another simple query
DEFINE_TESTCASE(simplequery3, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 10);

    // Check that 6 documents were returned.
    TEST_MSET_SIZE(mymset, 6);

    return true;
}

// tests for the right document count for a wildcard query
// FIXME: move this to querytest (and just use an InMemory DB).
DEFINE_TESTCASE(wildquery1, backend) {
    Xapian::QueryParser queryparser;
    unsigned flags = Xapian::QueryParser::FLAG_WILDCARD |
                     Xapian::QueryParser::FLAG_LOVEHATE;
    queryparser.set_stemmer(Xapian::Stem("english"));
    queryparser.set_stemming_strategy(Xapian::QueryParser::STEM_ALL);
    Xapian::Database db = get_database("apitest_simpledata");
    queryparser.set_database(db);
    Xapian::Enquire enquire(db);

    Xapian::Query qobj = queryparser.parse_query("th*", flags);
    tout << qobj.get_description() << endl;
    enquire.set_query(qobj);
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    // Check that 6 documents were returned.
    TEST_MSET_SIZE(mymset, 6);

    qobj = queryparser.parse_query("notindb* \"this\"", flags);
    tout << qobj.get_description() << endl;
    enquire.set_query(qobj);
    mymset = enquire.get_mset(0, 10);
    // Check that 6 documents were returned.
    TEST_MSET_SIZE(mymset, 6);

    qobj = queryparser.parse_query("+notindb* \"this\"", flags);
    tout << qobj.get_description() << endl;
    enquire.set_query(qobj);
    mymset = enquire.get_mset(0, 10);
    // Check that 0 documents were returned.
    TEST_MSET_SIZE(mymset, 0);

    return true;
}

// tests a query across multiple databases
DEFINE_TESTCASE(multidb1, backend) {
    Xapian::Database mydb1(get_database("apitest_simpledata", "apitest_simpledata2"));
    Xapian::Enquire enquire1(mydb1);

    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    Xapian::Enquire enquire2(mydb2);

    // make a simple query, with one word in it - "word".
    Xapian::Query myquery("word");
    enquire1.set_query(myquery);
    enquire2.set_query(myquery);

    // retrieve the top ten results from each method of accessing
    // multiple text files
    Xapian::MSet mymset1 = enquire1.get_mset(0, 10);
    Xapian::MSet mymset2 = enquire2.get_mset(0, 10);

    TEST_EQUAL(mymset1.size(), mymset2.size());
    TEST(mset_range_is_same_weights(mymset1, 0, mymset2, 0, mymset1.size()));
    return true;
}

// tests a query across multiple databases with terms only
// in one of the two databases
DEFINE_TESTCASE(multidb2, backend && !multi) {
    Xapian::Database mydb1(get_database("apitest_simpledata",
                                  "apitest_simpledata2"));
    Xapian::Enquire enquire1(mydb1);

    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    Xapian::Enquire enquire2(mydb2);

    // make a simple query
    Xapian::Query myquery = query(Xapian::Query::OP_OR, "inmemory", "word");
    enquire1.set_query(myquery);
    enquire2.set_query(myquery);

    // retrieve the top ten results from each method of accessing
    // multiple text files
    Xapian::MSet mymset1 = enquire1.get_mset(0, 10);
    Xapian::MSet mymset2 = enquire2.get_mset(0, 10);

    TEST_EQUAL(mymset1.size(), mymset2.size());
    TEST(mset_range_is_same_weights(mymset1, 0, mymset2, 0, mymset1.size()));
    return true;
}

// test that a multidb with 2 dbs query returns correct docids
DEFINE_TESTCASE(multidb3, backend && !multi) {
    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    Xapian::Enquire enquire(mydb2);

    // make a query
    Xapian::Query myquery = query(Xapian::Query::OP_OR, "inmemory", "word");
    enquire.set_weighting_scheme(Xapian::BoolWeight());
    enquire.set_query(myquery);

    // retrieve the top ten results
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    mset_expect_order(mymset, 2, 3, 7);

    return true;
}

// test that a multidb with 3 dbs query returns correct docids
DEFINE_TESTCASE(multidb4, backend && !multi) {
    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    mydb2.add_database(get_database("apitest_termorder"));
    Xapian::Enquire enquire(mydb2);

    // make a query
    Xapian::Query myquery = query(Xapian::Query::OP_OR, "inmemory", "word");
    enquire.set_weighting_scheme(Xapian::BoolWeight());
    enquire.set_query(myquery);

    // retrieve the top ten results
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    mset_expect_order(mymset, 2, 3, 4, 10);

    return true;
}

// tests MultiPostList::skip_to().
DEFINE_TESTCASE(multidb5, backend && !multi) {
    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    Xapian::Enquire enquire(mydb2);

    // make a query
    Xapian::Query myquery = query(Xapian::Query::OP_AND, "inmemory", "word");
    enquire.set_weighting_scheme(Xapian::BoolWeight());
    enquire.set_query(myquery);

    // retrieve the top ten results
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    mset_expect_order(mymset, 2);

    return true;
}

// tests that when specifying maxitems to get_mset, no more than
// that are returned.
DEFINE_TESTCASE(msetmaxitems1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 1);
    TEST_MSET_SIZE(mymset, 1);

    mymset = enquire.get_mset(0, 5);
    TEST_MSET_SIZE(mymset, 5);

    return true;
}

// tests the returned weights are as expected (regression test for remote
// backend which was using the average weight rather than the actual document
// weight for computing weights - fixed in 1.0.0).
DEFINE_TESTCASE(expandweights1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    Xapian::ESet eset = enquire.get_eset(3, myrset, enquire.USE_EXACT_TERMFREQ);
    TEST_EQUAL(eset.size(), 3);
    TEST_EQUAL_DOUBLE(eset[0].get_weight(), 6.08904001099445);
    TEST_EQUAL_DOUBLE(eset[1].get_weight(), 6.08904001099445);
    TEST_EQUAL_DOUBLE(eset[2].get_weight(), 4.73383620844021);

    return true;
}

// Just like test_expandweights1 but without USE_EXACT_TERMFREQ.
DEFINE_TESTCASE(expandweights2, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    Xapian::ESet eset = enquire.get_eset(3, myrset);
    TEST_EQUAL(eset.size(), 3);
    if (strcmp(get_dbtype(), "multi") != 0) {
        // For a single database, the weights should be the same with or
        // without USE_EXACT_TERMFREQ.
        TEST_EQUAL_DOUBLE(eset[0].get_weight(), 6.08904001099445);
        TEST_EQUAL_DOUBLE(eset[1].get_weight(), 6.08904001099445);
        TEST_EQUAL_DOUBLE(eset[2].get_weight(), 4.73383620844021);
    } else {
        // For multiple databases, we expect that using USE_EXACT_TERMFREQ
        // will result in different weights in some cases.
        TEST_NOT_EQUAL_DOUBLE(eset[0].get_weight(), 6.08904001099445);
        TEST_EQUAL_DOUBLE(eset[1].get_weight(), 6.08904001099445);
        TEST_NOT_EQUAL_DOUBLE(eset[2].get_weight(), 4.73383620844021);
    }

    return true;
}

// tests that when specifying maxitems to get_eset, no more than
// that are returned.
DEFINE_TESTCASE(expandmaxitems1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);
    tout << "mymset.size() = " << mymset.size() << endl;
    TEST(mymset.size() >= 2);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    Xapian::ESet myeset = enquire.get_eset(1, myrset);
    TEST_EQUAL(myeset.size(), 1);

    return true;
}

// tests that a pure boolean query has all weights set to 0
DEFINE_TESTCASE(boolquery1, backend) {
    Xapian::Query myboolquery(query("this"));

    // open the database (in this case a simple text file
    // we prepared earlier)
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(myboolquery);
    enquire.set_weighting_scheme(Xapian::BoolWeight());

    // retrieve the top results
    Xapian::MSet mymset = enquire.get_mset(0, 10);

    TEST_NOT_EQUAL(mymset.size(), 0);
    TEST_EQUAL(mymset.get_max_possible(), 0);
    for (Xapian::MSetIterator i = mymset.begin(); i != mymset.end(); ++i) {
        TEST_EQUAL(i.get_weight(), 0);
    }
    return true;
}

// tests that get_mset() specifying "this" works as expected
DEFINE_TESTCASE(msetfirst1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset1 = enquire.get_mset(0, 6);
    Xapian::MSet mymset2 = enquire.get_mset(3, 3);
    TEST(mset_range_is_same(mymset1, 3, mymset2, 0, 3));

    // Regression test - we weren't adjusting the index into items[] by
    // firstitem in api/omenquire.cc.
    TEST_EQUAL(mymset1[5].get_document().get_data(),
               mymset2[2].get_document().get_data());
    return true;
}

// tests the converting-to-percent functions
DEFINE_TESTCASE(topercent1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 20);

    int last_pct = 100;
    Xapian::MSetIterator i = mymset.begin();
    for ( ; i != mymset.end(); ++i) {
        int pct = mymset.convert_to_percent(i);
        TEST_AND_EXPLAIN(pct == i.get_percent(),
                         "convert_to_%(msetitor) != convert_to_%(wt)");
        TEST_AND_EXPLAIN(pct == mymset.convert_to_percent(i.get_weight()),
                         "convert_to_%(msetitor) != convert_to_%(wt)");
        TEST_AND_EXPLAIN(pct >= 0 && pct <= 100,
                         "percentage out of range: " << pct);
        TEST_AND_EXPLAIN(pct <= last_pct, "percentage increased down mset");
        last_pct = pct;
    }
    return true;
}

// tests the percentage values returned
DEFINE_TESTCASE(topercent2, backend) {
    BackendManagerLocal local_manager;
    local_manager.set_datadir(test_driver::get_srcdir() + "/testdata/");
    Xapian::Enquire localenq(local_manager.get_database("apitest_simpledata"));
    Xapian::Enquire enquire(get_database("apitest_simpledata"));

    int pct;

    // First, test a search in which the top document scores 100%.
    enquire.set_query(query("this"));
    localenq.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 20);
    Xapian::MSet localmset = localenq.get_mset(0, 20);

    Xapian::MSetIterator i = mymset.begin();
    TEST(i != mymset.end());
    pct = mymset.convert_to_percent(i);
    TEST_EQUAL(pct, 100);

    TEST_EQUAL(mymset.get_matches_lower_bound(), localmset.get_matches_lower_bound());
    TEST_EQUAL(mymset.get_matches_upper_bound(), localmset.get_matches_upper_bound());
    TEST_EQUAL(mymset.get_matches_estimated(), localmset.get_matches_estimated());
    TEST_EQUAL_DOUBLE(mymset.get_max_attained(), localmset.get_max_attained());
    TEST_EQUAL_DOUBLE(mymset.get_max_possible(), localmset.get_max_possible());
    TEST_EQUAL(mymset.size(), localmset.size());
    TEST(mset_range_is_same(mymset, 0, localmset, 0, mymset.size()));

    // A search in which the top document doesn't have 100%
    Xapian::Query q = query(Xapian::Query::OP_OR,
                            "this", "line", "paragraph", "rubbish");
    enquire.set_query(q);
    localenq.set_query(q);
    mymset = enquire.get_mset(0, 20);
    localmset = localenq.get_mset(0, 20);

    i = mymset.begin();
    TEST(i != mymset.end());
    pct = mymset.convert_to_percent(i);
    TEST_GREATER(pct, 65);
    TEST_LESSER(pct, 75);

    ++i;

    TEST(i != mymset.end());
    pct = mymset.convert_to_percent(i);
    TEST_GREATER(pct, 40);
    TEST_LESSER(pct, 50);

    TEST_EQUAL(mymset.get_matches_lower_bound(), localmset.get_matches_lower_bound());
    TEST_EQUAL(mymset.get_matches_upper_bound(), localmset.get_matches_upper_bound());
    TEST_EQUAL(mymset.get_matches_estimated(), localmset.get_matches_estimated());
    TEST_EQUAL_DOUBLE(mymset.get_max_attained(), localmset.get_max_attained());
    TEST_EQUAL_DOUBLE(mymset.get_max_possible(), localmset.get_max_possible());
    TEST_EQUAL(mymset.size(), localmset.size());
    TEST(mset_range_is_same(mymset, 0, localmset, 0, mymset.size()));

    return true;
}

class myExpandFunctor : public Xapian::ExpandDecider {
    public:
        bool operator()(const string & tname) const {
            unsigned long sum = 0;
            for (string::const_iterator i=tname.begin(); i!=tname.end(); ++i) {
                sum += *i;
            }
//          if (verbose) {
//              tout << tname << "==> " << sum << "\n";
//          }
            return (sum % 2) == 0;
        }
};

// tests the expand decision functor
DEFINE_TESTCASE(expandfunctor1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);
    TEST(mymset.size() >= 2);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    myExpandFunctor myfunctor;

    Xapian::ESet myeset_orig = enquire.get_eset(1000, myrset);
    unsigned int neweset_size = 0;
    Xapian::ESetIterator j = myeset_orig.begin();
    for ( ; j != myeset_orig.end(); ++j) {
        if (myfunctor(*j)) neweset_size++;
    }
    Xapian::ESet myeset = enquire.get_eset(neweset_size, myrset, &myfunctor);

#if 0
    // Compare myeset with the hand-filtered version of myeset_orig.
    if (verbose) {
        tout << "orig_eset: ";
        copy(myeset_orig.begin(), myeset_orig.end(),
             ostream_iterator<Xapian::ESetItem>(tout, " "));
        tout << "\n";

        tout << "new_eset: ";
        copy(myeset.begin(), myeset.end(),
             ostream_iterator<Xapian::ESetItem>(tout, " "));
        tout << "\n";
    }
#endif
    Xapian::ESetIterator orig = myeset_orig.begin();
    Xapian::ESetIterator filt = myeset.begin();
    for (; orig != myeset_orig.end() && filt != myeset.end(); ++orig, ++filt) {
        // skip over items that shouldn't be in myeset
        while (orig != myeset_orig.end() && !myfunctor(*orig)) {
            ++orig;
        }

        TEST_AND_EXPLAIN(*orig == *filt &&
                         orig.get_weight() == filt.get_weight(),
                         "Mismatch in items " << *orig << " vs. " << *filt
                         << " after filtering");
    }

    while (orig != myeset_orig.end() && !myfunctor(*orig)) {
        ++orig;
    }

    TEST_EQUAL(orig, myeset_orig.end());
    TEST_AND_EXPLAIN(filt == myeset.end(),
                     "Extra items in the filtered eset.");
    return true;
}

// tests the percent cutoff option
DEFINE_TESTCASE(pctcutoff1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query(Xapian::Query::OP_OR,
                            "this", "line", "paragraph", "rubbish"));
    Xapian::MSet mymset1 = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Original mset pcts:";
        print_mset_percentages(mymset1);
        tout << "\n";
    }

    unsigned int num_items = 0;
    int my_pct = 100;
    int changes = 0;
    Xapian::MSetIterator i = mymset1.begin();
    int c = 0;
    for ( ; i != mymset1.end(); ++i, ++c) {
        int new_pct = mymset1.convert_to_percent(i);
        if (new_pct != my_pct) {
            changes++;
            if (changes > 3) break;
            num_items = c;
            my_pct = new_pct;
        }
    }

    TEST_AND_EXPLAIN(changes > 3, "MSet not varied enough to test");
    if (verbose) {
        tout << "Cutoff percent: " << my_pct << "\n";
    }

    enquire.set_cutoff(my_pct);
    Xapian::MSet mymset2 = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Percentages after cutoff:";
        print_mset_percentages(mymset2);
        tout << "\n";
    }

    TEST_AND_EXPLAIN(mymset2.size() >= num_items,
                     "Match with % cutoff lost too many items");

    TEST_AND_EXPLAIN(mymset2.size() == num_items ||
                     (mymset2.convert_to_percent(mymset2[num_items]) == my_pct &&
                      mymset2.convert_to_percent(mymset2.back()) == my_pct),
                     "Match with % cutoff returned too many items");

    return true;
}

// Tests the percent cutoff option combined with collapsing
DEFINE_TESTCASE(pctcutoff2, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));
    enquire.set_query(Xapian::Query(Xapian::Query::OP_AND_NOT, Xapian::Query("this"), Xapian::Query("banana")));
    Xapian::MSet mset = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Original mset pcts:";
        print_mset_percentages(mset);
        tout << "\n";
    }

    TEST(mset.size() >= 2);
    TEST(mset[0].get_percent() - mset[1].get_percent() >= 2);

    Xapian::percent cutoff = mset[0].get_percent() + mset[1].get_percent();
    cutoff /= 2;

    enquire.set_cutoff(cutoff);
    enquire.set_collapse_key(1234); // Value which is always empty.

    mset = enquire.get_mset(0, 1);
    TEST_EQUAL(mset.size(), 1);
    TEST_EQUAL(mset.get_matches_lower_bound(), 1);

    return true;
}

// tests the cutoff option
DEFINE_TESTCASE(cutoff1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query(Xapian::Query::OP_OR,
                            "this", "line", "paragraph", "rubbish"));
    Xapian::MSet mymset1 = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Original mset weights:";
        print_mset_weights(mymset1);
        tout << "\n";
    }

    unsigned int num_items = 0;
    Xapian::weight my_wt = -100;
    int changes = 0;
    Xapian::MSetIterator i = mymset1.begin();
    int c = 0;
    for ( ; i != mymset1.end(); ++i, ++c) {
        Xapian::weight new_wt = i.get_weight();
        if (new_wt != my_wt) {
            changes++;
            if (changes > 3) break;
            num_items = c;
            my_wt = new_wt;
        }
    }

    TEST_AND_EXPLAIN(changes > 3, "MSet not varied enough to test");
    if (verbose) {
        tout << "Cutoff weight: " << my_wt << "\n";
    }

    enquire.set_cutoff(0, my_wt);
    Xapian::MSet mymset2 = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Weights after cutoff:";
        print_mset_weights(mymset2);
        tout << "\n";
    }

    TEST_AND_EXPLAIN(mymset2.size() >= num_items,
                     "Match with cutoff lost too many items");

    TEST_AND_EXPLAIN(mymset2.size() == num_items ||
                     (mymset2[num_items].get_weight() == my_wt &&
                      mymset2.back().get_weight() == my_wt),
                     "Match with cutoff returned too many items");

    return true;
}

// tests the allow query terms expand option
DEFINE_TESTCASE(allowqterms1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);
    TEST(mymset.size() >= 2);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    Xapian::ESet myeset = enquire.get_eset(1000, myrset);
    Xapian::ESetIterator j = myeset.begin();
    for ( ; j != myeset.end(); ++j) {
        TEST_NOT_EQUAL(*j, "this");
    }

    Xapian::ESet myeset2 = enquire.get_eset(1000, myrset, Xapian::Enquire::INCLUDE_QUERY_TERMS);
    j = myeset2.begin();
    for ( ; j != myeset2.end(); ++j) {
        if (*j == "this") break;
    }
    TEST(j != myeset2.end());
    return true;
}

// tests that the MSet max_attained works
DEFINE_TESTCASE(maxattain1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 100);

    Xapian::weight mymax = 0;
    Xapian::MSetIterator i = mymset.begin();
    for ( ; i != mymset.end(); ++i) {
        if (i.get_weight() > mymax) mymax = i.get_weight();
    }
    TEST_EQUAL(mymax, mymset.get_max_attained());

    return true;
}

// tests a reversed boolean query
DEFINE_TESTCASE(reversebool1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));
    enquire.set_weighting_scheme(Xapian::BoolWeight());

    Xapian::MSet mymset1 = enquire.get_mset(0, 100);
    TEST_AND_EXPLAIN(mymset1.size() > 1,
                     "Mset was too small to test properly");

    enquire.set_docid_order(Xapian::Enquire::ASCENDING);
    Xapian::MSet mymset2 = enquire.get_mset(0, 100);
    enquire.set_docid_order(Xapian::Enquire::DESCENDING);
    Xapian::MSet mymset3 = enquire.get_mset(0, 100);

    // mymset1 and mymset2 should be identical
    TEST_EQUAL(mymset1.size(), mymset2.size());

    {
        Xapian::MSetIterator i = mymset1.begin();
        Xapian::MSetIterator j = mymset2.begin();
        for ( ; i != mymset1.end(), j != mymset2.end(); ++i, j++) {
            // if this fails, then setting match_sort_forward=true was not
            // the same as the default.
            TEST_EQUAL(*i, *j);
        }
    }

    // mymset1 and mymset3 should be same but reversed
    TEST_EQUAL(mymset1.size(), mymset3.size());

    {
        Xapian::MSetIterator i = mymset1.begin();
        vector<Xapian::docid> rev(mymset3.begin(), mymset3.end());
        // Next iterator not const because of compiler brokenness (egcs 1.1.2)
        vector<Xapian::docid>::reverse_iterator j = rev.rbegin();
        for ( ; i != mymset1.end(); ++i, j++) {
            // if this fails, then setting match_sort_forward=false didn't
            // reverse the results.
            TEST_EQUAL(*i, *j);
        }
    }

    return true;
}

// tests a reversed boolean query, where the full mset isn't returned
DEFINE_TESTCASE(reversebool2, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));
    enquire.set_weighting_scheme(Xapian::BoolWeight());

    Xapian::MSet mymset1 = enquire.get_mset(0, 100);

    TEST_AND_EXPLAIN(mymset1.size() > 1,
                     "Mset was too small to test properly");

    enquire.set_docid_order(Xapian::Enquire::ASCENDING);
    Xapian::doccount msize = mymset1.size() / 2;
    Xapian::MSet mymset2 = enquire.get_mset(0, msize);
    enquire.set_docid_order(Xapian::Enquire::DESCENDING);
    Xapian::MSet mymset3 = enquire.get_mset(0, msize);

    // mymset2 should be first msize items of mymset1
    TEST_EQUAL(msize, mymset2.size());
    {
        Xapian::MSetIterator i = mymset1.begin();
        Xapian::MSetIterator j = mymset2.begin();
        for ( ; i != mymset1.end(), j != mymset2.end(); ++i, j++) {
            // if this fails, then setting match_sort_forward=true was not
            // the same as the default.
            TEST_EQUAL(*i, *j);
        }
    }

    // mymset3 should be last msize items of mymset1, in reverse order
    TEST_EQUAL(msize, mymset3.size());
    {
        vector<Xapian::docid> rev(mymset1.begin(), mymset1.end());
        // Next iterator not const because of compiler brokenness (egcs 1.1.2)
        vector<Xapian::docid>::reverse_iterator i = rev.rbegin();
        Xapian::MSetIterator j = mymset3.begin();
        for ( ; j != mymset3.end(); ++i, j++) {
            // if this fails, then setting match_sort_forward=false didn't
            // reverse the results.
            TEST_EQUAL(*i, *j);
        }
    }

    return true;
}

// tests that get_matching_terms() returns the terms in the right order
DEFINE_TESTCASE(getmterms1, backend) {
    list<string> answers_list;
    answers_list.push_back("one");
    answers_list.push_back("two");
    answers_list.push_back("three");
    answers_list.push_back("four");

    Xapian::Database mydb(get_database("apitest_termorder"));
    Xapian::Enquire enquire(mydb);

    Xapian::Query myquery(Xapian::Query::OP_OR,
            Xapian::Query(Xapian::Query::OP_AND,
                    Xapian::Query("one", 1, 1),
                    Xapian::Query("three", 1, 3)),
            Xapian::Query(Xapian::Query::OP_OR,
                    Xapian::Query("four", 1, 4),
                    Xapian::Query("two", 1, 2)));

    enquire.set_query(myquery);

    Xapian::MSet mymset = enquire.get_mset(0, 10);

    TEST_MSET_SIZE(mymset, 1);
    list<string> list(enquire.get_matching_terms_begin(mymset.begin()),
                          enquire.get_matching_terms_end(mymset.begin()));
    TEST(list == answers_list);

    return true;
}

// tests that get_matching_terms() returns the terms only once
DEFINE_TESTCASE(getmterms2, backend) {
    list<string> answers_list;
    answers_list.push_back("one");
    answers_list.push_back("two");
    answers_list.push_back("three");

    Xapian::Database mydb(get_database("apitest_termorder"));
    Xapian::Enquire enquire(mydb);

    Xapian::Query myquery(Xapian::Query::OP_OR,
            Xapian::Query(Xapian::Query::OP_AND,
                    Xapian::Query("one", 1, 1),
                    Xapian::Query("three", 1, 3)),
            Xapian::Query(Xapian::Query::OP_OR,
                    Xapian::Query("one", 1, 4),
                    Xapian::Query("two", 1, 2)));

    enquire.set_query(myquery);

    Xapian::MSet mymset = enquire.get_mset(0, 10);

    TEST_MSET_SIZE(mymset, 1);
    list<string> list(enquire.get_matching_terms_begin(mymset.begin()),
                          enquire.get_matching_terms_end(mymset.begin()));
    TEST(list == answers_list);

    return true;
}

// tests that the collapsing on termpos optimisation works
DEFINE_TESTCASE(poscollapse1, backend) {
    Xapian::Query myquery1(Xapian::Query::OP_OR,
                     Xapian::Query("this", 1, 1),
                     Xapian::Query("this", 1, 1));
    Xapian::Query myquery2("this", 2, 1);

    if (verbose) {
        tout << myquery1.get_description() << "\n";
        tout << myquery2.get_description() << "\n";
    }

    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(myquery1);
    Xapian::MSet mymset1 = enquire.get_mset(0, 10);

    enquire.set_query(myquery2);
    Xapian::MSet mymset2 = enquire.get_mset(0, 10);

    TEST_EQUAL(mymset1, mymset2);

    return true;
}

// test that running a query twice returns the same results
DEFINE_TESTCASE(repeatquery1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    enquire.set_query(query(Xapian::Query::OP_OR, "this", "word"));

    Xapian::MSet mymset1 = enquire.get_mset(0, 10);
    Xapian::MSet mymset2 = enquire.get_mset(0, 10);
    TEST_EQUAL(mymset1, mymset2);

    return true;
}

// test that prefetching documents works (at least, gives same results)
DEFINE_TESTCASE(fetchdocs1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    enquire.set_query(query(Xapian::Query::OP_OR, "this", "word"));

    Xapian::MSet mymset1 = enquire.get_mset(0, 10);
    Xapian::MSet mymset2 = enquire.get_mset(0, 10);
    TEST_EQUAL(mymset1, mymset2);
    mymset2.fetch(mymset2[0], mymset2[mymset2.size() - 1]);
    mymset2.fetch(mymset2.begin(), mymset2.end());
    mymset2.fetch(mymset2.begin());
    mymset2.fetch();

    Xapian::MSetIterator it1 = mymset1.begin();
    Xapian::MSetIterator it2 = mymset2.begin();

    while (it1 != mymset1.end() && it2 != mymset2.end()) {
        TEST_EQUAL(it1.get_document().get_data(),
                   it2.get_document().get_data());
        TEST_NOT_EQUAL(it1.get_document().get_data(), "");
        TEST_NOT_EQUAL(it2.get_document().get_data(), "");
        it1++;
        it2++;
    }
    TEST_EQUAL(it1, mymset1.end());
    TEST_EQUAL(it1, mymset2.end());

    return true;
}

// test that searching for a term not in the database fails nicely
DEFINE_TESTCASE(absentterm1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_weighting_scheme(Xapian::BoolWeight());
    enquire.set_query(Xapian::Query("frink"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);
    mset_expect_order(mymset);

    return true;
}

// as absentterm1, but setting query from a vector of terms
DEFINE_TESTCASE(absentterm2, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    vector<string> terms;
    terms.push_back("frink");

    Xapian::Query query(Xapian::Query::OP_OR, terms.begin(), terms.end());
    enquire.set_query(query);

    Xapian::MSet mymset = enquire.get_mset(0, 10);
    mset_expect_order(mymset);

    return true;
}

// test that rsets do sensible things
DEFINE_TESTCASE(rset1, backend) {
    Xapian::Database mydb(get_database("apitest_rset"));
    Xapian::Enquire enquire(mydb);
    Xapian::Query myquery = query(Xapian::Query::OP_OR, "giraffe", "tiger");
    enquire.set_query(myquery);

    Xapian::MSet mymset1 = enquire.get_mset(0, 10);

    Xapian::RSet myrset;
    myrset.add_document(1);

    Xapian::MSet mymset2 = enquire.get_mset(0, 10, &myrset);

    // We should have the same documents turn up, but 1 and 3 should
    // have higher weights with the RSet.
    TEST_MSET_SIZE(mymset1, 3);
    TEST_MSET_SIZE(mymset2, 3);

    return true;
}

// test that rsets do more sensible things
DEFINE_TESTCASE(rset2, backend) {
    Xapian::Database mydb(get_database("apitest_rset"));
    Xapian::Enquire enquire(mydb);
    Xapian::Query myquery = query(Xapian::Query::OP_OR, "cuddly", "people");
    enquire.set_query(myquery);

    Xapian::MSet mymset1 = enquire.get_mset(0, 10);

    Xapian::RSet myrset;
    myrset.add_document(2);

    Xapian::MSet mymset2 = enquire.get_mset(0, 10, &myrset);

    mset_expect_order(mymset1, 1, 2);
    mset_expect_order(mymset2, 2, 1);

    return true;
}

// test that rsets behave correctly with multiDBs
DEFINE_TESTCASE(rsetmultidb1, backend && !multi) {
    Xapian::Database mydb1(get_database("apitest_rset", "apitest_simpledata2"));
    Xapian::Database mydb2(get_database("apitest_rset"));
    mydb2.add_database(get_database("apitest_simpledata2"));

    Xapian::Enquire enquire1(mydb1);
    Xapian::Enquire enquire2(mydb2);

    Xapian::Query myquery = query(Xapian::Query::OP_OR, "cuddly", "multiple");

    enquire1.set_query(myquery);
    enquire2.set_query(myquery);

    Xapian::RSet myrset1;
    Xapian::RSet myrset2;
    myrset1.add_document(4);
    myrset2.add_document(2);

    Xapian::MSet mymset1a = enquire1.get_mset(0, 10);
    Xapian::MSet mymset1b = enquire1.get_mset(0, 10, &myrset1);
    Xapian::MSet mymset2a = enquire2.get_mset(0, 10);
    Xapian::MSet mymset2b = enquire2.get_mset(0, 10, &myrset2);

    mset_expect_order(mymset1a, 1, 4);
    mset_expect_order(mymset1b, 4, 1);
    mset_expect_order(mymset2a, 1, 2);
    mset_expect_order(mymset2b, 2, 1);

    TEST(mset_range_is_same_weights(mymset1a, 0, mymset2a, 0, 2));
    TEST(mset_range_is_same_weights(mymset1b, 0, mymset2b, 0, 2));
    TEST_NOT_EQUAL(mymset1a, mymset1b);
    TEST_NOT_EQUAL(mymset2a, mymset2b);

    return true;
}

// regression tests - used to cause assertion in stats.h to fail
// Doesn't actually fail for multi but it doesn't make sense to run there.
DEFINE_TESTCASE(rsetmultidb3, backend && !multi) {
    Xapian::Enquire enquire(get_database("apitest_simpledata2"));
    enquire.set_query(query(Xapian::Query::OP_OR, "cuddly", "people"));
    Xapian::MSet mset = enquire.get_mset(0, 10); // used to fail assertion
    return true;
}

DEFINE_TESTCASE(eliteset1, backend) {
    // FIXME: OP_ELITE_SET erroneously picks the best N terms separately in
    // each sub-database!
    SKIP_TEST_FOR_BACKEND("multi");

    Xapian::Database mydb(get_database("apitest_simpledata"));
    Xapian::Enquire enquire(mydb);

    Xapian::Query myquery1 = query(Xapian::Query::OP_OR, "word");

    Xapian::Query myquery2 = query(Xapian::Query::OP_ELITE_SET, 1,
                                   "simple", "word");

    enquire.set_query(myquery1, 2); // So the query lengths are the same.
    Xapian::MSet mymset1 = enquire.get_mset(0, 10);

    enquire.set_query(myquery2);
    Xapian::MSet mymset2 = enquire.get_mset(0, 10);

    TEST_EQUAL(mymset1, mymset2);
    return true;
}

DEFINE_TESTCASE(eliteset2, backend) {
    // FIXME: OP_ELITE_SET erroneously picks the best N terms separately in
    // each sub-database!
    SKIP_TEST_FOR_BACKEND("multi");

    Xapian::Database mydb(get_database("apitest_simpledata"));
    Xapian::Enquire enquire(mydb);

    Xapian::Query myquery1 = query(Xapian::Query::OP_AND, "word", "search");

    vector<Xapian::Query> qs;
    qs.push_back(query("this"));
    qs.push_back(query(Xapian::Query::OP_AND, "word", "search"));
    Xapian::Query myquery2(Xapian::Query::OP_ELITE_SET,
                           qs.begin(), qs.end(), 1);

    enquire.set_query(myquery1);
    Xapian::MSet mymset1 = enquire.get_mset(0, 10);

    enquire.set_query(myquery2);
    Xapian::MSet mymset2 = enquire.get_mset(0, 10);

    TEST_EQUAL(mymset1, mymset2);
    // query lengths differ so mset weights not the same (with some weighting
    // parameters)
    //test_mset_order_equal(mymset1, mymset2);

    return true;
}

DEFINE_TESTCASE(eliteset3, backend) {
    Xapian::Database mydb1(get_database("apitest_simpledata"));
    Xapian::Enquire enquire1(mydb1);

    Xapian::Database mydb2(get_database("apitest_simpledata"));
    Xapian::Enquire enquire2(mydb2);

    // make a query
    Xapian::Stem stemmer("english");

    string term1 = stemmer("word");
    string term2 = stemmer("rubbish");
    string term3 = stemmer("banana");

    vector<string> terms;
    terms.push_back(term1);
    terms.push_back(term2);
    terms.push_back(term3);

    Xapian::Query myquery1(Xapian::Query::OP_OR, terms.begin(), terms.end());
    enquire1.set_query(myquery1);

    Xapian::Query myquery2(Xapian::Query::OP_ELITE_SET, terms.begin(), terms.end(), 3);
    enquire2.set_query(myquery2);

    // retrieve the results
    Xapian::MSet mymset1 = enquire1.get_mset(0, 10);
    Xapian::MSet mymset2 = enquire2.get_mset(0, 10);

    TEST_EQUAL(mymset1.get_termfreq(term1),
               mymset2.get_termfreq(term1));
    TEST_EQUAL(mymset1.get_termweight(term1),
               mymset2.get_termweight(term1));
    TEST_EQUAL(mymset1.get_termfreq(term2),
               mymset2.get_termfreq(term2));
    TEST_EQUAL(mymset1.get_termweight(term2),
               mymset2.get_termweight(term2));
    TEST_EQUAL(mymset1.get_termfreq(term3),
               mymset2.get_termfreq(term3));
    TEST_EQUAL(mymset1.get_termweight(term3),
               mymset2.get_termweight(term3));
//    TEST_EQUAL(mymset1, mymset2);

    return true;
}

DEFINE_TESTCASE(eliteset4, backend) {
    // FIXME: OP_ELITE_SET erroneously picks the best N terms separately in
    // each sub-database!
    SKIP_TEST_FOR_BACKEND("multi");

    Xapian::Database mydb1(get_database("apitest_simpledata"));
    Xapian::Enquire enquire1(mydb1);

    Xapian::Database mydb2(get_database("apitest_simpledata"));
    Xapian::Enquire enquire2(mydb2);

    Xapian::Query myquery1 = query("rubbish");
    Xapian::Query myquery2 = query(Xapian::Query::OP_ELITE_SET, 1,
                                   "word", "rubbish", "fibble");
    enquire1.set_query(myquery1);
    enquire2.set_query(myquery2);

    // retrieve the results
    Xapian::MSet mymset1 = enquire1.get_mset(0, 10);
    Xapian::MSet mymset2 = enquire2.get_mset(0, 10);

    TEST_NOT_EQUAL(mymset2.size(), 0);
    TEST_EQUAL(mymset1, mymset2);
//    TEST_EQUAL(mymset1, mymset2);

    return true;
}

DEFINE_TESTCASE(eliteset5, backend) {
    SKIP_TEST_FOR_BACKEND("multi");

    Xapian::Database mydb1(get_database("apitest_simpledata"));
    Xapian::Enquire enquire1(mydb1);

    vector<string> v;
    for (int i = 0; i != 3; ++i) {
        v.push_back("simpl");
        v.push_back("queri");

        v.push_back("rubbish");
        v.push_back("rubbish");
        v.push_back("rubbish");
        v.push_back("word");
        v.push_back("word");
        v.push_back("word");
    }

    Xapian::Query myquery1 = Xapian::Query(Xapian::Query::OP_ELITE_SET,
                                           v.begin(), v.end(), 1);
    myquery1 = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT,
                             myquery1,
                             0.004);

    enquire1.set_query(myquery1);
    // On architectures with excess precision (or, at least, on x86), the
    // following call used to result in a segfault.
    enquire1.get_mset(0, 10);

    return true;
}

DEFINE_TESTCASE(termlisttermfreq1, backend) {
    Xapian::Database mydb(get_database("apitest_simpledata"));
    Xapian::Enquire enquire(mydb);
    Xapian::Stem stemmer("english");
    Xapian::RSet rset1;
    Xapian::RSet rset2;
    rset1.add_document(5);
    rset2.add_document(6);

    Xapian::ESet eset1 = enquire.get_eset(1000, rset1);
    Xapian::ESet eset2 = enquire.get_eset(1000, rset2);

    // search for weight of term 'another'
    string theterm = stemmer("another");

    Xapian::weight wt1 = 0;
    Xapian::weight wt2 = 0;
    {
        Xapian::ESetIterator i = eset1.begin();
        for ( ; i != eset1.end(); i++) {
            if (*i == theterm) {
                wt1 = i.get_weight();
                break;
            }
        }
    }
    {
        Xapian::ESetIterator i = eset2.begin();
        for ( ; i != eset2.end(); i++) {
            if (*i == theterm) {
                wt2 = i.get_weight();
                break;
            }
        }
    }

    TEST_NOT_EQUAL(wt1, 0);
    TEST_NOT_EQUAL(wt2, 0);
    TEST_EQUAL(wt1, wt2);

    return true;
}

DEFINE_TESTCASE(qterminfo1, backend) {
    Xapian::Database mydb1(get_database("apitest_simpledata", "apitest_simpledata2"));
    Xapian::Enquire enquire1(mydb1);

    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    Xapian::Enquire enquire2(mydb2);

    // make a query
    Xapian::Stem stemmer("english");

    string term1 = stemmer("word");
    string term2 = stemmer("inmemory");
    string term3 = stemmer("flibble");

    Xapian::Query myquery(Xapian::Query::OP_OR,
                    Xapian::Query(term1),
                    Xapian::Query(Xapian::Query::OP_OR,
                            Xapian::Query(term2),
                            Xapian::Query(term3)));
    enquire1.set_query(myquery);
    enquire2.set_query(myquery);

    // retrieve the results
    Xapian::MSet mymset1a = enquire1.get_mset(0, 0);
    Xapian::MSet mymset2a = enquire2.get_mset(0, 0);

    TEST_EQUAL(mymset1a.get_termfreq(term1),
               mymset2a.get_termfreq(term1));
    TEST_EQUAL(mymset1a.get_termweight(term1),
               mymset2a.get_termweight(term1));
    TEST_EQUAL(mymset1a.get_termfreq(term2),
               mymset2a.get_termfreq(term2));
    TEST_EQUAL(mymset1a.get_termweight(term2),
               mymset2a.get_termweight(term2));
    TEST_EQUAL(mymset1a.get_termfreq(term3),
               mymset2a.get_termfreq(term3));
    TEST_EQUAL(mymset1a.get_termweight(term3),
               mymset2a.get_termweight(term3));

    TEST_EQUAL(mymset1a.get_termfreq(term1), 3);
    TEST_EQUAL(mymset1a.get_termfreq(term2), 1);
    TEST_EQUAL(mymset1a.get_termfreq(term3), 0);

    TEST_NOT_EQUAL(mymset1a.get_termweight(term1), 0);
    TEST_NOT_EQUAL(mymset1a.get_termweight(term2), 0);
    // non-existent terms still have weight
    TEST_NOT_EQUAL(mymset1a.get_termweight(term3), 0);

    TEST_EXCEPTION(Xapian::InvalidArgumentError,
                   mymset1a.get_termfreq("sponge"));

    return true;
}

DEFINE_TESTCASE(qterminfo2, backend) {
    Xapian::Database db(get_database("apitest_simpledata"));
    Xapian::Enquire enquire(db);

    // make a query
    Xapian::Stem stemmer("english");

    string term1 = stemmer("paragraph");
    string term2 = stemmer("another");

    Xapian::Query query(Xapian::Query::OP_AND_NOT, term1,
            Xapian::Query(Xapian::Query::OP_AND, term1, term2));
    enquire.set_query(query);

    // retrieve the results
    // Note: get_mset() used to throw "AssertionError" in debug builds
    Xapian::MSet mset = enquire.get_mset(0, 10);

    TEST_NOT_EQUAL(mset.get_termweight("paragraph"), 0);

    return true;
}

// tests that when specifying that no items are to be returned, those
// statistics which should be the same are.
DEFINE_TESTCASE(msetzeroitems1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset1 = enquire.get_mset(0, 0);

    Xapian::MSet mymset2 = enquire.get_mset(0, 1);

    TEST_EQUAL(mymset1.get_max_possible(), mymset2.get_max_possible());

    return true;
}

// test that the matches_* of a simple query are as expected
DEFINE_TESTCASE(matches1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    Xapian::Query myquery;
    Xapian::MSet mymset;

    myquery = query("word");
    enquire.set_query(myquery);
    mymset = enquire.get_mset(0, 10);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 2);
    TEST_EQUAL(mymset.get_matches_estimated(), 2);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 2);

    myquery = query(Xapian::Query::OP_OR, "inmemory", "word");
    enquire.set_query(myquery);
    mymset = enquire.get_mset(0, 10);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 2);
    TEST_EQUAL(mymset.get_matches_estimated(), 2);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 2);

    myquery = query(Xapian::Query::OP_AND, "inmemory", "word");
    enquire.set_query(myquery);
    mymset = enquire.get_mset(0, 10);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 0);
    TEST_EQUAL(mymset.get_matches_estimated(), 0);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 0);

    myquery = query(Xapian::Query::OP_AND, "simple", "word");
    enquire.set_query(myquery);
    mymset = enquire.get_mset(0, 10);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 2);
    TEST_EQUAL(mymset.get_matches_estimated(), 2);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 2);

    myquery = query(Xapian::Query::OP_AND, "simple", "word");
    enquire.set_query(myquery);
    mymset = enquire.get_mset(0, 0);
    // For a single database, this is true, but not for "multi" (since there
    // one sub-database has 3 documents and simple and word both have termfreq
    // of 2, so the matcher can tell at least one document must match!)
    // TEST_EQUAL(mymset.get_matches_lower_bound(), 0);
    TEST(mymset.get_matches_lower_bound() <= mymset.get_matches_estimated());
    TEST_EQUAL(mymset.get_matches_estimated(), 1);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 2);

    mymset = enquire.get_mset(0, 1);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 2);
    TEST_EQUAL(mymset.get_matches_estimated(), 2);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 2);

    mymset = enquire.get_mset(0, 2);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 2);
    TEST_EQUAL(mymset.get_matches_estimated(), 2);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 2);

    myquery = query(Xapian::Query::OP_AND, "paragraph", "another");
    enquire.set_query(myquery);
    mymset = enquire.get_mset(0, 0);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 1);
    TEST_EQUAL(mymset.get_matches_estimated(), 2);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 2);

    mymset = enquire.get_mset(0, 1);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 1);
    TEST_EQUAL(mymset.get_matches_estimated(), 2);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 2);

    mymset = enquire.get_mset(0, 2);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 1);
    TEST_EQUAL(mymset.get_matches_estimated(), 1);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 1);

    mymset = enquire.get_mset(1, 20);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 1);
    TEST_EQUAL(mymset.get_matches_estimated(), 1);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 1);

    return true;
}

// tests that wqf affects the document weights
DEFINE_TESTCASE(wqf1, backend) {
    // Both queries have length 2; in q1 word has wqf=2, in q2 word has wqf=1
    Xapian::Query q1("word", 2);
    Xapian::Query q2("word");
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(q1);
    Xapian::MSet mset1 = enquire.get_mset(0, 10);
    enquire.set_query(q2);
    Xapian::MSet mset2 = enquire.get_mset(0, 2);
    // Check the weights
    TEST(mset1.begin().get_weight() > mset2.begin().get_weight());
    return true;
}

// tests that query length affects the document weights
DEFINE_TESTCASE(qlen1, backend) {
    Xapian::Query q1("word");
    Xapian::Query q2("word");
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(q1);
    Xapian::MSet mset1 = enquire.get_mset(0, 10);
    enquire.set_query(q2);
    Xapian::MSet mset2 = enquire.get_mset(0, 2);
    // Check the weights
    //TEST(mset1.begin().get_weight() < mset2.begin().get_weight());
    TEST(mset1.begin().get_weight() == mset2.begin().get_weight());
    return true;
}

// tests that opening a non-existent termlist throws the correct exception
DEFINE_TESTCASE(termlist1, backend) {
    Xapian::Database db(get_database("apitest_onedoc"));
    TEST_EXCEPTION(Xapian::InvalidArgumentError,
                   Xapian::TermIterator t = db.termlist_begin(0));
    TEST_EXCEPTION(Xapian::DocNotFoundError,
                   Xapian::TermIterator t = db.termlist_begin(2));
    /* Cause the database to be used properly, showing up problems
     * with the link being in a bad state.  CME */
    Xapian::TermIterator temp = db.termlist_begin(1);
    TEST_EXCEPTION(Xapian::DocNotFoundError,
                   Xapian::TermIterator t = db.termlist_begin(999999999));
    return true;
}

// tests that a Xapian::TermIterator works as an STL iterator
DEFINE_TESTCASE(termlist2, backend) {
    Xapian::Database db(get_database("apitest_onedoc"));
    Xapian::TermIterator t = db.termlist_begin(1);
    Xapian::TermIterator tend = db.termlist_end(1);

    // test operator= creates a copy which compares equal
    Xapian::TermIterator t_copy = t;
    TEST_EQUAL(t, t_copy);

    // test copy constructor creates a copy which compares equal
    Xapian::TermIterator t_clone(t);
    TEST_EQUAL(t, t_clone);

    vector<string> v(t, tend);

    t = db.termlist_begin(1);
    tend = db.termlist_end(1);
    vector<string>::const_iterator i;
    for (i = v.begin(); i != v.end(); i++) {
        TEST_NOT_EQUAL(t, tend);
        TEST_EQUAL(*i, *t);
        t++;
    }
    TEST_EQUAL(t, tend);
    return true;
}

static Xapian::TermIterator
test_termlist3_helper()
{
    Xapian::Database db(get_database("apitest_onedoc"));
    return db.termlist_begin(1);
}

// tests that a Xapian::TermIterator still works when the DB is deleted
DEFINE_TESTCASE(termlist3, backend) {
    Xapian::TermIterator u = test_termlist3_helper();
    Xapian::Database db(get_database("apitest_onedoc"));
    Xapian::TermIterator t = db.termlist_begin(1);
    Xapian::TermIterator tend = db.termlist_end(1);

    while (t != tend) {
        TEST_EQUAL(*t, *u);
        t++;
        u++;
    }
    return true;
}

// tests skip_to
DEFINE_TESTCASE(termlist4, backend) {
    Xapian::Database db(get_database("apitest_onedoc"));
    Xapian::TermIterator i = db.termlist_begin(1);
    i.skip_to("");
    i.skip_to("\xff");
    return true;
}

// tests punctuation is OK in terms (particularly in remote queries)
DEFINE_TESTCASE(puncterms1, backend) {
    Xapian::Database db(get_database("apitest_punc"));
    Xapian::Enquire enquire(db);

    Xapian::Query q1("semi;colon");
    enquire.set_query(q1);
    Xapian::MSet m1 = enquire.get_mset(0, 10);

    Xapian::Query q2("col:on");
    enquire.set_query(q2);
    Xapian::MSet m2 = enquire.get_mset(0, 10);

    Xapian::Query q3("com,ma");
    enquire.set_query(q3);
    Xapian::MSet m3 = enquire.get_mset(0, 10);

    return true;
}

// test that searching for a term with a space or backslash in it works
DEFINE_TESTCASE(spaceterms1, backend) {
    Xapian::Enquire enquire(get_database("apitest_space"));
    Xapian::MSet mymset;
    Xapian::doccount count;
    Xapian::MSetIterator m;
    Xapian::Stem stemmer("english");

    enquire.set_query(stemmer("space man"));
    mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 1);
    count = 0;
    for (m = mymset.begin(); m != mymset.end(); ++m) ++count;
    TEST_EQUAL(count, 1);

    for (Xapian::valueno value_no = 1; value_no < 7; ++value_no) {
        TEST_NOT_EQUAL(mymset.begin().get_document().get_data(), "");
        TEST_NOT_EQUAL(mymset.begin().get_document().get_value(value_no), "");
    }

    enquire.set_query(stemmer("tab\tby"));
    mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 1);
    count = 0;
    for (m = mymset.begin(); m != mymset.end(); ++m) ++count;
    TEST_EQUAL(count, 1);

    for (Xapian::valueno value_no = 1; value_no < 7; ++value_no) {
        string value = mymset.begin().get_document().get_value(value_no);
        TEST_NOT_EQUAL(value, "");
        if (value_no == 0) {
            TEST(value.size() > 262);
            TEST_EQUAL(static_cast<unsigned char>(value[261]), 255);
        }
    }

    enquire.set_query(stemmer("back\\slash"));
    mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 1);
    count = 0;
    for (m = mymset.begin(); m != mymset.end(); ++m) ++count;
    TEST_EQUAL(count, 1);

    return true;
}

// test that XOR queries work
DEFINE_TESTCASE(xor1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));
    Xapian::Stem stemmer("english");

    vector<string> terms;
    terms.push_back(stemmer("this"));
    terms.push_back(stemmer("word"));
    terms.push_back(stemmer("of"));

    Xapian::Query query(Xapian::Query::OP_XOR, terms.begin(), terms.end());
    enquire.set_weighting_scheme(Xapian::BoolWeight());
    enquire.set_query(query);

    Xapian::MSet mymset = enquire.get_mset(0, 10);
    mset_expect_order(mymset, 1, 2, 5, 6);

    return true;
}

// test Xapian::Database::get_document()
DEFINE_TESTCASE(getdoc1, backend) {
    Xapian::Database db(get_database("apitest_onedoc"));
    Xapian::Document doc(db.get_document(1));
    TEST_EXCEPTION(Xapian::InvalidArgumentError, db.get_document(0));
    TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(999999999));
    TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(123456789));
    TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(3));
    TEST_EXCEPTION(Xapian::DocNotFoundError, db.get_document(2));
    // Check that Document works as a handle on modification
    // (this was broken for the first try at Xapian::Document prior to 0.7).
    Xapian::Document doc2 = doc;
    doc.set_data("modified!");
    TEST_EQUAL(doc.get_data(), "modified!");
    TEST_EQUAL(doc.get_data(), doc2.get_data());
    return true;
}

// test whether operators with no elements work as a null query
DEFINE_TESTCASE(emptyop1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    vector<Xapian::Query> nullvec;

    Xapian::Query query1(Xapian::Query::OP_XOR, nullvec.begin(), nullvec.end());

    enquire.set_query(query1);
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 0);
    TEST_EXCEPTION(Xapian::InvalidArgumentError, enquire.get_matching_terms_begin(1));

    return true;
}

// Regression test for check_at_least SEGV when there are no matches.
DEFINE_TESTCASE(checkatleast1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("thom"));
    Xapian::MSet mymset = enquire.get_mset(0, 10, 11);
    TEST_EQUAL(0, mymset.size());

    return true;
}

// Regression test - if check_at_least was set we returned (check_at_least - 1)
// results, rather than the requested msize.  Fixed in 1.0.2.
DEFINE_TESTCASE(checkatleast2, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("paragraph"));

    Xapian::MSet mymset = enquire.get_mset(0, 3, 10);
    TEST_MSET_SIZE(mymset, 3);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 5);

    mymset = enquire.get_mset(0, 2, 4);
    TEST_MSET_SIZE(mymset, 2);
    TEST_GREATER_OR_EQUAL(mymset.get_matches_lower_bound(), 4);

    return true;
}

// Feature tests - check_at_least with various sorting options.
DEFINE_TESTCASE(checkatleast3, backend) {
    Xapian::Enquire enquire(get_database("etext"));
    enquire.set_query(Xapian::Query("prussian")); // 60 matches.

    for (int order = 0; order < 3; ++order) {
        switch (order) {
            case 0:
                enquire.set_docid_order(Xapian::Enquire::ASCENDING);
                break;
            case 1:
                enquire.set_docid_order(Xapian::Enquire::DESCENDING);
                break;
            case 2:
                enquire.set_docid_order(Xapian::Enquire::DONT_CARE);
                break;
        }

        for (int sort = 0; sort < 7; ++sort) {
            bool reverse = (sort & 1);
            switch (sort) {
                case 0:
                    enquire.set_sort_by_relevance();
                    break;
                case 1: case 2:
                    enquire.set_sort_by_value(0, reverse);
                    break;
                case 3: case 4:
                    enquire.set_sort_by_value_then_relevance(0, reverse);
                    break;
                case 5: case 6:
                    enquire.set_sort_by_relevance_then_value(0, reverse);
                    break;
            }

            Xapian::MSet mset = enquire.get_mset(0, 100, 500);
            TEST_MSET_SIZE(mset, 60);
            TEST_EQUAL(mset.get_matches_lower_bound(), 60);
            TEST_EQUAL(mset.get_matches_estimated(), 60);
            TEST_EQUAL(mset.get_matches_upper_bound(), 60);

            mset = enquire.get_mset(0, 50, 100);
            TEST_MSET_SIZE(mset, 50);
            TEST_EQUAL(mset.get_matches_lower_bound(), 60);
            TEST_EQUAL(mset.get_matches_estimated(), 60);
            TEST_EQUAL(mset.get_matches_upper_bound(), 60);

            mset = enquire.get_mset(0, 10, 50);
            TEST_MSET_SIZE(mset, 10);
            TEST(mset.get_matches_lower_bound() >= 50);
        }
    }

    return true;
}

// tests all document postlists
DEFINE_TESTCASE(allpostlist1, backend) {
    Xapian::Database db(get_database("apitest_manydocs"));
    Xapian::PostingIterator i = db.postlist_begin("");
    unsigned int j = 1;
    while (i != db.postlist_end("")) {
        TEST_EQUAL(*i, j);
        i++;
        j++;
    }
    TEST_EQUAL(j, 513);

    i = db.postlist_begin("");
    j = 1;
    while (i != db.postlist_end("")) {
        TEST_EQUAL(*i, j);
        i++;
        j++;
        if (j == 50) {
            j += 10;
            i.skip_to(j);
        }
    }
    TEST_EQUAL(j, 513);

    return true;
}

static void test_emptyterm1_helper(Xapian::Database & db)
{
    // Don't bother with postlist_begin() because allpostlist tests cover that.
    TEST_EXCEPTION(Xapian::InvalidArgumentError, db.positionlist_begin(1, ""));
    TEST_EQUAL(db.get_doccount(), db.get_termfreq(""));
    TEST_EQUAL(db.get_doccount() != 0, db.term_exists(""));
    TEST_EQUAL(db.get_doccount(), db.get_collection_freq(""));
}

// tests results of passing an empty term to various methods
DEFINE_TESTCASE(emptyterm1, backend) {
    Xapian::Database db(get_database("apitest_manydocs"));
    TEST_EQUAL(db.get_doccount(), 512);
    test_emptyterm1_helper(db);

    db = get_database("apitest_onedoc");
    TEST_EQUAL(db.get_doccount(), 1);
    test_emptyterm1_helper(db);

    db = get_database("");
    TEST_EQUAL(db.get_doccount(), 0);
    test_emptyterm1_helper(db);

    return true;
}

// Feature test for Query::OP_VALUE_RANGE.
DEFINE_TESTCASE(valuerange1, backend) {
    Xapian::Database db(get_database("apitest_phrase"));
    Xapian::Enquire enq(db);
    static const char * vals[] = {
        "", " ", "a", "aa", "abcd", "e", "g", "h", "hzz", "i", "l", "z", NULL
    };
    for (const char **start = vals; *start; ++start) {
        for (const char **end = vals; *end; ++end) {
            Xapian::Query query(Xapian::Query::OP_VALUE_RANGE, 1, *start, *end);
            enq.set_query(query);
            Xapian::MSet mset = enq.get_mset(0, 20);
            // Check that documents in the MSet match the value range filter.
            set<Xapian::docid> matched;
            Xapian::MSetIterator i;
            for (i = mset.begin(); i != mset.end(); ++i) {
                matched.insert(*i);
                string value = db.get_document(*i).get_value(1);
                tout << "'" << *start << "' <= '" << value << "' <= '" << *end << "'" << endl;
                TEST(value >= *start);
                TEST(value <= *end);
            }
            // Check that documents not in the MSet don't match the value range filter.
            for (Xapian::docid j = db.get_lastdocid(); j != 0; --j) {
                if (matched.find(j) == matched.end()) {
                    string value = db.get_document(j).get_value(1);
                    tout << value << " < '" << *start << "' or > '" << *end << "'" << endl;
                    TEST(value < *start || value > *end);
                }
            }
        }
    }
    return true;
}

// Regression test for Query::OP_VALUE_LE - used to return document IDs for
// non-existent documents.
DEFINE_TESTCASE(valuerange2, writable) {
    Xapian::WritableDatabase db = get_writable_database();
    Xapian::Document doc;
    doc.set_data("5");
    doc.add_value(0, "5");
    db.replace_document(5, doc);
    Xapian::Enquire enq(db);

    Xapian::Query query(Xapian::Query::OP_VALUE_LE, 0, "6");
    enq.set_query(query);
    Xapian::MSet mset = enq.get_mset(0, 20);

    TEST_EQUAL(mset.size(), 1);
    TEST_EQUAL(*(mset[0]), 5);
    return true;
}

// Test for alldocs postlist with a sparse database.
DEFINE_TESTCASE(alldocspl1, writable) {
    Xapian::WritableDatabase db = get_writable_database();
    Xapian::Document doc;
    doc.set_data("5");
    doc.add_value(0, "5");
    db.replace_document(5, doc);

    Xapian::PostingIterator i = db.postlist_begin("");
    TEST(i != db.postlist_end(""));
    TEST_EQUAL(*i, 5);
    TEST_EQUAL(i.get_doclength(), 0);
    TEST_EQUAL(i.get_wdf(), 1);
    ++i;
    TEST(i == db.postlist_end(""));

    return true;
}

// Test reading and writing a modified alldocspostlist.
DEFINE_TESTCASE(alldocspl2, writable) {
    Xapian::PostingIterator i, end;
    {
        Xapian::WritableDatabase db = get_writable_database();
        Xapian::Document doc;
        doc.set_data("5");
        doc.add_value(0, "5");
        db.replace_document(5, doc);

        // Test iterating before flushing the changes.
        i = db.postlist_begin("");
        end = db.postlist_end("");
        TEST(i != end);
        TEST_EQUAL(*i, 5);
        TEST_EQUAL(i.get_doclength(), 0);
        TEST_EQUAL(i.get_wdf(), 1);
        ++i;
        TEST(i == end);

        db.flush();

        // Test iterating after flushing the changes.
        i = db.postlist_begin("");
        end = db.postlist_end("");
        TEST(i != end);
        TEST_EQUAL(*i, 5);
        TEST_EQUAL(i.get_doclength(), 0);
        TEST_EQUAL(i.get_wdf(), 1);
        ++i;
        TEST(i == end);

        // Add another document.
        doc = Xapian::Document();
        doc.set_data("5");
        doc.add_value(0, "7");
        db.replace_document(7, doc);

        // Test iterating through before flushing the changes.
        i = db.postlist_begin("");
        end = db.postlist_end("");
        TEST(i != end);
        TEST_EQUAL(*i, 5);
        TEST_EQUAL(i.get_doclength(), 0);
        TEST_EQUAL(i.get_wdf(), 1);
        ++i;
        TEST(i != end);
        TEST_EQUAL(*i, 7);
        TEST_EQUAL(i.get_doclength(), 0);
        TEST_EQUAL(i.get_wdf(), 1);
        ++i;
        TEST(i == end);

        // Delete the first document.
        db.delete_document(5);

        // Test iterating through before flushing the changes.
        i = db.postlist_begin("");
        end = db.postlist_end("");
        TEST(i != end);
        TEST_EQUAL(*i, 7);
        TEST_EQUAL(i.get_doclength(), 0);
        TEST_EQUAL(i.get_wdf(), 1);
        ++i;
        TEST(i == end);

        // Test iterating through after flushing the changes, and dropping the reference to the main DB.
        db.flush();
        i = db.postlist_begin("");
        end = db.postlist_end("");
    }

    TEST(i != end);
    TEST_EQUAL(*i, 7);
    TEST_EQUAL(i.get_doclength(), 0);
    TEST_EQUAL(i.get_wdf(), 1);
    ++i;
    TEST(i == end);

    return true;
}

// Feature test for Query::OP_VALUE_GE.
DEFINE_TESTCASE(valuege1, backend) {
    Xapian::Database db(get_database("apitest_phrase"));
    Xapian::Enquire enq(db);
    static const char * vals[] = {
        "", " ", "a", "aa", "abcd", "e", "g", "h", "hzz", "i", "l", "z", NULL
    };
    for (const char **start = vals; *start; ++start) {
        Xapian::Query query(Xapian::Query::OP_VALUE_GE, 1, *start);
        enq.set_query(query);
        Xapian::MSet mset = enq.get_mset(0, 20);
        // Check that documents in the MSet match the value range filter.
        set<Xapian::docid> matched;
        Xapian::MSetIterator i;
        for (i = mset.begin(); i != mset.end(); ++i) {
            matched.insert(*i);
            string value = db.get_document(*i).get_value(1);
            tout << "'" << *start << "' <= '" << value << "'" << endl;
            TEST(value >= *start);
        }
        // Check that documents not in the MSet don't match the value range filter.
        for (Xapian::docid j = db.get_lastdocid(); j != 0; --j) {
            if (matched.find(j) == matched.end()) {
                string value = db.get_document(j).get_value(1);
                tout << value << " < '" << *start << "'" << endl;
                TEST(value < *start);
            }
        }
    }
    return true;
}

// Feature test for Query::OP_VALUE_LE.
DEFINE_TESTCASE(valuele1, backend) {
    Xapian::Database db(get_database("apitest_phrase"));
    Xapian::Enquire enq(db);
    static const char * vals[] = {
        "", " ", "a", "aa", "abcd", "e", "g", "h", "hzz", "i", "l", "z", NULL
    };
    for (const char **end = vals; *end; ++end) {
        Xapian::Query query(Xapian::Query::OP_VALUE_LE, 1, *end);
        enq.set_query(query);
        Xapian::MSet mset = enq.get_mset(0, 20);
        // Check that documents in the MSet match the value range filter.
        set<Xapian::docid> matched;
        Xapian::MSetIterator i;
        for (i = mset.begin(); i != mset.end(); ++i) {
            matched.insert(*i);
            string value = db.get_document(*i).get_value(1);
            tout << "'" << *end << "' <= '" << value << "'" << endl;
            TEST(value <= *end);
        }
        // Check that documents not in the MSet don't match the value range filter.
        for (Xapian::docid j = db.get_lastdocid(); j != 0; --j) {
            if (matched.find(j) == matched.end()) {
                string value = db.get_document(j).get_value(1);
                tout << value << " < '" << *end << "'" << endl;
                TEST(value > *end);
            }
        }
    }
    return true;
}

// Feature test for Query::OP_SCALE_WEIGHT.
DEFINE_TESTCASE(scaleweight1, backend) {
    Xapian::Database db(get_database("apitest_phrase"));
    Xapian::Enquire enq(db);
    Xapian::QueryParser qp;

    static const char * queries[] = {
        "pad",
        "milk fridge",
        "leave milk on fridge",
        "ordered milk operator",
        "ordered phrase operator",
        "leave \"milk on fridge\"",
        "notpresent",
        "leave \"milk notpresent\"",
        NULL
    };
    static double multipliers[] = {
        -1000000, -2.5, -1, -0.5, 0, 0.5, 1, 2.5, 1000000,
        0, 0
    };

    for (const char **qstr = queries; *qstr; ++qstr) {
        Xapian::Query query1 = qp.parse_query(*qstr);
        tout << "query1: " << query1.get_description() << endl;
        for (double *multp = multipliers; multp[0] != multp[1]; ++multp) {
            double mult = *multp;
            if (mult < 0) {
                TEST_EXCEPTION(Xapian::InvalidArgumentError,
                               Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT,
                                             query1, mult));
                continue;
            }
            Xapian::Query query2(Xapian::Query::OP_SCALE_WEIGHT, query1, mult);
            tout << "query2: " << query2.get_description() << endl;

            enq.set_query(query1);
            Xapian::MSet mset1 = enq.get_mset(0, 20);
            enq.set_query(query2);
            Xapian::MSet mset2 = enq.get_mset(0, 20);

            TEST_EQUAL(mset1.size(), mset2.size());

            Xapian::MSetIterator i1, i2;
            if (mult > 0) {
                for (i1 = mset1.begin(), i2 = mset2.begin();
                     i1 != mset1.end() && i2 != mset2.end(); ++i1, ++i2) {
                    TEST_EQUAL_DOUBLE(i1.get_weight() * mult, i2.get_weight());
                    TEST_EQUAL(*i1, *i2);
                }
            } else {
                // Weights in mset2 are 0; so it should be sorted by docid.
                vector<Xapian::docid> ids1;
                vector<Xapian::docid> ids2;
                for (i1 = mset1.begin(), i2 = mset2.begin();
                     i1 != mset1.end() && i2 != mset2.end(); ++i1, ++i2) {
                    TEST_NOT_EQUAL_DOUBLE(i1.get_weight(), 0);
                    TEST_EQUAL_DOUBLE(i2.get_weight(), 0);
                    ids1.push_back(*i1);
                    ids2.push_back(*i2);
                }
                sort(ids1.begin(), ids1.end());
                TEST_EQUAL(ids1, ids2);
            }
        }
    }
    return true;
}

// Test Query::OP_SCALE_WEIGHT being used to multiply some of the weights of a
// search by zero.
DEFINE_TESTCASE(scaleweight2, backend) {
    Xapian::Database db(get_database("apitest_phrase"));
    Xapian::Enquire enq(db);
    Xapian::MSetIterator i;

    Xapian::Query query1("fridg");
    Xapian::Query query2(Xapian::Query::OP_SCALE_WEIGHT, query1, 2.5);
    Xapian::Query query3("milk");
    Xapian::Query query4(Xapian::Query::OP_SCALE_WEIGHT, query3, 0);
    Xapian::Query query5(Xapian::Query::OP_OR, query2, query4);

    // query5 should first return the same results as query1, in the same
    // order, and then return the results of query3 which aren't also results
    // of query1, in ascending docid order.  We test that this happens.

    // First, build a vector of docids matching the first part of the query,
    // and append the non-duplicate docids matching the second part of the
    // query.
    vector<Xapian::docid> ids1;
    set<Xapian::docid> idsin1;
    vector<Xapian::docid> ids3;

    enq.set_query(query1);
    Xapian::MSet mset1 = enq.get_mset(0, 20);
    enq.set_query(query3);
    Xapian::MSet mset3 = enq.get_mset(0, 20);
    TEST_NOT_EQUAL(mset1.size(), 0);
    for (i = mset1.begin(); i != mset1.end(); ++i) {
        ids1.push_back(*i);
        idsin1.insert(*i);
    }
    TEST_NOT_EQUAL(mset3.size(), 0);
    for (i = mset3.begin(); i != mset3.end(); ++i) {
        if (idsin1.find(*i) != idsin1.end())
            continue;
        ids3.push_back(*i);
    }
    sort(ids3.begin(), ids3.end());
    ids1.insert(ids1.end(), ids3.begin(), ids3.end());

    // Now, run the combined query and build a vector of the matching docids.
    vector<Xapian::docid> ids5;
    enq.set_query(query5);
    Xapian::MSet mset5 = enq.get_mset(0, 20);
    for (i = mset5.begin(); i != mset5.end(); ++i) {
        ids5.push_back(*i);
    }

    TEST_EQUAL(ids1, ids5);
    return true;
}

// Regression test for bug fixed in 1.0.5 - this test would failed under
// valgrind because it used an uninitialised value.
DEFINE_TESTCASE(bm25weight1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_weighting_scheme(Xapian::BM25Weight(1, 25, 1, 0.01, 0.5));
    enquire.set_query(Xapian::Query("word") );

    Xapian::MSet mset = enquire.get_mset(0, 25);

    return true;
}

// Feature test for TradWeight.
DEFINE_TESTCASE(tradweight1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_weighting_scheme(Xapian::TradWeight());
    enquire.set_query(Xapian::Query("word") );

    Xapian::MSet mset = enquire.get_mset(0, 25);

    enquire.set_weighting_scheme(Xapian::TradWeight(0));
    enquire.set_query(Xapian::Query("word") );

    mset = enquire.get_mset(0, 25);
    // FIXME: should check that TradWeight(0) means wdf and doc length really
    // don't affect the weights as stated in the documentation.

    return true;
}

---