api_opsynonym.cc

Go to the documentation of this file.

/* Copyright 2009,2011,2014,2022 Olly Betts
 * Copyright 2007,2008,2009 Lemur Consulting Ltd
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */

#include <config.h>

#include "api_opsynonym.h"

#include <map>
#include <set>
#include <vector>

#include <xapian.h>

#include "backendmanager.h"
#include "testsuite.h"
#include "testutils.h"

#include "apitest.h"

using namespace std;

// #######################################################################
// # Tests start here

struct synonym1_data_type {
    // How many results should have the same weight when combined with
    // OP_SYNONYM instead of OP_OR.
    int sameweight_count;
    // How many results should have a different weight when combined with
    // OP_SYNONYM instead of OP_OR.
    int diffweight_count;
    // How many subqueries.
    unsigned n_subqs;
    // The subqueries (use NOQ for unused ones).
    Xapian::Query subqs[4];
};

#define NOQ Xapian::Query::MatchNothing
static const synonym1_data_type synonym1_data[] = {
    {
        // Single term - all 33 results should be same weight.
        33, 0, 1,
        { Xapian::Query("date"), NOQ, NOQ, NOQ }
    },
    {
        // Two terms, which co-occur in some documents.
        //
        // All 34 results should be different.
        0, 34, 2,
        { Xapian::Query("sky"), Xapian::Query("date"), NOQ, NOQ }
    },
    {
        // Two terms which are entirely disjoint, and where the maximum weight
        // doesn't occur in the first or second match.
        //
        // All 18 results should be different.
        0, 18, 2,
        { Xapian::Query("gutenberg"), Xapian::Query("blockhead"), NOQ, NOQ }
    },
    {
        // All 34 results should be different.
        0, 34, 2,
        {
            Xapian::Query("date"),
            Xapian::Query(Xapian::Query::OP_OR,
                          Xapian::Query("sky"),
                          Xapian::Query("glove")),
            NOQ, NOQ
        }
    },
    {
        // All 34 results should be different.
        0, 34, 2,
        {
            Xapian::Query("date"),
            Xapian::Query(Xapian::Query::OP_OR,
                          Xapian::Query("sky"),
                          Xapian::Query("date")),
            NOQ, NOQ
        }
    },
    {
        // All 34 results should be different.
        0, 34, 2,
        {
            Xapian::Query("date"),
            Xapian::Query(Xapian::Query::OP_AND_MAYBE,
                          Xapian::Query("sky"),
                          Xapian::Query("date")),
            NOQ, NOQ
        }
    },
    {
        // All 34 results should be different.
        0, 34, 2,
        {
            Xapian::Query("date"),
            Xapian::Query(Xapian::Query::OP_AND_NOT,
                          Xapian::Query("sky"),
                          Xapian::Query("date")),
            NOQ, NOQ
        }
    },
    {
        // The AND only matches 1 document, so the estimated termfreq for the
        // whole synonym works out as 33 (due to rounding), which is the same
        // as the termfreq for "date".  Therefore most of the weights are the
        // same as just for the pure "date" search, and the only document which
        // gets a different weight is the one also matched by "sky" (because it
        // has a wdf boost).
        32, 1, 2,
        {
            Xapian::Query("date"),
            Xapian::Query(Xapian::Query::OP_AND,
                          Xapian::Query("sky"),
                          Xapian::Query("date")),
            NOQ, NOQ
        }
    },
    {
        // All 34 results should be different.
        0, 34, 2,
        {
            Xapian::Query("date"),
            Xapian::Query(Xapian::Query::OP_XOR,
                          Xapian::Query("sky"),
                          Xapian::Query("date")),
            NOQ, NOQ
        }
    },
    {
        // When the top-level operator is OR, the synonym part has an estimated
        // termfreq of 35.  When the top-level operator is SYNONYM, the whole
        // query has an estimated termfreq of 66, which is rather bogus, but
        // that's the current situation here (1.2 did better as it flattened
        // this into a single OP_SYNONYM operator and then merged the two
        // "date" terms to one with wqf=2.  We've decided we shouldn't do such
        // merging from 1.3.x on (merging to sum the scale_factors is fine, but
        // we don't do that yet - FIXME).
        //
        // Anyway, this means that currently the weights are different for all
        // matches.
        0, 34, 2,
        {
            Xapian::Query("date"),
            Xapian::Query(Xapian::Query::OP_SYNONYM,
                          Xapian::Query("sky"),
                          Xapian::Query("date")),
            NOQ, NOQ
        }
    },
    {
        // All 34 results should be different.  MAX under SYNONYM should just
        // be treated as OR.
        0, 34, 2,
        {
            Xapian::Query("date"),
            Xapian::Query(Xapian::Query::OP_MAX,
                          Xapian::Query("sky"),
                          Xapian::Query("date")),
            NOQ, NOQ
        }
    },
    {
        // All 35 results should be different.
        0, 35, 4,
        {
            Xapian::Query("sky"),
            Xapian::Query("date"),
            Xapian::Query("stein"),
            Xapian::Query("ally")
        }
    },
    {
        // The estimated term frequency for the synoynm is 2 (because the
        // estimate for the phrase is 0), which is the same as the term
        // frequency of "attitud".  Thus, the synonym gets the same weight as
        // "attitud", so documents with only "attitud" (but not the phrase) in
        // them get the same wdf, and have the same total weight.  There turns
        // out to be exactly one such document.
        1, 3, 2,
        {
            Xapian::Query("attitud"),
            Xapian::Query(Xapian::Query::OP_PHRASE,
                          Xapian::Query("german"),
                          Xapian::Query("adventur")),
            NOQ, NOQ
        }
    },
    {
        // All 54 results should be different.
        0, 54, 2,
        {
            Xapian::Query("attitud"),
            Xapian::Query(Xapian::Query::OP_OR,
                          Xapian::Query("german"),
                          Xapian::Query(Xapian::Query::OP_SYNONYM,
                                        Xapian::Query("sky"),
                                        Xapian::Query("date"))),
            NOQ, NOQ
        }
    }
};

// Check a synonym search
DEFINE_TESTCASE(synonym1, backend) {
    Xapian::Database db(get_database("etext"));

    TEST_REL(db.get_doclength_upper_bound(), >, 0);

    const Xapian::doccount lots = 214;

    for (size_t subqgroup = 0;
         subqgroup != sizeof(synonym1_data) / sizeof(synonym1_data[0]);
         ++subqgroup) {
        const synonym1_data_type & data = synonym1_data[subqgroup];
        const Xapian::Query * qlist = data.subqs;
        const Xapian::Query * qlist_end = qlist + data.n_subqs;

        // Run two queries, one joining the subqueries with OR and one joining
        // them with SYNONYM.
        Xapian::Enquire enquire(db);

        // Do the search with OP_OR, getting all the results.
        Xapian::Query orquery(Xapian::Query::OP_OR, qlist, qlist_end);
        enquire.set_query(orquery);
        Xapian::MSet ormset = enquire.get_mset(0, lots);

        // Do the search with OP_SYNONYM, getting all the results.
        Xapian::Query synquery(Xapian::Query::OP_SYNONYM, qlist, qlist_end);
        enquire.set_query(synquery);
        Xapian::MSet synmset = enquire.get_mset(0, lots);

        tout << "Comparing " << orquery << " with " << synquery << '\n';

        // Check that the queries return some results.
        TEST_NOT_EQUAL(synmset.size(), 0);
        // Check that the queries return the same number of results.
        TEST_EQUAL(synmset.size(), ormset.size());
        map<Xapian::docid, double> values_or;
        map<Xapian::docid, double> values_synonym;
        for (Xapian::doccount i = 0; i < synmset.size(); ++i) {
            values_or[*ormset[i]] = ormset[i].get_weight();
            values_synonym[*synmset[i]] = synmset[i].get_weight();
        }
        TEST_EQUAL(values_or.size(), values_synonym.size());

        /* Check that the most of the weights for items in the "or" mset are
         * different from those in the "synonym" mset. */
        int same_weight = 0;
        int different_weight = 0;
        for (map<Xapian::docid, double>::const_iterator
             j = values_or.begin(); j != values_or.end(); ++j) {
            Xapian::docid did = j->first;
            // Check that all the results in the or tree make it to the synonym
            // tree.
            TEST(values_synonym.find(did) != values_synonym.end());
            if (values_or[did] == values_synonym[did]) {
                ++same_weight;
            } else {
                ++different_weight;
            }
        }

        TEST_EQUAL(different_weight, data.diffweight_count);
        TEST_EQUAL(same_weight, data.sameweight_count);

        // Do the search with synonym, but just get the top result.
        // (Regression test - the OR subquery in the synonym postlist tree used
        // to shortcut incorrectly, and return the wrong result here).
        Xapian::MSet mset_top = enquire.get_mset(0, 1);
        TEST_EQUAL(mset_top.size(), 1);
        TEST(mset_range_is_same(mset_top, 0, synmset, 0, 1));
    }
}

// Regression test - test a synonym search with a MultiAndPostlist.
DEFINE_TESTCASE(synonym2, backend) {
    Xapian::Query query;
    vector<Xapian::Query> subqueries;
    subqueries.push_back(Xapian::Query("file"));
    subqueries.push_back(Xapian::Query("the"));
    subqueries.push_back(Xapian::Query("next"));
    subqueries.push_back(Xapian::Query("reader"));
    query = Xapian::Query(Xapian::Query::OP_AND, subqueries.begin(), subqueries.end());
    subqueries.clear();
    subqueries.push_back(query);
    subqueries.push_back(Xapian::Query("gutenberg"));
    query = Xapian::Query(Xapian::Query::OP_SYNONYM, subqueries.begin(), subqueries.end());

    tout << query << '\n';

    Xapian::Database db(get_database("etext"));
    Xapian::Enquire enquire(db);
    enquire.set_query(query);
    Xapian::MSet mset = enquire.get_mset(0, 10);
    tout << mset << '\n';

    // Regression test that OP_SCALE_WEIGHT works with OP_SYNONYM
    double maxposs = mset.get_max_possible();
    query = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 10.0);
    enquire.set_query(query);
    mset = enquire.get_mset(0, 10);
    double maxposs2 = mset.get_max_possible();

    TEST_EQUAL_DOUBLE(maxposs * 10.0, maxposs2);
}

static void
check_msets_contain_same_docs(const Xapian::MSet & mset1,
                              const Xapian::MSet & mset2)
{
    TEST_EQUAL(mset1.size(), mset2.size());

    set<Xapian::docid> docids;
    for (Xapian::doccount i = 0; i < mset1.size(); ++i) {
        docids.insert(*mset1[i]);
    }

    // Check that all the results in mset1 are in mset2.
    for (Xapian::doccount j = 0; j < mset2.size(); ++j) {
        // Check that we can erase each entry from mset2 element.  Since mset1
        // and mset2 are the same size this means we can be sure that there
        // were no repeated docids in either (it would be a bug if there were).
        TEST(docids.erase(*mset2[j]));
    }
}

// Test a synonym search which has had its weight scaled to 0.
DEFINE_TESTCASE(synonym3, backend) {
    Xapian::Query query = Xapian::Query(Xapian::Query::OP_SYNONYM,
                                        Xapian::Query("sky"),
                                        Xapian::Query("date"));

    Xapian::Database db(get_database("etext"));
    Xapian::Enquire enquire(db);
    enquire.set_query(query);
    Xapian::MSet mset_orig = enquire.get_mset(0, db.get_doccount());

    tout << query << '\n';
    tout << mset_orig << '\n';

    // Test that OP_SCALE_WEIGHT with a factor of 0.0 works with OP_SYNONYM
    // (this has a special codepath to avoid doing the synonym calculation).
    query = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, query, 0.0);
    enquire.set_query(query);
    Xapian::MSet mset_zero = enquire.get_mset(0, db.get_doccount());

    tout << query << '\n';
    tout << mset_zero << '\n';

    // Check that the queries return some results.
    TEST_NOT_EQUAL(mset_zero.size(), 0);
    // Check that the queries return the same document IDs, and the zero
    // one has zero weight.
    check_msets_contain_same_docs(mset_orig, mset_zero);
    for (Xapian::doccount i = 0; i < mset_orig.size(); ++i) {
        TEST_NOT_EQUAL(mset_orig[i].get_weight(), 0.0);
        TEST_EQUAL(mset_zero[i].get_weight(), 0.0);
    }
}

// Test synonym searches combined with various operators.
DEFINE_TESTCASE(synonym4, backend) {
    Xapian::Database db(get_database("etext"));
    Xapian::Enquire enquire(db);
    Xapian::Query syn_query = Xapian::Query(Xapian::Query::OP_SYNONYM,
                                            Xapian::Query("gutenberg"),
                                            Xapian::Query("blockhead"));
    Xapian::Query or_query = Xapian::Query(Xapian::Query::OP_OR,
                                           Xapian::Query("gutenberg"),
                                           Xapian::Query("blockhead"));
    Xapian::Query date_query = Xapian::Query("date");

    // Check some queries.
    static const Xapian::Query::op operators[] = {
        Xapian::Query::OP_AND_MAYBE,
        Xapian::Query::OP_AND_NOT,
        Xapian::Query::OP_AND,
        Xapian::Query::OP_XOR,
        Xapian::Query::OP_OR,
        Xapian::Query::OP_SYNONYM
    };
    const Xapian::Query::op * end;
    end = operators + sizeof(operators) / sizeof(operators[0]);
    for (const Xapian::Query::op * i = operators; i != end; ++i) {
        tout.str(string());
        Xapian::Query query1(*i, syn_query, date_query);
        Xapian::Query query2(*i, or_query, date_query);

        enquire.set_query(query1);
        tout << "query1:" << query1 << '\n';
        Xapian::MSet mset1 = enquire.get_mset(0, db.get_doccount());
        tout << "mset1:" << mset1 << '\n';
        enquire.set_query(query2);
        tout << "query2:" << query2 << '\n';
        Xapian::MSet mset2 = enquire.get_mset(0, db.get_doccount());
        tout << "mset2:" << mset2 << '\n';

        TEST_NOT_EQUAL(mset1.size(), 0);
        if (*i != Xapian::Query::OP_XOR) {
            TEST_EQUAL(mset1[0].get_percent(), 100);
        } else {
            TEST(mset1[0].get_percent() != 100);
        }
        check_msets_contain_same_docs(mset1, mset2);
    }
}

DEFINE_TESTCASE(opmax1, backend) {
    Xapian::Database db(get_database("etext"));
    Xapian::Enquire enq(db);
    Xapian::Query q1("king");
    Xapian::Query q2("friedrich");
    Xapian::Query qmax(Xapian::Query::OP_MAX, q1, q2);
    enq.set_query(q1);
    Xapian::MSet mset1 = enq.get_mset(0, db.get_doccount());
    enq.set_query(q2);
    Xapian::MSet mset2 = enq.get_mset(0, db.get_doccount());
    enq.set_query(qmax);
    Xapian::MSet msetmax = enq.get_mset(0, db.get_doccount());

    // Check that the weights in msetmax are the maximum of the weights in
    // mset1 and mset2 for each docid.
    map<Xapian::docid, double> expected_weights;
    Xapian::MSetIterator i;
    for (i = mset1.begin(); i != mset1.end(); ++i) {
        expected_weights[*i] = i.get_weight();
    }
    for (i = mset2.begin(); i != mset2.end(); ++i) {
        map<Xapian::docid, double>::iterator j;
        j = expected_weights.find(*i);
        if (j != expected_weights.end()) {
            j->second = max(j->second, i.get_weight());
        } else {
            expected_weights[*i] = i.get_weight();
        }
    }

    for (i = msetmax.begin(); i != msetmax.end(); ++i) {
        map<Xapian::docid, double>::iterator j;
        j = expected_weights.find(*i);
        TEST(j != expected_weights.end());
        TEST_EQUAL_DOUBLE(j->second, i.get_weight());
        expected_weights.erase(j);
        tout << expected_weights.size() << '\n';
    }

    // Any document in mset1 or mset2 should also be in msetmax.
    TEST_EQUAL(expected_weights.size(), 0);
}