api/matchspy.cc

Go to the documentation of this file.

/* Copyright (C) 2007,2008,2009,2010 Olly Betts
 * Copyright (C) 2007,2009 Lemur Consulting Ltd
 * Copyright (C) 2010 Richard Boulton
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 */

#include <config.h>
#include <xapian/matchspy.h>

#include <xapian/document.h>
#include <xapian/error.h>
#include <xapian/queryparser.h>
#include <xapian/registry.h>

#include <map>
#include <string>
#include <vector>

#include "autoptr.h"
#include "debuglog.h"
#include "omassert.h"
#include "serialise.h"
#include "stringutils.h"
#include "str.h"
#include "termlist.h"

#include <cfloat>
#include <cmath>

using namespace std;
using namespace Xapian;

MatchSpy::~MatchSpy() {}

MatchSpy *
MatchSpy::clone() const {
    throw UnimplementedError("MatchSpy not suitable for use with remote searches - clone() method unimplemented");
}

string
MatchSpy::name() const {
    throw UnimplementedError("MatchSpy not suitable for use with remote searches - name() method unimplemented");
}

string
MatchSpy::serialise() const {
    throw UnimplementedError("MatchSpy not suitable for use with remote searches - serialise() method unimplemented");
}

MatchSpy *
MatchSpy::unserialise(const string &, const Registry &) const {
    throw UnimplementedError("MatchSpy not suitable for use with remote searches - unserialise() method unimplemented");
}

string
MatchSpy::serialise_results() const {
    throw UnimplementedError("MatchSpy not suitable for use with remote searches - serialise_results() method unimplemented");
}

void
MatchSpy::merge_results(const string &) {
    throw UnimplementedError("MatchSpy not suitable for use with remote searches - merge_results() method unimplemented");
}

string
MatchSpy::get_description() const {
    return "Xapian::MatchSpy()";
}

XAPIAN_NORETURN(static void unsupported_method());
static void unsupported_method() {
    throw Xapian::InvalidOperationError("Method not supported for this type of termlist");
}

class ValueCountTermList : public TermList {
  private:
    map<string, Xapian::doccount>::const_iterator it;
    bool started;
    Xapian::Internal::RefCntPtr<Xapian::ValueCountMatchSpy::Internal> spy;
  public:

    ValueCountTermList(ValueCountMatchSpy::Internal * spy_) : spy(spy_) {
        it = spy->values.begin();
        started = false;
    }

    string get_termname() const {
        Assert(started);
        Assert(!at_end());
        return it->first;
    }

    Xapian::doccount get_termfreq() const {
        Assert(started);
        Assert(!at_end());
        return it->second;
    }

    TermList * next() {
        if (!started) {
            started = true;
        } else {
            Assert(!at_end());
            ++it;
        }
        return NULL;
    }

    TermList * skip_to(const string & term) {
        while (it != spy->values.end() && it->first < term) {
            ++it;
        }
        started = true;
        return NULL;
    }

    bool at_end() const {
        Assert(started);
        return it == spy->values.end();
    }

    Xapian::termcount get_approx_size() const { unsupported_method(); return 0; }
    Xapian::termcount get_wdf() const { unsupported_method(); return 0; }
    Xapian::PositionIterator positionlist_begin() const {
        unsupported_method();
        return Xapian::PositionIterator();
    }
    Xapian::termcount positionlist_count() const { unsupported_method(); return 0; }
};

class StringAndFrequency {
    std::string str;
    Xapian::doccount frequency;
  public:
    StringAndFrequency(std::string str_, Xapian::doccount frequency_)
            : str(str_), frequency(frequency_) {}

    std::string get_string() const { return str; }

    Xapian::doccount get_frequency() const { return frequency; }
};

class StringAndFreqCmpByFreq {
  public:
    StringAndFreqCmpByFreq() {}

    bool operator()(const StringAndFrequency &a,
                    const StringAndFrequency &b) const {
        if (a.get_frequency() > b.get_frequency()) return true;
        if (a.get_frequency() < b.get_frequency()) return false;
        if (a.get_string() > b.get_string()) return false;
        return true;
    }
};

class StringAndFreqTermList : public TermList {
  private:
    vector<StringAndFrequency>::const_iterator it;
    bool started;
  public:
    vector<StringAndFrequency> values;

    void init() {
        it = values.begin();
        started = false;
    }

    string get_termname() const {
        Assert(started);
        Assert(!at_end());
        return it->get_string();
    }

    Xapian::doccount get_termfreq() const {
        Assert(started);
        Assert(!at_end());
        return it->get_frequency();
    }

    TermList * next() {
        if (!started) {
            started = true;
        } else {
            Assert(!at_end());
            ++it;
        }
        return NULL;
    }

    TermList * skip_to(const string & term) {
        while (it != values.end() && it->get_string() < term) {
            ++it;
        }
        started = true;
        return NULL;
    }

    bool at_end() const {
        Assert(started);
        return it == values.end();
    }

    Xapian::termcount get_approx_size() const { unsupported_method(); return 0; }
    Xapian::termcount get_wdf() const { unsupported_method(); return 0; }
    Xapian::PositionIterator positionlist_begin() const {
        unsupported_method();
        return Xapian::PositionIterator();
    }
    Xapian::termcount positionlist_count() const { unsupported_method(); return 0; }
};

static void
get_most_frequent_items(vector<StringAndFrequency> & result,
                        const map<string, doccount> & items,
                        size_t maxitems)
{
    result.clear();
    result.reserve(maxitems);
    StringAndFreqCmpByFreq cmpfn;
    bool is_heap(false);

    for (map<string, doccount>::const_iterator i = items.begin();
         i != items.end(); i++) {
        Assert(result.size() <= maxitems);
        result.push_back(StringAndFrequency(i->first, i->second));
        if (result.size() > maxitems) {
            // Make the list back into a heap.
            if (is_heap) {
                // Only the new element isn't in the right place.
                push_heap(result.begin(), result.end(), cmpfn);
            } else {
                // Need to build heap from scratch.
                make_heap(result.begin(), result.end(), cmpfn);
                is_heap = true;
            }
            pop_heap(result.begin(), result.end(), cmpfn);
            result.pop_back();
        }
    }

    if (is_heap) {
        sort_heap(result.begin(), result.end(), cmpfn);
    } else {
        sort(result.begin(), result.end(), cmpfn);
    }
}

void
ValueCountMatchSpy::operator()(const Document &doc, weight) {
    ++(internal->total);
    string val(doc.get_value(internal->slot));
    if (!val.empty()) ++(internal->values[val]);
}

TermIterator
ValueCountMatchSpy::values_begin() const
{
    AutoPtr<ValueCountTermList> termlist(new ValueCountTermList(internal.get()));
    return Xapian::TermIterator(termlist.release());
}

TermIterator
ValueCountMatchSpy::top_values_begin(size_t maxvalues) const
{
    AutoPtr<StringAndFreqTermList> termlist(new StringAndFreqTermList);
    get_most_frequent_items(termlist->values, internal->values, maxvalues);
    termlist->init();
    return Xapian::TermIterator(termlist.release());
}

MatchSpy *
ValueCountMatchSpy::clone() const {
    return new ValueCountMatchSpy(internal->slot);
}

string
ValueCountMatchSpy::name() const {
    return "Xapian::ValueCountMatchSpy";
}

string
ValueCountMatchSpy::serialise() const {
    string result;
    result += encode_length(internal->slot);
    return result;
}

MatchSpy *
ValueCountMatchSpy::unserialise(const string & s, const Registry &) const
{
    const char * p = s.data();
    const char * end = p + s.size();

    valueno new_slot = decode_length(&p, end, false);
    if (p != end) {
        throw NetworkError("Junk at end of serialised ValueCountMatchSpy");
    }

    return new ValueCountMatchSpy(new_slot);
}

string
ValueCountMatchSpy::serialise_results() const {
    LOGCALL(REMOTE, string, "ValueCountMatchSpy::serialise_results", NO_ARGS);
    string result;
    result += encode_length(internal->total);
    result += encode_length(internal->values.size());
    for (map<string, doccount>::const_iterator i = internal->values.begin();
         i != internal->values.end(); ++i) {
        result += encode_length(i->first.size());
        result += i->first;
        result += encode_length(i->second);
    }
    RETURN(result);
}

void
ValueCountMatchSpy::merge_results(const string & s) {
    LOGCALL_VOID(REMOTE, "ValueCountMatchSpy::merge_results", s);
    const char * p = s.data();
    const char * end = p + s.size();

    internal->total += decode_length(&p, end, false);

    map<string, doccount>::size_type items = decode_length(&p, end, false);
    while (p != end) {
        while (items != 0) {
            size_t vallen = decode_length(&p, end, true);
            string val(p, vallen);
            p += vallen;
            doccount freq = decode_length(&p, end, false);
            internal->values[val] += freq;
            --items;
        }
    }
}

string
ValueCountMatchSpy::get_description() const {
    return "Xapian::ValueCountMatchSpy(" + str(internal->total) +
            " docs seen, looking in " + str(internal->values.size()) + " slots)";
}

---