tfidfweight.cc

Go to the documentation of this file.

 
/* Copyright (C) 2013 Aarsh Shah
 * Copyright (C) 2016 Vivek Pal
 * Copyright (C) 2016,2017,2024 Olly Betts
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see
 * <https://www.gnu.org/licenses/>.
 */
 
#include <config.h>
 
#include "xapian/weight.h"
#include "keyword.h"
#include "weight/idf-norm-dispatch.h"
#include "weight/wdf-norm-dispatch.h"
#include "weightinternal.h"
#include <cmath>
#include <cstring>
 
#include "debuglog.h"
#include "omassert.h"
#include "serialise-double.h"
 
#include "xapian/error.h"
 
using namespace std;
 
namespace Xapian {
 
static TfIdfWeight::wdf_norm
decode_wdf_norm(const string& normalizations)
{
    if (normalizations.length() != 3)
        throw Xapian::InvalidArgumentError("Normalization string is invalid");
    switch (normalizations[0]) {
        case 'b':
            return TfIdfWeight::wdf_norm::BOOLEAN;
        case 's':
            return TfIdfWeight::wdf_norm::SQUARE;
        case 'l':
            return TfIdfWeight::wdf_norm::LOG;
        case 'P':
            return TfIdfWeight::wdf_norm::PIVOTED;
        case 'L':
            return TfIdfWeight::wdf_norm::LOG_AVERAGE;
        case 'n':
            return TfIdfWeight::wdf_norm::NONE;
        case 'm':
            return TfIdfWeight::wdf_norm::MAX;
        case 'a':
            return TfIdfWeight::wdf_norm::AUG;
    }
    throw Xapian::InvalidArgumentError("Normalization string is invalid");
}
 
static TfIdfWeight::idf_norm
decode_idf_norm(const string& normalizations)
{
    if (normalizations.length() != 3)
        throw Xapian::InvalidArgumentError("Normalization string is invalid");
    switch (normalizations[1]) {
        case 'n':
            return TfIdfWeight::idf_norm::NONE;
        case 's':
            return TfIdfWeight::idf_norm::SQUARE;
        case 'f':
            return TfIdfWeight::idf_norm::FREQ;
        case 'P':
            return TfIdfWeight::idf_norm::PIVOTED;
        case 'p':
            return TfIdfWeight::idf_norm::PROB;
        case 't':
            return TfIdfWeight::idf_norm::TFIDF;
    }
    throw Xapian::InvalidArgumentError("Normalization string is invalid");
}
 
static TfIdfWeight::wt_norm
decode_wt_norm(const string& normalizations)
{
    if (normalizations.length() != 3)
        throw Xapian::InvalidArgumentError("Normalization string is invalid");
    switch (normalizations[2]) {
        case 'n':
            return TfIdfWeight::wt_norm::NONE;
    }
    throw Xapian::InvalidArgumentError("Normalization string is invalid");
}
 
TfIdfWeight::TfIdfWeight(const std::string& normals,
                         double slope, double delta)
    : TfIdfWeight::TfIdfWeight(decode_wdf_norm(normals),
                               decode_idf_norm(normals),
                               decode_wt_norm(normals),
                               slope, delta) {}
 
TfIdfWeight::TfIdfWeight(wdf_norm wdf_normalization,
                         idf_norm idf_normalization,
                         wt_norm wt_normalization,
                         double slope, double delta)
    : wdf_norm_(wdf_normalization), idf_norm_(idf_normalization),
      wt_norm_(wt_normalization), param_slope(slope), param_delta(delta)
{
    if (param_slope <= 0)
        throw Xapian::InvalidArgumentError("Parameter slope is invalid");
    if (param_delta <= 0)
        throw Xapian::InvalidArgumentError("Parameter delta is invalid");
    if (idf_norm_ != idf_norm::NONE) {
        need_stat(TERMFREQ);
        need_stat(COLLECTION_SIZE);
    }
    need_stat(WDF);
    need_stat(WDF_MAX);
    need_stat(WQF);
    if (wdf_norm_ == wdf_norm::PIVOTED || idf_norm_ == idf_norm::PIVOTED) {
        need_stat(AVERAGE_LENGTH);
        need_stat(DOC_LENGTH);
        need_stat(DOC_LENGTH_MIN);
    }
    if (wdf_norm_ == wdf_norm::LOG_AVERAGE ||
        wdf_norm_ == wdf_norm::AUG_AVERAGE) {
        need_stat(DOC_LENGTH);
        need_stat(DOC_LENGTH_MIN);
        need_stat(DOC_LENGTH_MAX);
        need_stat(UNIQUE_TERMS);
    }
    if (wdf_norm_ == wdf_norm::MAX || wdf_norm_ == wdf_norm::AUG) {
        need_stat(WDF_DOC_MAX);
    }
    if (idf_norm_ == idf_norm::GLOBAL_FREQ ||
        idf_norm_ == idf_norm::LOG_GLOBAL_FREQ ||
        idf_norm_ == idf_norm::INCREMENTED_GLOBAL_FREQ ||
        idf_norm_ == idf_norm::SQRT_GLOBAL_FREQ) {
        need_stat(COLLECTION_FREQ);
    }
}
 
TfIdfWeight *
TfIdfWeight::clone() const
{
    return new TfIdfWeight(wdf_norm_, idf_norm_, wt_norm_,
                           param_slope, param_delta);
}
 
void
TfIdfWeight::init(double factor_)
{
    if (factor_ == 0.0) {
        // This object is for the term-independent contribution, and that's
        // always zero for this scheme.
        return;
    }
 
    wqf_factor = get_wqf() * factor_;
    idfn = get_idfn(idf_norm_);
}
 
string
TfIdfWeight::name() const
{
    return "tfidf";
}
 
string
TfIdfWeight::serialise() const
{
    string result = serialise_double(param_slope);
    result += serialise_double(param_delta);
    result += static_cast<unsigned char>(wdf_norm_);
    result += static_cast<unsigned char>(idf_norm_);
    result += static_cast<unsigned char>(wt_norm_);
    return result;
}
 
TfIdfWeight *
TfIdfWeight::unserialise(const string & s) const
{
    const char *ptr = s.data();
    const char *end = ptr + s.size();
    double slope = unserialise_double(&ptr, end);
    double delta = unserialise_double(&ptr, end);
    if (rare(end - ptr != 3))
        throw Xapian::SerialisationError
              ("Incorrect data in TfIdfWeight::unserialise()");
    wdf_norm wdf_normalization = static_cast<wdf_norm>(*(ptr)++);
    idf_norm idf_normalization = static_cast<idf_norm>(*(ptr)++);
    wt_norm wt_normalization = static_cast<wt_norm>(*(ptr)++);
    return new TfIdfWeight(wdf_normalization, idf_normalization,
                           wt_normalization, slope, delta);
}
 
double
TfIdfWeight::get_sumpart(Xapian::termcount wdf,
                         Xapian::termcount doclen,
                         Xapian::termcount uniqterms,
                         Xapian::termcount wdfdocmax) const
{
    double wdfn = get_wdfn(wdf, doclen, uniqterms, wdfdocmax, wdf_norm_);
    return get_wtn(wdfn * idfn, wt_norm_) * wqf_factor;
}
 
// An upper bound can be calculated simply on the basis of wdf_max as termfreq
// and N are constants.
double
TfIdfWeight::get_maxpart() const
{
    Xapian::termcount wdf_max = get_wdf_upper_bound();
    Xapian::termcount len_min = get_doclength_lower_bound();
    double wdfn = get_wdfn(wdf_max, len_min, len_min, wdf_max, wdf_norm_);
    return get_wtn(wdfn * idfn, wt_norm_) * wqf_factor;
}
 
// Return normalized wdf, idf and weight depending on the normalization string.
double
TfIdfWeight::get_wdfn(Xapian::termcount wdf, Xapian::termcount doclen,
                      Xapian::termcount uniqterms,
                      Xapian::termcount wdfdocmax,
                      wdf_norm wdf_normalization) const
{
    switch (wdf_normalization) {
        case wdf_norm::BOOLEAN:
            if (wdf == 0) return 0;
            return 1.0;
        case wdf_norm::SQUARE:
            return (wdf * wdf);
        case wdf_norm::LOG:
            if (wdf == 0) return 0;
            return (1 + log(double(wdf)));
        case wdf_norm::PIVOTED: {
            if (wdf == 0) return 0;
            double normlen = doclen / get_average_length();
            double norm_factor = 1 / (1 - param_slope + (param_slope * normlen));
            return ((1 + log(1 + log(double(wdf)))) * norm_factor + param_delta);
        }
        case wdf_norm::LOG_AVERAGE: {
            if (wdf == 0) return 0;
            double uniqterm_double = uniqterms;
            double doclen_double = doclen;
            double wdf_avg = 1;
            if (doclen_double == 0 || uniqterm_double == 0)
                wdf_avg = 1;
            else
                wdf_avg = doclen_double / uniqterm_double;
            double num = 1 + log(double(wdf));
            double den = 1 + log(wdf_avg);
            return num / den;
        }
        case wdf_norm::AUG_LOG: {
            if (wdf == 0) return 0;
            return (0.2 + 0.8 * log(1.0 + wdf));
        }
        case wdf_norm::SQRT: {
            if (wdf == 0) return 0;
            return (sqrt(wdf - 0.5) + 1);
        }
        case wdf_norm::AUG_AVERAGE: {
            if (wdf == 0) return 0;
            return 0.9 + 0.1 * (double(wdf) / (double(doclen) / uniqterms));
        }
        case wdf_norm::MAX:
            if (rare(wdfdocmax == 0)) return 0;
            return double(wdf) / wdfdocmax;
        case wdf_norm::AUG: {
            if (wdf == 0) return 0;
            return 0.5 + 0.5 * (double(wdf) / wdfdocmax);
        }
        case wdf_norm::NONE:
            break;
    }
    return wdf;
}
 
double
TfIdfWeight::get_idfn(idf_norm idf_normalization) const
{
    Xapian::doccount termfreq = 1;
    if (idf_normalization != idf_norm::NONE) termfreq = get_termfreq();
    double N = 1.0;
    if (idf_normalization == idf_norm::PROB ||
        idf_normalization == idf_norm::SQUARE ||
        idf_normalization == idf_norm::PIVOTED ||
        idf_normalization == idf_norm::TFIDF)
            N = get_collection_size();
    Xapian::termcount collfreq = 1;
    switch (idf_normalization) {
        case idf_norm::NONE:
            return 1.0;
        case idf_norm::PROB:
            // All documents are indexed by the term
            if (N == termfreq) return 0;
            return log((N - termfreq) / termfreq);
        case idf_norm::FREQ:
            return (1.0 / termfreq);
        case idf_norm::SQUARE: {
            double x = log(N / termfreq);
            return x * x;
        }
        case idf_norm::PIVOTED:
            return log((N + 1) / termfreq);
        case idf_norm::GLOBAL_FREQ: {
            collfreq = get_collection_freq();
            return (double(collfreq) / termfreq);
        }
        case idf_norm::LOG_GLOBAL_FREQ: {
            collfreq = get_collection_freq();
            return log(double(collfreq) / termfreq + 1);
        }
        case idf_norm::INCREMENTED_GLOBAL_FREQ: {
            collfreq = get_collection_freq();
            return (double(collfreq) / termfreq + 1);
        }
        case idf_norm::SQRT_GLOBAL_FREQ: {
            collfreq = get_collection_freq();
            return sqrt(double(collfreq) / termfreq - 0.9);
        }
        case idf_norm::TFIDF:
            break;
    }
    return (log(N / termfreq));
}
 
double
TfIdfWeight::get_wtn(double wt, wt_norm wt_normalization) const
{
    (void)wt_normalization;
    return wt;
}
 
[[noreturn]]
static inline void
parameter_error(const char* message, const char* params)
{
    Xapian::Weight::Internal::parameter_error(message, "tfidf", params);
}
 
TfIdfWeight *
TfIdfWeight::create_from_parameters(const char* params) const
{
    const char* p = params;
    if (*p == '\0')
        return new Xapian::TfIdfWeight();
 
    string s;
    int code = 0;
 
    if (!Xapian::Weight::Internal::param_name(&p, s) ||
        (code = keyword(wdf_norm_tab, s.data(), s.size())) < 0) {
        if (code < 0 && s.size() == 3 && *p == '\0') {
            // Support 3 letter SMART codes such as "ntn".
            return new Xapian::TfIdfWeight(s);
        }
        parameter_error("Parameter 1 (wdf_normalisation) is invalid", params);
    }
    wdf_norm wdf_normalisation_ = static_cast<wdf_norm>(code);
 
    s.resize(0);
    if (!Xapian::Weight::Internal::param_name(&p, s) ||
        (code = keyword(idf_norm_tab, s.data(), s.size())) < 0) {
        parameter_error("Parameter 2 (idf_normalisation) is invalid", params);
    }
    idf_norm idf_normalisation_ = static_cast<idf_norm>(code);
 
    s.resize(0);
    if (!Xapian::Weight::Internal::param_name(&p, s) || s != "NONE") {
        parameter_error("Parameter 3 (wt_normalisation) is invalid", params);
    }
    wt_norm wt_normalisation_ = wt_norm::NONE;
 
    if (*p)
        parameter_error("Extra data after parameter 3", params);
    return new Xapian::TfIdfWeight(wdf_normalisation_, idf_normalisation_,
                                   wt_normalisation_);
}
 
}