xapian-core  1.4.26
pl2weight.cc
Go to the documentation of this file.
1 
4 /* Copyright (C) 2013 Aarsh Shah
5  * Copyright (C) 2013,2014,2016 Olly Betts
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License as
9  * published by the Free Software Foundation; either version 2 of the
10  * License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include <config.h>
23 
24 #include "xapian/weight.h"
25 #include "common/log2.h"
26 
27 #include "serialise-double.h"
28 
29 #include "xapian/error.h"
30 
31 #include <algorithm>
32 
33 using namespace std;
34 
35 namespace Xapian {
36 
37 PL2Weight::PL2Weight(double c) : param_c(c)
38 {
39  if (param_c <= 0)
40  throw Xapian::InvalidArgumentError("Parameter c is invalid");
47  need_stat(WDF);
49  need_stat(WQF);
50 }
51 
52 PL2Weight *
54 {
55  return new PL2Weight(param_c);
56 }
57 
58 void
59 PL2Weight::init(double factor_)
60 {
61  if (factor_ == 0.0) {
62  // This object is for the term-independent contribution, and that's
63  // always zero for this scheme.
64  return;
65  }
66 
67  // lower_bound is really factor.
68  lower_bound = factor_;
69 
70  if (get_wdf_upper_bound() == 0) {
71  // The "extra" weight object is cloned, init() called and then
72  // get_maxextra() is called and we discover that we don't need it.
73  // So we need to handle that case (which will give us 0 from
74  // get_wdf_upper_bound() here).
75  upper_bound = 0;
76  return;
77  }
78 
79  // lower_bound is really factor.
80  lower_bound *= get_wqf();
81 
83 
84  double base_change(1.0 / log(2.0));
85  double mean = double(get_collection_freq()) / get_collection_size();
86  P1 = mean * base_change + 0.5 * log2(2.0 * M_PI);
87  P2 = log2(mean) + base_change;
88 
89  double wdfn_lower = log2(1 + cl / get_doclength_upper_bound());
90  double divisior = max(get_wdf_upper_bound(), get_doclength_lower_bound());
91  double wdfn_upper = get_wdf_upper_bound() * log2(1 + cl / divisior);
92 
93  // Calculate an upper bound on the weights which get_sumpart() can return.
94  //
95  // We consider the equation for P as the sum of two parts which we
96  // maximise individually:
97  //
98  // (a) (wdfn + 0.5) / (wdfn + 1) * log2(wdfn)
99  // (b) (P1 - P2 * wdfn) / (wdfn + 1)
100  //
101  // To maximise (a), the fractional part is always positive (since wdfn>0)
102  // and is maximised by maximising wdfn - clearer when rewritten as:
103  // (1 - 0.5 / (wdfn + 1))
104  //
105  // The log part of (a) is clearly also maximised by maximising wdfn,
106  // so we want to evaluate (a) at wdfn=wdfn_upper.
107  double P_max2a = (wdfn_upper + 0.5) * log2(wdfn_upper) / (wdfn_upper + 1.0);
108  // To maximise (b) substitute x=wdfn+1 (so x>1) and we get:
109  //
110  // (P1 + P2)/x - P2
111  //
112  // Differentiating wrt x gives:
113  //
114  // -(P1 + P2)/x²
115  //
116  // So there are no local minima or maxima, and the function is continuous
117  // in the range of interest, so the sign of this differential tells us
118  // whether we want to maximise or minimise wdfn, and since x>1, we can
119  // just consider the sign of: (P1 + P2)
120  //
121  // Commonly P1 + P2 > 0, in which case we evaluate P at wdfn=wdfn_upper
122  // giving us a bound that can't be bettered if wdfn_upper is tight.
123  double wdfn_optb = P1 + P2 > 0 ? wdfn_upper : wdfn_lower;
124  double P_max2b = (P1 - P2 * wdfn_optb) / (wdfn_optb + 1.0);
125  // lower_bound is really factor.
126  upper_bound = lower_bound * (P_max2a + P_max2b);
127 
128  if (rare(upper_bound <= 0)) upper_bound = 0;
129 }
130 
131 string
133 {
134  return "Xapian::PL2Weight";
135 }
136 
137 string
139 {
140  return serialise_double(param_c);
141 }
142 
143 PL2Weight *
144 PL2Weight::unserialise(const string & s) const
145 {
146  const char *ptr = s.data();
147  const char *end = ptr + s.size();
148  double c = unserialise_double(&ptr, end);
149  if (rare(ptr != end))
150  throw Xapian::SerialisationError("Extra data in PL2Weight::unserialise()");
151  return new PL2Weight(c);
152 }
153 
154 double
156  Xapian::termcount) const
157 {
158  if (wdf == 0) return 0.0;
159 
160  double wdfn = wdf * log2(1 + cl / len);
161 
162  double P = P1 + (wdfn + 0.5) * log2(wdfn) - P2 * wdfn;
163  if (rare(P <= 0)) return 0.0;
164 
165  // lower_bound is really factor.
166  return lower_bound * P / (wdfn + 1.0);
167 }
168 
169 double
171 {
172  return upper_bound;
173 }
174 
175 double
177 {
178  return 0;
179 }
180 
181 double
183 {
184  return 0;
185 }
186 
187 }
The Xapian namespace contains public interfaces for the Xapian library.
Definition: compactor.cc:80
Xapian::doccount get_collection_size() const
The number of documents in the collection.
Definition: weight.h:374
Xapian::termcount get_collection_freq() const
The collection frequency of the term.
Definition: weight.h:389
double param_c
The wdf normalization parameter in the formula.
Definition: weight.h:1198
Upper bound on document lengths.
Definition: weight.h:68
double upper_bound
The upper bound on the weight.
Definition: weight.h:1210
STL namespace.
Lower bound on (non-zero) document lengths.
Definition: weight.h:62
double get_maxextra() const
Return an upper bound on what get_sumextra() can return for any document.
Definition: pl2weight.cc:182
#define rare(COND)
Definition: config.h:575
double lower_bound
The factor to multiply weights by.
Definition: weight.h:1207
Hierarchy of classes which Xapian can throw as exceptions.
unsigned XAPIAN_TERMCOUNT_BASE_TYPE termcount
A counts of terms.
Definition: types.h:72
functions to serialise and unserialise a double
Length of the current document (sum wdf).
Definition: weight.h:56
InvalidArgumentError indicates an invalid parameter value was passed to the API.
Definition: error.h:241
Xapian::termcount get_doclength_lower_bound() const
A lower bound on the minimum length of any document in the shard.
Definition: weight.h:411
double unserialise_double(const char **p, const char *end)
Unserialise a double serialised by serialise_double.
Indicates an error in the std::string serialisation of an object.
Definition: error.h:929
Within-query-frequency of the current term.
Definition: weight.h:52
Average length of documents in the collection.
Definition: weight.h:44
PL2Weight * clone() const
Clone this object.
Definition: pl2weight.cc:53
void init(double factor)
Allow the subclass to perform any initialisation it needs to.
Definition: pl2weight.cc:59
std::string serialise() const
Return this object&#39;s parameters serialised as a single string.
Definition: pl2weight.cc:138
std::string name() const
Return the name of this weighting scheme.
Definition: pl2weight.cc:132
This class implements the PL2 weighting scheme.
Definition: weight.h:1196
Xapian::termcount get_wqf() const
The within-query-frequency of this term.
Definition: weight.h:395
double get_sumpart(Xapian::termcount wdf, Xapian::termcount doclen, Xapian::termcount uniqterms) const
Calculate the weight contribution for this object&#39;s term to a document.
Definition: pl2weight.cc:155
Xapian::termcount get_doclength_upper_bound() const
An upper bound on the maximum length of any document in the shard.
Definition: weight.h:401
Sum of wdf over the whole collection for the current term.
Definition: weight.h:76
Weighting scheme API.
Within-document-frequency of the current term in the current document.
Definition: weight.h:54
double P1
Constants for a given term in a given query.
Definition: weight.h:1213
Upper bound on wdf.
Definition: weight.h:74
double log2(double x)
Definition: log2.h:31
Xapian::doclength get_average_length() const
The average length of a document in the collection.
Definition: weight.h:380
std::string serialise_double(double v)
Serialise a double to a string.
double cl
Set by init() to (param_c * get_average_length())
Definition: weight.h:1216
double get_maxpart() const
Return an upper bound on what get_sumpart() can return for any document.
Definition: pl2weight.cc:170
PL2Weight * unserialise(const std::string &serialised) const
Unserialise parameters.
Definition: pl2weight.cc:144
Number of documents in the collection.
Definition: weight.h:40
Defines a log2() function to find the logarithm to base 2 if not already defined in the library...
void need_stat(stat_flags flag)
Tell Xapian that your subclass will want a particular statistic.
Definition: weight.h:94
double get_sumextra(Xapian::termcount doclen, Xapian::termcount uniqterms) const
Calculate the term-independent weight component for a document.
Definition: pl2weight.cc:176
Xapian::termcount get_wdf_upper_bound() const
An upper bound on the wdf of this term in the shard.
Definition: weight.h:419