xapian-core  1.4.19
termgenerator_internal.cc
Go to the documentation of this file.
1 
4 /* Copyright (C) 2007,2010,2011,2012,2015,2016,2017,2018,2019,2020 Olly Betts
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <config.h>
22 
23 #include "termgenerator_internal.h"
24 
25 #include "api/omenquireinternal.h"
26 #include "api/queryinternal.h"
27 
28 #include <xapian/document.h>
29 #include <xapian/queryparser.h>
30 #include <xapian/stem.h>
31 #include <xapian/unicode.h>
32 
33 #include "stringutils.h"
34 
35 #include <algorithm>
36 #include <cmath>
37 #include <deque>
38 #include <limits>
39 #include <list>
40 #include <string>
41 #include <unordered_map>
42 #include <vector>
43 
44 #include "cjk-tokenizer.h"
45 
46 using namespace std;
47 
48 namespace Xapian {
49 
50 static inline bool
51 U_isupper(unsigned ch)
52 {
53  return ch < 128 && C_isupper(static_cast<unsigned char>(ch));
54 }
55 
56 static inline unsigned
57 check_wordchar(unsigned ch)
58 {
59  if (Unicode::is_wordchar(ch)) return Unicode::tolower(ch);
60  return 0;
61 }
62 
63 static inline bool
64 should_stem(const std::string & term)
65 {
66  const unsigned int SHOULD_STEM_MASK =
70  (1 << Unicode::OTHER_LETTER);
71  Utf8Iterator u(term);
72  return ((SHOULD_STEM_MASK >> Unicode::get_category(*u)) & 1);
73 }
74 
78 static const unsigned UNICODE_IGNORE = numeric_limits<unsigned>::max();
79 
80 static inline unsigned
81 check_infix(unsigned ch)
82 {
83  if (ch == '\'' || ch == '&' || ch == 0xb7 || ch == 0x5f4 || ch == 0x2027) {
84  // Unicode includes all these except '&' in its word boundary rules,
85  // as well as 0x2019 (which we handle below) and ':' (for Swedish
86  // apparently, but we ignore this for now as it's problematic in
87  // real world cases).
88  return ch;
89  }
90  // 0x2019 is Unicode apostrophe and single closing quote.
91  // 0x201b is Unicode single opening quote with the tail rising.
92  if (ch == 0x2019 || ch == 0x201b) return '\'';
93  if (ch >= 0x200b && (ch <= 0x200d || ch == 0x2060 || ch == 0xfeff))
94  return UNICODE_IGNORE;
95  return 0;
96 }
97 
98 static inline unsigned
99 check_infix_digit(unsigned ch)
100 {
101  // This list of characters comes from Unicode's word identifying algorithm.
102  switch (ch) {
103  case ',':
104  case '.':
105  case ';':
106  case 0x037e: // GREEK QUESTION MARK
107  case 0x0589: // ARMENIAN FULL STOP
108  case 0x060D: // ARABIC DATE SEPARATOR
109  case 0x07F8: // NKO COMMA
110  case 0x2044: // FRACTION SLASH
111  case 0xFE10: // PRESENTATION FORM FOR VERTICAL COMMA
112  case 0xFE13: // PRESENTATION FORM FOR VERTICAL COLON
113  case 0xFE14: // PRESENTATION FORM FOR VERTICAL SEMICOLON
114  return ch;
115  }
116  if (ch >= 0x200b && (ch <= 0x200d || ch == 0x2060 || ch == 0xfeff))
117  return UNICODE_IGNORE;
118  return 0;
119 }
120 
121 static inline bool
122 is_digit(unsigned ch) {
124 }
125 
126 static inline unsigned
127 check_suffix(unsigned ch)
128 {
129  if (ch == '+' || ch == '#') return ch;
130  // FIXME: what about '-'?
131  return 0;
132 }
133 
140 template<typename ACTION>
141 static void
142 parse_terms(Utf8Iterator itor, bool cjk_ngram, bool with_positions, ACTION action)
143 {
144  while (true) {
145  // Advance to the start of the next term.
146  unsigned ch;
147  while (true) {
148  if (itor == Utf8Iterator()) return;
149  ch = check_wordchar(*itor);
150  if (ch) break;
151  ++itor;
152  }
153 
154  string term;
155  // Look for initials separated by '.' (e.g. P.T.O., U.N.C.L.E).
156  // Don't worry if there's a trailing '.' or not.
157  if (U_isupper(*itor)) {
158  const Utf8Iterator end;
159  Utf8Iterator p = itor;
160  do {
162  } while (p != end && *p == '.' && ++p != end && U_isupper(*p));
163  // One letter does not make an acronym! If we handled a single
164  // uppercase letter here, we wouldn't catch M&S below.
165  if (term.size() > 1) {
166  // Check there's not a (lower case) letter or digit
167  // immediately after it.
168  if (p == end || !Unicode::is_wordchar(*p)) {
169  itor = p;
170  goto endofterm;
171  }
172  }
173  term.resize(0);
174  }
175 
176  while (true) {
177  if (cjk_ngram &&
178  CJK::codepoint_is_cjk(*itor) &&
179  Unicode::is_wordchar(*itor)) {
180  CJKTokenIterator tk(itor);
181  while (tk != CJKTokenIterator()) {
182  const string& cjk_token = *tk;
183  if (!action(cjk_token, with_positions && tk.unigram(),
184  tk.get_utf8iterator()))
185  return;
186  ++tk;
187  }
188  // Update itor to end of CJK text span.
189  itor = tk.get_utf8iterator();
190  while (true) {
191  if (itor == Utf8Iterator()) return;
192  ch = check_wordchar(*itor);
193  if (ch) break;
194  ++itor;
195  }
196  continue;
197  }
198  unsigned prevch;
199  do {
200  Unicode::append_utf8(term, ch);
201  prevch = ch;
202  if (++itor == Utf8Iterator() ||
203  (cjk_ngram && CJK::codepoint_is_cjk(*itor)))
204  goto endofterm;
205  ch = check_wordchar(*itor);
206  } while (ch);
207 
208  Utf8Iterator next(itor);
209  ++next;
210  if (next == Utf8Iterator()) break;
211  unsigned nextch = check_wordchar(*next);
212  if (!nextch) break;
213  unsigned infix_ch = *itor;
214  if (is_digit(prevch) && is_digit(*next)) {
215  infix_ch = check_infix_digit(infix_ch);
216  } else {
217  // Handle things like '&' in AT&T, apostrophes, etc.
218  infix_ch = check_infix(infix_ch);
219  }
220  if (!infix_ch) break;
221  if (infix_ch != UNICODE_IGNORE)
222  Unicode::append_utf8(term, infix_ch);
223  ch = nextch;
224  itor = next;
225  }
226 
227  {
228  size_t len = term.size();
229  unsigned count = 0;
230  while ((ch = check_suffix(*itor))) {
231  if (++count > 3) {
232  term.resize(len);
233  break;
234  }
235  Unicode::append_utf8(term, ch);
236  if (++itor == Utf8Iterator()) goto endofterm;
237  }
238  // Don't index fish+chips as fish+ chips.
239  if (Unicode::is_wordchar(*itor))
240  term.resize(len);
241  }
242 
243 endofterm:
244  if (!action(term, with_positions, itor))
245  return;
246  }
247 }
248 
249 void
250 TermGenerator::Internal::index_text(Utf8Iterator itor, termcount wdf_inc,
251  const string & prefix, bool with_positions)
252 {
253  bool cjk_ngram = (flags & FLAG_CJK_NGRAM) || CJK::is_cjk_enabled();
254 
255  stop_strategy current_stop_mode;
256  if (!stopper.get()) {
257  current_stop_mode = TermGenerator::STOP_NONE;
258  } else {
259  current_stop_mode = stop_mode;
260  }
261 
262  parse_terms(itor, cjk_ngram, with_positions,
263  [=](const string & term, bool positional, const Utf8Iterator &) {
264  if (term.size() > max_word_length) return true;
265 
266  if (current_stop_mode == TermGenerator::STOP_ALL && (*stopper)(term))
267  return true;
268 
269  if (strategy == TermGenerator::STEM_SOME ||
270  strategy == TermGenerator::STEM_NONE ||
271  strategy == TermGenerator::STEM_SOME_FULL_POS) {
272  if (positional) {
273  doc.add_posting(prefix + term, ++cur_pos, wdf_inc);
274  } else {
275  doc.add_term(prefix + term, wdf_inc);
276  }
277  }
278 
279  // MSVC seems to need "this->" on member variables in this
280  // situation.
281  if ((this->flags & FLAG_SPELLING) && prefix.empty())
282  db.add_spelling(term);
283 
284  if (strategy == TermGenerator::STEM_NONE || stemmer.is_none())
285  return true;
286 
287  if (strategy == TermGenerator::STEM_SOME ||
288  strategy == TermGenerator::STEM_SOME_FULL_POS) {
289  if (current_stop_mode == TermGenerator::STOP_STEMMED &&
290  (*stopper)(term))
291  return true;
292 
293  // Note, this uses the lowercased term, but that's OK as we
294  // only want to avoid stemming terms starting with a digit.
295  if (!should_stem(term)) return true;
296  }
297 
298  // Add stemmed form without positional information.
299  const string& stem = stemmer(term);
300  if (rare(stem.empty())) return true;
301  string stemmed_term;
302  if (strategy != TermGenerator::STEM_ALL) {
303  stemmed_term += "Z";
304  }
305  stemmed_term += prefix;
306  stemmed_term += stem;
307  if (strategy != TermGenerator::STEM_SOME && with_positions) {
308  if (strategy != TermGenerator::STEM_SOME_FULL_POS) ++cur_pos;
309  doc.add_posting(stemmed_term, cur_pos, wdf_inc);
310  } else {
311  doc.add_term(stemmed_term, wdf_inc);
312  }
313  return true;
314  });
315 }
316 
317 struct Sniplet {
318  double* relevance;
319 
320  size_t term_end;
321 
322  size_t highlight;
323 
324  Sniplet(double* r, size_t t, size_t h)
325  : relevance(r), term_end(t), highlight(h) { }
326 };
327 
328 class SnipPipe {
329  deque<Sniplet> pipe;
330  deque<Sniplet> best_pipe;
331 
332  // Requested length for snippet.
333  size_t length;
334 
335  // Position in text of start of current pipe contents.
336  size_t begin = 0;
337 
338  // Rolling sum of the current pipe contents.
339  double sum = 0;
340 
341  size_t phrase_len = 0;
342 
343  public:
344  size_t best_begin = 0;
345 
346  size_t best_end = 0;
347 
348  double best_sum = 0;
349 
350  // Add one to length to allow for inter-word space.
351  // FIXME: We ought to correctly allow for multiple spaces.
352  explicit SnipPipe(size_t length_) : length(length_ + 1) { }
353 
354  bool pump(double* r, size_t t, size_t h, unsigned flags);
355 
356  void done();
357 
358  bool drain(const string & input,
359  const string & hi_start,
360  const string & hi_end,
361  const string & omit,
362  string & output);
363 };
364 
365 #define DECAY 2.0
366 
367 inline bool
368 SnipPipe::pump(double* r, size_t t, size_t h, unsigned flags)
369 {
370  if (h > 1) {
371  if (pipe.size() >= h - 1) {
372  // The final term of a phrase is entering the window. Peg the
373  // phrase's relevance onto the first term of the phrase, so it'll
374  // be removed from `sum` when the phrase starts to leave the
375  // window.
376  auto & phrase_start = pipe[pipe.size() - (h - 1)];
377  if (phrase_start.relevance) {
378  *phrase_start.relevance *= DECAY;
379  sum -= *phrase_start.relevance;
380  }
381  sum += *r;
382  phrase_start.relevance = r;
383  phrase_start.highlight = h;
384  *r /= DECAY;
385  }
386  r = NULL;
387  h = 0;
388  }
389  pipe.emplace_back(r, t, h);
390  if (r) {
391  sum += *r;
392  *r /= DECAY;
393  }
394 
395  // If necessary, discard words from the start of the pipe until it has the
396  // desired length.
397  // FIXME: Also shrink the window past words with relevance < 0?
398  while (t - begin > length /* || pipe.front().relevance < 0.0 */) {
399  const Sniplet& word = pipe.front();
400  if (word.relevance) {
401  *word.relevance *= DECAY;
402  sum -= *word.relevance;
403  }
404  begin = word.term_end;
405  if (best_end >= begin)
406  best_pipe.push_back(word);
407  pipe.pop_front();
408  // E.g. can happen if the current term is longer than the requested
409  // length!
410  if (rare(pipe.empty())) break;
411  }
412 
413  // Using > here doesn't work well, as we don't extend a snippet over terms
414  // with 0 weight.
415  if (sum >= best_sum) {
416  // Discard any part of `best_pipe` which is before `begin`.
417  if (begin >= best_end) {
418  best_pipe.clear();
419  } else {
420  while (!best_pipe.empty() &&
421  best_pipe.front().term_end <= begin) {
422  best_pipe.pop_front();
423  }
424  }
425  best_sum = sum;
426  best_begin = begin;
427  best_end = t;
428  } else if ((flags & Xapian::MSet::SNIPPET_EXHAUSTIVE) == 0) {
429  if (best_sum > 0 && best_end < begin) {
430  // We found something, and we aren't still looking near it.
431  // FIXME: Benchmark this and adjust if necessary.
432  return false;
433  }
434  }
435  return true;
436 }
437 
438 inline void
439 SnipPipe::done()
440 {
441  // Discard any part of `pipe` which is after `best_end`.
442  if (begin >= best_end) {
443  pipe.clear();
444  } else {
445  // We should never empty the pipe (as that case should be handled
446  // above).
447  while (rare(!pipe.empty()) &&
448  pipe.back().term_end > best_end) {
449  pipe.pop_back();
450  }
451  }
452 }
453 
454 // Check if a non-word character is should be included at the start of the
455 // snippet. We want to include certain leading non-word characters, but not
456 // others.
457 static inline bool
459  if (Unicode::is_currency(ch) ||
462  return true;
463  }
464  switch (ch) {
465  case '"':
466  case '#':
467  case '%':
468  case '&':
469  case '\'':
470  case '+':
471  case '-':
472  case '/':
473  case '<':
474  case '@':
475  case '\\':
476  case '`':
477  case '~':
478  case 0x00A1: // INVERTED EXCLAMATION MARK
479  case 0x00A7: // SECTION SIGN
480  case 0x00BF: // INVERTED QUESTION MARK
481  return true;
482  }
483  return false;
484 }
485 
486 // Check if a non-word character is should be included at the end of the
487 // snippet. We want to include certain trailing non-word characters, but not
488 // others.
489 static inline bool
491  if (Unicode::is_currency(ch) ||
494  return true;
495  }
496  switch (ch) {
497  case '"':
498  case '%':
499  case '\'':
500  case '+':
501  case '-':
502  case '/':
503  case '>':
504  case '@':
505  case '\\':
506  case '`':
507  case '~':
508  return true;
509  }
510  return false;
511 }
512 
513 static inline void
514 append_escaping_xml(const char* p, const char* end, string& output)
515 {
516  while (p != end) {
517  char ch = *p++;
518  switch (ch) {
519  case '&':
520  output += "&amp;";
521  break;
522  case '<':
523  output += "&lt;";
524  break;
525  case '>':
526  output += "&gt;";
527  break;
528  default:
529  output += ch;
530  }
531  }
532 }
533 
534 inline bool
535 SnipPipe::drain(const string & input,
536  const string & hi_start,
537  const string & hi_end,
538  const string & omit,
539  string & output)
540 {
541  if (best_pipe.empty() && !pipe.empty()) {
542  swap(best_pipe, pipe);
543  }
544 
545  if (best_pipe.empty()) {
546  size_t tail_len = input.size() - best_end;
547  if (tail_len == 0) return false;
548 
549  // See if this is the end of a sentence.
550  // FIXME: This is quite simplistic - look at the Unicode rules:
551  // https://unicode.org/reports/tr29/#Sentence_Boundaries
552  bool sentence_end = false;
553  Utf8Iterator i(input.data() + best_end, tail_len);
554  while (i != Utf8Iterator()) {
555  unsigned ch = *i;
556  if (sentence_end && Unicode::is_whitespace(ch)) break;
557 
558  // Allow "...", "!!", "!?!", etc...
559  sentence_end = (ch == '.' || ch == '?' || ch == '!');
560 
561  if (Unicode::is_wordchar(ch)) break;
562  ++i;
563  }
564 
565  if (sentence_end) {
566  // Include end of sentence punctuation.
567  append_escaping_xml(input.data() + best_end, i.raw(), output);
568  return false;
569  }
570 
571  // Include trailing punctuation which includes meaning or context.
572  i.assign(input.data() + best_end, tail_len);
573  int trailing_punc = 0;
574  while (i != Utf8Iterator() && snippet_check_trailing_nonwordchar(*i)) {
575  // But limit how much trailing punctuation we include.
576  if (++trailing_punc > 4) {
577  trailing_punc = 0;
578  break;
579  }
580  ++i;
581  }
582  if (trailing_punc) {
583  append_escaping_xml(input.data() + best_end, i.raw(), output);
584  if (i == Utf8Iterator()) return false;
585  }
586 
587  // Append "..." or equivalent as this doesn't seem to be the start
588  // of a sentence.
589  output += omit;
590 
591  return false;
592  }
593 
594  const Sniplet & word = best_pipe.front();
595 
596  if (output.empty()) {
597  // Start of the snippet.
598  enum { NO, PUNC, YES } sentence_boundary = (best_begin == 0) ? YES : NO;
599 
600  Utf8Iterator i(input.data() + best_begin, word.term_end - best_begin);
601  while (i != Utf8Iterator()) {
602  unsigned ch = *i;
603  switch (sentence_boundary) {
604  case NO:
605  if (ch == '.' || ch == '?' || ch == '!') {
606  sentence_boundary = PUNC;
607  }
608  break;
609  case PUNC:
610  if (Unicode::is_whitespace(ch)) {
611  sentence_boundary = YES;
612  } else if (ch == '.' || ch == '?' || ch == '!') {
613  // Allow "...", "!!", "!?!", etc...
614  } else {
615  sentence_boundary = NO;
616  }
617  break;
618  case YES:
619  break;
620  }
621 
622  // Start the snippet at the start of the first word, but include
623  // certain punctuation too.
624  if (Unicode::is_wordchar(ch)) {
625  // But limit how much leading punctuation we include.
626  size_t word_begin = i.raw() - input.data();
627  if (word_begin - best_begin > 4) {
628  best_begin = word_begin;
629  }
630  break;
631  }
632  ++i;
634  best_begin = i.raw() - input.data();
635  }
636  }
637 
638  // Add "..." or equivalent if this doesn't seem to be the start of a
639  // sentence.
640  if (sentence_boundary != YES) {
641  output += omit;
642  }
643  }
644 
645  if (word.highlight) {
646  // Don't include inter-word characters in the highlight.
647  Utf8Iterator i(input.data() + best_begin, input.size() - best_begin);
648  while (i != Utf8Iterator()) {
649  unsigned ch = *i;
650  if (Unicode::is_wordchar(ch)) {
651  append_escaping_xml(input.data() + best_begin, i.raw(), output);
652  best_begin = i.raw() - input.data();
653  break;
654  }
655  ++i;
656  }
657  }
658 
659  if (!phrase_len) {
660  phrase_len = word.highlight;
661  if (phrase_len) output += hi_start;
662  }
663 
664  const char* p = input.data();
665  append_escaping_xml(p + best_begin, p + word.term_end, output);
666  best_begin = word.term_end;
667 
668  if (phrase_len && --phrase_len == 0) output += hi_end;
669 
670  best_pipe.pop_front();
671  return true;
672 }
673 
674 static void
676  list<vector<string>> & exact_phrases,
677  unordered_map<string, double> & loose_terms,
678  list<string> & wildcards,
679  size_t & longest_phrase)
680 {
681  // FIXME: OP_NEAR, non-tight OP_PHRASE, OP_PHRASE with non-term subqueries
682  size_t n_subqs = query.get_num_subqueries();
683  Xapian::Query::op op = query.get_type();
684  if (op == query.LEAF_TERM) {
685  const Xapian::Internal::QueryTerm & qt =
686  *static_cast<const Xapian::Internal::QueryTerm *>(query.internal.get());
687  loose_terms.insert(make_pair(qt.get_term(), 0));
688  } else if (op == query.OP_WILDCARD) {
690  *static_cast<const Xapian::Internal::QueryWildcard *>(query.internal.get());
691  wildcards.push_back(qw.get_pattern());
692  } else if (op == query.OP_PHRASE) {
693  const Xapian::Internal::QueryPhrase & phrase =
694  *static_cast<const Xapian::Internal::QueryPhrase *>(query.internal.get());
695  if (phrase.get_window() == n_subqs) {
696  // Tight phrase.
697  for (size_t i = 0; i != n_subqs; ++i) {
698  if (query.get_subquery(i).get_type() != query.LEAF_TERM)
699  goto non_term_subquery;
700  }
701 
702  // Tight phrase of terms.
703  exact_phrases.push_back(vector<string>());
704  vector<string> & terms = exact_phrases.back();
705  terms.reserve(n_subqs);
706  for (size_t i = 0; i != n_subqs; ++i) {
707  Xapian::Query q = query.get_subquery(i);
708  const Xapian::Internal::QueryTerm & qt =
709  *static_cast<const Xapian::Internal::QueryTerm *>(q.internal.get());
710  terms.push_back(qt.get_term());
711  }
712  if (n_subqs > longest_phrase) longest_phrase = n_subqs;
713  return;
714  }
715  }
716 non_term_subquery:
717  for (size_t i = 0; i != n_subqs; ++i)
718  check_query(query.get_subquery(i), exact_phrases, loose_terms,
719  wildcards, longest_phrase);
720 }
721 
722 static double*
723 check_term(unordered_map<string, double> & loose_terms,
724  const Xapian::Weight::Internal * stats,
725  const string & term,
726  double max_tw)
727 {
728  auto it = loose_terms.find(term);
729  if (it == loose_terms.end()) return NULL;
730 
731  if (it->second == 0.0) {
732  double relevance;
733  if (!stats->get_termweight(term, relevance)) {
734  // FIXME: Assert?
735  loose_terms.erase(it);
736  return NULL;
737  }
738 
739  it->second = relevance + max_tw;
740  }
741  return &it->second;
742 }
743 
744 string
745 MSet::Internal::snippet(const string & text,
746  size_t length,
747  const Xapian::Stem & stemmer,
748  unsigned flags,
749  const string & hi_start,
750  const string & hi_end,
751  const string & omit) const
752 {
753  if (hi_start.empty() && hi_end.empty() && text.size() <= length) {
754  // Too easy!
755  return text;
756  }
757 
758  bool cjk_ngram = (flags & MSet::SNIPPET_CJK_NGRAM);
759  if (!cjk_ngram) {
760  cjk_ngram = CJK::is_cjk_enabled();
761  }
762 
763  size_t term_start = 0;
764  double min_tw = 0, max_tw = 0;
765  if (stats) stats->get_max_termweight(min_tw, max_tw);
766  if (max_tw == 0.0) {
767  max_tw = 1.0;
768  } else {
769  // Scale up by (1 + 1/64) so that highlighting works better for terms
770  // with termweight 0 (which happens for terms not in the database, and
771  // also with some weighting schemes for terms which occur in almost all
772  // documents.
773  max_tw *= 1.015625;
774  }
775 
777  if (enquire.get()) {
778  query = enquire->query;
779  }
780  SnipPipe snip(length);
781 
782  list<vector<string>> exact_phrases;
783  unordered_map<string, double> loose_terms;
784  list<string> wildcards;
785  size_t longest_phrase = 0;
786  check_query(query, exact_phrases, loose_terms,
787  wildcards, longest_phrase);
788 
789  vector<double> exact_phrases_relevance;
790  exact_phrases_relevance.reserve(exact_phrases.size());
791  for (auto&& terms : exact_phrases) {
792  // FIXME: What relevance to use?
793  exact_phrases_relevance.push_back(max_tw * terms.size());
794  }
795 
796  vector<double> wildcards_relevance;
797  wildcards_relevance.reserve(exact_phrases.size());
798  for (auto&& pattern : wildcards) {
799  // FIXME: What relevance to use?
800  (void)pattern;
801  wildcards_relevance.push_back(max_tw + min_tw);
802  }
803 
804  // Background relevance is the same for a given MSet, so cache it
805  // between calls to MSet::snippet() on the same object.
806  unordered_map<string, double>& background = snippet_bg_relevance;
807 
808  vector<string> phrase;
809  if (longest_phrase) phrase.resize(longest_phrase - 1);
810  size_t phrase_next = 0;
811  bool matchfound = false;
812  parse_terms(Utf8Iterator(text), cjk_ngram, true,
813  [&](const string & term, bool positional, const Utf8Iterator & it) {
814  // FIXME: Don't hardcode this here.
815  const size_t max_word_length = 64;
816 
817  if (!positional) return true;
818  if (term.size() > max_word_length) return true;
819 
820  // We get segments with any "inter-word" characters in front of
821  // each word, e.g.:
822  // [The][ cat][ sat][ on][ the][ mat]
823  size_t term_end = text.size() - it.left();
824 
825  double* relevance = 0;
826  size_t highlight = 0;
827  if (stats) {
828  size_t i = 0;
829  for (auto&& terms : exact_phrases) {
830  if (term == terms.back()) {
831  size_t n = terms.size() - 1;
832  bool match = true;
833  while (n--) {
834  if (terms[n] != phrase[(n + phrase_next) % (longest_phrase - 1)]) {
835  match = false;
836  break;
837  }
838  }
839  if (match) {
840  // FIXME: Sort phrases, highest score first!
841  relevance = &exact_phrases_relevance[i];
842  highlight = terms.size();
843  goto relevance_done;
844  }
845  }
846  ++i;
847  }
848 
849  relevance = check_term(loose_terms, stats, term, max_tw);
850  if (relevance) {
851  // Matched unstemmed term.
852  highlight = 1;
853  goto relevance_done;
854  }
855 
856  string stem = "Z";
857  stem += stemmer(term);
858  relevance = check_term(loose_terms, stats, stem, max_tw);
859  if (relevance) {
860  // Matched stemmed term.
861  highlight = 1;
862  goto relevance_done;
863  }
864 
865  // Check wildcards.
866  // FIXME: Sort wildcards, shortest pattern first or something?
867  i = 0;
868  for (auto&& pattern : wildcards) {
869  if (startswith(term, pattern)) {
870  relevance = &wildcards_relevance[i];
871  highlight = 1;
872  goto relevance_done;
873  }
874  ++i;
875  }
876 
878  // Background document model.
879  auto bgit = background.find(term);
880  if (bgit == background.end()) bgit = background.find(stem);
881  if (bgit == background.end()) {
882  Xapian::doccount tf = enquire->db.get_termfreq(term);
883  if (!tf) {
884  tf = enquire->db.get_termfreq(stem);
885  } else {
886  stem = term;
887  }
888  double r = 0.0;
889  if (tf) {
890  // Add one to avoid log(0) when a term indexes all
891  // documents.
892  Xapian::doccount num_docs = stats->collection_size + 1;
893  r = max_tw * log((num_docs - tf) / double(tf));
894  r /= (length + 1) * log(double(num_docs));
895 #if 0
896  if (r <= 0) {
897  Utf8Iterator i(text.data() + term_start, text.data() + term_end);
898  while (i != Utf8Iterator()) {
900  r = max_tw * 0.05;
901  }
902  }
903  }
904 #endif
905  }
906  bgit = background.emplace(make_pair(stem, r)).first;
907  }
908  relevance = &bgit->second;
909  }
910  } else {
911 #if 0
912  // In the absence of weight information, assume longer terms
913  // are more relevant, and that unstemmed matches are a bit more
914  // relevant than stemmed matches.
915  if (queryterms.find(term) != queryterms.end()) {
916  relevance = term.size() * 3;
917  } else {
918  string stem = "Z";
919  stem += stemmer(term);
920  if (queryterms.find(stem) != queryterms.end()) {
921  relevance = term.size() * 2;
922  }
923  }
924 #endif
925  }
926 
927  // FIXME: Allow Enquire without a DB set or an empty MSet() to be
928  // used if you don't want the collection model?
929 
930 #if 0
931  // FIXME: Punctuation should somehow be included in the model, but this
932  // approach is problematic - we don't want the first word of a sentence
933  // to be favoured when it's at the end of the window.
934 
935  // Give first word in each sentence a relevance boost.
936  if (term_start == 0) {
937  relevance += 10;
938  } else {
939  for (size_t i = term_start; i + term.size() < term_end; ++i) {
940  if (text[i] == '.' && Unicode::is_whitespace(text[i + 1])) {
941  relevance += 10;
942  break;
943  }
944  }
945  }
946 #endif
947 
948 relevance_done:
949  if (longest_phrase) {
950  phrase[phrase_next] = term;
951  phrase_next = (phrase_next + 1) % (longest_phrase - 1);
952  }
953 
954  if (highlight) matchfound = true;
955 
956  if (!snip.pump(relevance, term_end, highlight, flags)) return false;
957 
958  term_start = term_end;
959  return true;
960  });
961 
962  snip.done();
963 
964  // Put together the snippet.
965  string result;
966  if (matchfound || (flags & SNIPPET_EMPTY_WITHOUT_MATCH) == 0) {
967  while (snip.drain(text, hi_start, hi_end, omit, result)) { }
968  }
969 
970  return result;
971 }
972 
973 }
Unicode and UTF-8 related classes and functions.
bool is_none() const
Return true if this is a no-op stemmer.
Definition: stem.h:166
The Xapian namespace contains public interfaces for the Xapian library.
Definition: compactor.cc:80
void append_utf8(std::string &s, unsigned ch)
Append the UTF-8 representation of a single Unicode character to a std::string.
Definition: unicode.h:332
Wildcard expansion.
Definition: query.h:255
Exhaustively evaluate candidate snippets in MSet::snippet().
Definition: mset.h:179
Letter, modifier (Lm)
Definition: unicode.h:225
const Query get_subquery(size_t n) const
Read a top level subquery.
Definition: query.cc:226
static unsigned check_suffix(unsigned ch)
const char * raw() const
Return the raw const char* pointer for the current position.
Definition: unicode.h:54
unsigned tolower(unsigned ch)
Convert a Unicode character to lowercase.
Definition: unicode.h:376
bool drain(const string &input, const string &hi_start, const string &hi_end, const string &omit, string &output)
Letter, other (Lo)
Definition: unicode.h:226
Xapian::Query internals.
static double * check_term(unordered_map< string, double > &loose_terms, const Xapian::Weight::Internal *stats, const string &term, double max_tw)
size_t left() const
Return the number of bytes left in the iterator&#39;s buffer.
Definition: unicode.h:59
bool is_digit(unsigned ch)
Class representing a stemming algorithm.
Definition: stem.h:62
bool U_isupper(unsigned ch)
op
Query operators.
Definition: query.h:78
bool is_currency(unsigned ch)
Test if a given Unicode character is a currency symbol.
Definition: unicode.h:371
Number, decimal digit (Nd)
Definition: unicode.h:230
Definition: header.h:63
bool is_cjk_enabled()
Should we use the CJK n-gram code?
bool unigram() const
Is this a unigram?
Definition: cjk-tokenizer.h:88
STL namespace.
deque< Sniplet > best_pipe
Xapian::Internal::intrusive_ptr< Internal > internal
Definition: query.h:49
static Xapian::Stem stemmer
Definition: stemtest.cc:41
#define DECAY
bool codepoint_is_cjk(unsigned codepoint)
#define rare(COND)
Definition: config.h:543
static void check_query(const Xapian::Query &query, list< vector< string >> &exact_phrases, unordered_map< string, double > &loose_terms, list< string > &wildcards, size_t &longest_phrase)
Letter, lowercase (Ll)
Definition: unicode.h:223
static unsigned check_wordchar(unsigned ch)
TermGenerator class internals.
unsigned XAPIAN_TERMCOUNT_BASE_TYPE termcount
A counts of terms.
Definition: types.h:72
const std::string & get_pattern() const
const std::string & get_term() const
Definition: queryinternal.h:55
stop_strategy
Stopper strategies, for use with set_stopper_strategy().
Letter, titlecase (Lt)
Definition: unicode.h:224
static void append_escaping_xml(const char *p, const char *end, string &output)
Match only documents where all subqueries match near and in order.
Definition: query.h:152
const unsigned UNICODE_IGNORE
Value representing "ignore this" when returned by check_infix() or check_infix_digit().
Class to hold statistics for a given collection.
Letter, uppercase (Lu)
Definition: unicode.h:222
Punctuation, close (Pe)
Definition: unicode.h:243
Punctuation, open (Ps)
Definition: unicode.h:242
Model the relevancy of non-query terms in MSet::snippet().
Definition: mset.h:172
Tokenise CJK text as n-grams.
static bool snippet_check_leading_nonwordchar(unsigned ch)
bool startswith(const std::string &s, char pfx)
Definition: stringutils.h:46
int flags
For backward compatibility with Xapian 1.2.
Definition: termgenerator.h:98
bool pump(double *r, size_t t, size_t h, unsigned flags)
bool get_termweight(const std::string &term, double &termweight) const
Get the termweight.
unsigned check_infix_digit(unsigned ch)
Iterator returning unigrams and bigrams.
Definition: cjk-tokenizer.h:56
static void parse_terms(Utf8Iterator itor, bool cjk_ngram, bool with_positions, ACTION action)
Templated framework for processing terms.
bool should_stem(const string &term)
An iterator which returns Unicode character values from a UTF-8 encoded string.
Definition: unicode.h:38
static Xapian::Query query(Xapian::Query::op op, const string &t1=string(), const string &t2=string(), const string &t3=string(), const string &t4=string(), const string &t5=string(), const string &t6=string(), const string &t7=string(), const string &t8=string(), const string &t9=string(), const string &t10=string())
Definition: api_anydb.cc:63
static bool snippet_check_trailing_nonwordchar(unsigned ch)
const Xapian::Utf8Iterator & get_utf8iterator() const
Definition: cjk-tokenizer.h:90
bool is_wordchar(unsigned ch)
Test if a given Unicode character is "word character".
Definition: unicode.h:343
size_t get_num_subqueries() const
Get the number of subqueries of the top level query.
Definition: query.cc:220
unsigned XAPIAN_DOCID_BASE_TYPE doccount
A count of documents.
Definition: types.h:38
Value returned by get_type() for a term.
Definition: query.h:266
Various handy helpers which std::string really should provide.
bool is_whitespace(unsigned ch)
Test if a given Unicode character is a whitespace character.
Definition: unicode.h:361
Punctuation, initial quote (Pi)
Definition: unicode.h:244
Sniplet(double *r, size_t t, size_t h)
op get_type() const
Get the type of the top level of the query.
Definition: query.cc:212
category get_category(int info)
Definition: unicode.h:271
Punctuation, final quote (Pf)
Definition: unicode.h:245
Class representing a query.
Definition: query.h:46
API for working with documents.
unsigned check_infix(unsigned ch)
stemming algorithms
parsing a user query string to build a Xapian::Query object