xapian-core  1.4.20
cjk-tokenizer.h
Go to the documentation of this file.
1 
4 /* Copyright (c) 2007, 2008 Yung-chung Lin (henearkrxern@gmail.com)
5  * Copyright (c) 2011 Richard Boulton (richard@tartarus.org)
6  * Copyright (c) 2011 Brandon Schaefer (brandontschaefer@gmail.com)
7  * Copyright (c) 2011,2019 Olly Betts
8  *
9  * Permission is hereby granted, free of charge, to any person obtaining a copy
10  * of this software and associated documentation files (the "Software"), to deal
11  * deal in the Software without restriction, including without limitation the
12  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
13  * sell copies of the Software, and to permit persons to whom the Software is
14  * furnished to do so, subject to the following conditions:
15  *
16  * The above copyright notice and this permission notice shall be included in
17  * all copies or substantial portions of the Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25  * IN THE SOFTWARE.
26  */
27 
28 #ifndef XAPIAN_INCLUDED_CJK_TOKENIZER_H
29 #define XAPIAN_INCLUDED_CJK_TOKENIZER_H
30 
31 #ifndef PACKAGE
32 # error config.h must be included first in each C++ source file
33 #endif
34 
35 #include "xapian/unicode.h"
36 
37 #include <string>
38 
39 namespace CJK {
40 
47 bool is_cjk_enabled();
48 
49 bool codepoint_is_cjk(unsigned codepoint);
50 
52 
53 }
54 
58 
63  unsigned offset = 0;
64 
65  std::string current_token;
66 
68  void init();
69 
70  public:
71  explicit CJKTokenIterator(const std::string& s) : it(s) {
72  init();
73  }
74 
75  explicit CJKTokenIterator(const Xapian::Utf8Iterator& it_) : it(it_) {
76  init();
77  }
78 
80 
81  const std::string& operator*() const {
82  return current_token;
83  }
84 
85  CJKTokenIterator& operator++();
86 
88  bool unigram() const { return offset == 0; }
89 
90  const Xapian::Utf8Iterator& get_utf8iterator() const { return it; }
91 
92  bool operator==(const CJKTokenIterator& other) const {
93  // We only really care about comparisons where one or other is an end
94  // iterator.
95  return current_token.empty() && other.current_token.empty();
96  }
97 
98  bool operator!=(const CJKTokenIterator& other) const {
99  return !(*this == other);
100  }
101 };
102 
103 #endif // XAPIAN_INCLUDED_CJK_TOKENIZER_H
Unicode and UTF-8 related classes and functions.
const std::string & operator*() const
Definition: cjk-tokenizer.h:81
bool is_cjk_enabled()
Should we use the CJK n-gram code?
bool unigram() const
Is this a unigram?
Definition: cjk-tokenizer.h:88
CJKTokenIterator(const Xapian::Utf8Iterator &it_)
Definition: cjk-tokenizer.h:75
bool operator!=(const CJKTokenIterator &other) const
Definition: cjk-tokenizer.h:98
bool codepoint_is_cjk(unsigned codepoint)
Xapian::Utf8Iterator it
Definition: cjk-tokenizer.h:57
Iterator returning unigrams and bigrams.
Definition: cjk-tokenizer.h:56
CJKTokenIterator(const std::string &s)
Definition: cjk-tokenizer.h:71
An iterator which returns Unicode character values from a UTF-8 encoded string.
Definition: unicode.h:38
bool operator==(const CJKTokenIterator &other) const
Definition: cjk-tokenizer.h:92
const Xapian::Utf8Iterator & get_utf8iterator() const
Definition: cjk-tokenizer.h:90
void get_cjk(Xapian::Utf8Iterator &it)
std::string current_token
Definition: cjk-tokenizer.h:65