unicode.h

Go to the documentation of this file.

/* Copyright (C) 2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2019 Olly Betts
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 */

#ifndef XAPIAN_INCLUDED_UNICODE_H
#define XAPIAN_INCLUDED_UNICODE_H

#if !defined XAPIAN_IN_XAPIAN_H && !defined XAPIAN_LIB_BUILD
# error Never use <xapian/unicode.h> directly; include <xapian.h> instead.
#endif

#include <xapian/attributes.h>
#include <xapian/visibility.h>

#include <string>

namespace Xapian {

class XAPIAN_VISIBILITY_DEFAULT Utf8Iterator {
    const unsigned char* p;
    const unsigned char* end;
    mutable unsigned seqlen;

    bool XAPIAN_NOTHROW(calculate_sequence_length() const);

    unsigned get_char() const;

    Utf8Iterator(const unsigned char* p_,
                 const unsigned char* end_,
                 unsigned seqlen_)
        : p(p_), end(end_), seqlen(seqlen_) { }

  public:
    const char* raw() const {
        return reinterpret_cast<const char*>(p ? p : end);
    }

    size_t left() const { return p ? end - p : 0; }

    void assign(const char* p_, size_t len) {
        if (len) {
            p = reinterpret_cast<const unsigned char*>(p_);
            end = p + len;
            seqlen = 0;
        } else {
            p = NULL;
        }
    }

    void assign(const std::string& s) { assign(s.data(), s.size()); }

    explicit Utf8Iterator(const char* p_);

    Utf8Iterator(const char* p_, size_t len) { assign(p_, len); }

    Utf8Iterator(const std::string& s) { assign(s.data(), s.size()); }

    XAPIAN_NOTHROW(Utf8Iterator())
        : p(NULL), end(0), seqlen(0) { }

    unsigned XAPIAN_NOTHROW(operator*() const) XAPIAN_PURE_FUNCTION;

    unsigned XAPIAN_NOTHROW(strict_deref() const) XAPIAN_PURE_FUNCTION;

    Utf8Iterator operator++(int) {
        // If we've not calculated seqlen yet, do so.
        if (seqlen == 0) calculate_sequence_length();
        const unsigned char* old_p = p;
        unsigned old_seqlen = seqlen;
        p += seqlen;
        if (p == end) p = NULL;
        seqlen = 0;
        return Utf8Iterator(old_p, end, old_seqlen);
    }

    Utf8Iterator& operator++() {
        if (seqlen == 0) calculate_sequence_length();
        p += seqlen;
        if (p == end) p = NULL;
        seqlen = 0;
        return *this;
    }

    bool XAPIAN_NOTHROW(operator==(const Utf8Iterator& other) const) {
        return p == other.p;
    }

    bool XAPIAN_NOTHROW(operator!=(const Utf8Iterator& other) const) {
        return p != other.p;
    }


    typedef std::input_iterator_tag iterator_category;
    typedef unsigned value_type;
    typedef size_t difference_type;
    typedef const unsigned* pointer;
    typedef const unsigned& reference;
};

namespace Unicode {

typedef enum {
    UNASSIGNED,                         
    UPPERCASE_LETTER,                   
    LOWERCASE_LETTER,                   
    TITLECASE_LETTER,                   
    MODIFIER_LETTER,                    
    OTHER_LETTER,                       
    NON_SPACING_MARK,                   
    ENCLOSING_MARK,                     
    COMBINING_SPACING_MARK,             
    DECIMAL_DIGIT_NUMBER,               
    LETTER_NUMBER,                      
    OTHER_NUMBER,                       
    SPACE_SEPARATOR,                    
    LINE_SEPARATOR,                     
    PARAGRAPH_SEPARATOR,                
    CONTROL,                            
    FORMAT,                             
    PRIVATE_USE,                        
    SURROGATE,                          
    CONNECTOR_PUNCTUATION,              
    DASH_PUNCTUATION,                   
    OPEN_PUNCTUATION,                   
    CLOSE_PUNCTUATION,                  
    INITIAL_QUOTE_PUNCTUATION,          
    FINAL_QUOTE_PUNCTUATION,            
    OTHER_PUNCTUATION,                  
    MATH_SYMBOL,                        
    CURRENCY_SYMBOL,                    
    MODIFIER_SYMBOL,                    
    OTHER_SYMBOL                        
} category;

namespace Internal {
    XAPIAN_VISIBILITY_DEFAULT
    int XAPIAN_NOTHROW(get_character_info(unsigned ch)) XAPIAN_CONST_FUNCTION;

    inline int get_case_type(int info) { return ((info & 0xe0) >> 5); }

    inline category get_category(int info) {
        return static_cast<category>(info & 0x1f);
    }

    inline int get_delta(int info) {
        /* It's implementation defined if sign extension happens when right
         * shifting a signed int, although in practice sign extension is what
         * most compilers implement.
         *
         * Some compilers are smart enough to spot common idioms for sign
         * extension, but not all (e.g. GCC < 7 doesn't spot the one used
         * below), so check what the implementation-defined behaviour is with
         * a constant conditional which should get optimised away.
         *
         * We use the ternary operator here to avoid various compiler
         * warnings which writing this as an `if` results in.
         */
        return ((-1 >> 1) == -1 ?
                // Right shift sign-extends.
                info >> 8 :
                // Right shift shifts in zeros so bitwise-not before and after
                // the shift for negative values.
                (info >= 0) ? (info >> 8) : (~(~info >> 8)));
    }
}

XAPIAN_VISIBILITY_DEFAULT
unsigned nonascii_to_utf8(unsigned ch, char* buf);

inline unsigned to_utf8(unsigned ch, char* buf) {
    if (ch < 128) {
        *buf = static_cast<unsigned char>(ch);
        return 1;
    }
    return Xapian::Unicode::nonascii_to_utf8(ch, buf);
}

inline void append_utf8(std::string& s, unsigned ch) {
    char buf[4];
    s.append(buf, to_utf8(ch, buf));
}

inline category get_category(unsigned ch) {
    return Internal::get_category(Internal::get_character_info(ch));
}

inline bool is_wordchar(unsigned ch) {
    const unsigned int WORDCHAR_MASK =
            (1 << Xapian::Unicode::UPPERCASE_LETTER) |
            (1 << Xapian::Unicode::LOWERCASE_LETTER) |
            (1 << Xapian::Unicode::TITLECASE_LETTER) |
            (1 << Xapian::Unicode::MODIFIER_LETTER) |
            (1 << Xapian::Unicode::OTHER_LETTER) |
            (1 << Xapian::Unicode::NON_SPACING_MARK) |
            (1 << Xapian::Unicode::ENCLOSING_MARK) |
            (1 << Xapian::Unicode::COMBINING_SPACING_MARK) |
            (1 << Xapian::Unicode::DECIMAL_DIGIT_NUMBER) |
            (1 << Xapian::Unicode::LETTER_NUMBER) |
            (1 << Xapian::Unicode::OTHER_NUMBER) |
            (1 << Xapian::Unicode::CONNECTOR_PUNCTUATION);
    return ((WORDCHAR_MASK >> get_category(ch)) & 1);
}

inline bool is_whitespace(unsigned ch) {
    const unsigned int WHITESPACE_MASK =
            (1 << Xapian::Unicode::CONTROL) | // For TAB, CR, LF, FF.
            (1 << Xapian::Unicode::SPACE_SEPARATOR) |
            (1 << Xapian::Unicode::LINE_SEPARATOR) |
            (1 << Xapian::Unicode::PARAGRAPH_SEPARATOR);
    return ((WHITESPACE_MASK >> get_category(ch)) & 1);
}

inline bool is_currency(unsigned ch) {
    return (get_category(ch) == Xapian::Unicode::CURRENCY_SYMBOL);
}

inline unsigned tolower(unsigned ch) {
    int info = Xapian::Unicode::Internal::get_character_info(ch);
    if (!(Internal::get_case_type(info) & 2))
        return ch;
    return ch + Internal::get_delta(info);
}

inline unsigned toupper(unsigned ch) {
    int info = Xapian::Unicode::Internal::get_character_info(ch);
    if (!(Internal::get_case_type(info) & 4))
        return ch;
    return ch - Internal::get_delta(info);
}

inline std::string
tolower(const std::string& term)
{
    std::string result;
    result.reserve(term.size());
    for (Utf8Iterator i(term); i != Utf8Iterator(); ++i) {
        append_utf8(result, tolower(*i));
    }
    return result;
}

inline std::string
toupper(const std::string& term)
{
    std::string result;
    result.reserve(term.size());
    for (Utf8Iterator i(term); i != Utf8Iterator(); ++i) {
        append_utf8(result, toupper(*i));
    }
    return result;
}

}

}

#endif // XAPIAN_INCLUDED_UNICODE_H