unicodehelpers.h

Go to the documentation of this file.

//-*- C++ -*-
/*
  Phrasehunter - index and query text corpora
  Copyright (C) 2006  Torsten Marek (shlomme@gmx.de) &
  Armin Schmidt (armin.sch@gmail.com)
  
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
  as published by the Free Software Foundation; either version 2
  of the License, or (at your option) any later version.
  
  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#ifndef UNICODEHELPERS_H
#define UNICODEHELPERS_H UNICODEHELPERS_H

#include <vector>
#include <unicode/ucnv.h>
#include <unicode/schriter.h>
#include <unicode/regex.h>
#include <boost/shared_ptr.hpp>
#include <boost/tokenizer.hpp>
#include <boost/shared_array.hpp>

namespace schma {

typedef boost::shared_ptr<UnicodeString> UnicodePtr;
typedef boost::shared_array<char> charArray;
typedef std::vector<UnicodePtr> UnicodeVector;

inline UnicodeVector splitString(const std::string& s)
{
    UnicodeVector result;
    
    boost::char_separator<char> sep(" ");
    boost::tokenizer<boost::char_separator<char> > tok(s, sep);
    
    for(boost::tokenizer<boost::char_separator<char> >::const_iterator it = tok.begin();
        it != tok.end(); ++it){
        result.push_back(UnicodePtr(new UnicodeString(it->c_str())));
    }
    return result;
}

/**
   \brief Class for converting utf8-encoded input into UnicodeString.
   Don't use directly. Call UTF8Converter() to get a pointer to a singleton
   _UTFConverter
*/
class _UTF8Converter
{
    UConverter* m_conv;
public:
    _UTF8Converter()
    {
        UErrorCode errcode = U_ZERO_ERROR;
        m_conv = ucnv_open("UTF8", &errcode);
        assert(errcode == U_ZERO_ERROR);
    }
    ~_UTF8Converter() 
    {
        ucnv_close(m_conv);
    }
    inline UConverter* converter() const 
    {
        return m_conv;
    }
};

//! Returns a pointer to a singleton _UTF8Converter object.
inline UConverter* UTF8Converter() 
{
    static _UTF8Converter u;
    return u.converter();
}

//! Convert the UnicodeString a UnicodePtr points to into a shared char array.
inline charArray toCharArray(UnicodePtr u)
{
    char* buffer = new char[u->length()*4];
    UErrorCode c = U_ZERO_ERROR;
    u->extract(buffer,u->length()*4, 0, c);
    return charArray(buffer);
}

//! Convert the UnicodeString a UnicodePtr points to into a std::string.
inline std::string toStdString(UnicodePtr u)
{
    char buffer[u->length()*4];
    UErrorCode c = U_ZERO_ERROR;
    u->extract(buffer,u->length()*4, 0, c);
    return std::string(buffer);
}

}

#endif

---