tokenizer.h

Go to the documentation of this file.

// -*- C++ -*-
/*
  Phrasehunter - index and query text corpora
  Copyright (C) 2006  Torsten Marek (shlomme@gmx.de) &
  Armin Schmidt (armin.sch@gmail.com)
  
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
  as published by the Free Software Foundation; either version 2
  of the License, or (at your option) any later version.
  
  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#ifndef TOKENIZER_H
#define TOKENIZER_H TOKENIZER_H

#include <string>
#include <fstream>
#include <iostream>
#include <stdexcept>
#include <boost/scoped_ptr.hpp>

#include <unicode/unistr.h>
#include <unicode/ucnv.h>
#include <unicode/normlzr.h>

#include "ph_types.h"
#include "phexception.h"
#include "support/unicodehelpers.h"

#define MAX_TOKEN_LEN 1024

namespace PhraseHunter {

class Input 
{
public:
    virtual ~Input() {};
    virtual bool hasMoreData() const = 0;
    virtual void getMoreData(std::string*) = 0;
};

class StringInput: public Input
{
    std::string m_inputstring;
    bool m_delivered;
    
public:
    StringInput(const std::string& inputstring): m_inputstring(inputstring), m_delivered(false)
    {}
    inline bool hasMoreData() const
    {
        return !m_delivered;
    }
    inline void getMoreData(std::string* s) 
    {
        m_delivered = true;
        *s = m_inputstring;
    }
};

class FileInput: public Input
{
    std::ifstream m_input;
public:
    FileInput(const std::string& filename)
        throw (Exceptions::FileError)
        : m_input(filename.c_str(), std::ios::in)
    {  
        if (!m_input)
            throw Exceptions::FileError(filename);
    }
    inline bool hasMoreData() const
    {
        return !m_input.eof();
    }
    inline void getMoreData(std::string* s) 
    {
        assert(!m_input.eof());
        std::getline(m_input, *s);
    }
};


class Tokenizer 
{
protected:
    UConverter* m_conv;
    UErrorCode m_errcode;
    
    std::string m_tokenbuffer;
    size_t m_pos;
    std::string m_all_delim;
    
    static std::string m_whitespace;
    static std::string m_delimiters;
    
    bool m_insertWhitespace;
    
    bool containsTokens(const std::string& buffer, int pos = 0) const
    {
        return buffer.length() > 0 && buffer.find_first_not_of(m_whitespace, pos) != std::string::npos;
    }
    inline bool bufferIsFull() const
    {
        return containsTokens(m_tokenbuffer, m_pos);
    }
    
    bool fillBuffer() 
    {
        while(m_input->hasMoreData()) {
            m_input->getMoreData(&m_tokenbuffer);
            if(containsTokens(m_tokenbuffer)) {
                m_insertWhitespace = true;
                m_pos = 0;
                return true;
            }
        }
        return false;
    }
    
    boost::scoped_ptr<Input> m_input;
    
public:
    Tokenizer(Input* i) 
        : m_errcode(U_ZERO_ERROR), m_input(i)
    {
        m_conv = ucnv_open("UTF8", &m_errcode);
        assert(m_errcode == U_ZERO_ERROR);
        m_errcode = U_ZERO_ERROR;
        m_pos = 0;
        m_all_delim = m_delimiters;
        m_all_delim.append(m_whitespace);
        
        fillBuffer();
        m_insertWhitespace = false;
    }
    ~Tokenizer() 
    {
        ucnv_close(m_conv);
    }
    
    inline bool hasMoreTokens()
    {
        return bufferIsFull() || fillBuffer();
    }
    // TODO: less copying around!
    std::string nextUnencodedToken(bool* hasWhitespace = 0);
    schma::UnicodePtr nextToken(bool* hasWhitespace = 0);
};

class TextSaver 
{
    char buf[1024];
    Tokenizer m_tokenizer;
    std::ofstream m_normalized;
    
    

public:
    struct TokenInformation 
    {
        std::streamoff offset;
        schma::UnicodePtr token;
      
        TokenInformation(std::streamoff o, schma::UnicodePtr t)
            : offset(o), token(t)
        {}
        TokenInformation()
        {}
      
    };

    TextSaver(const std::string& infile, const std::string& outfile) 
        throw(Exceptions::FileError)
        : m_tokenizer(new FileInput(infile)), 
          m_normalized(outfile.c_str(), std::ios::out | std::ios::binary) // die, windows scum, die! you'll never read our precious files!
        
    {
        if(!m_normalized)
            throw Exceptions::FileError(outfile);
    }
    inline bool hasMoreTokens() 
    {
        return m_tokenizer.hasMoreTokens();
    }
    TokenInformation nextToken()
        throw(std::out_of_range);
};

}


#endif // TOKENIZER_H

---