contextreader.cpp

Go to the documentation of this file.

/*
  Phrasehunter - index and query text corpora
  Copyright (C) 2006  Torsten Marek (shlomme@gmx.de) &
  Armin Schmidt (armin.sch@gmail.com)
  
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
  as published by the Free Software Foundation; either version 2
  of the License, or (at your option) any later version.
  
  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#include <fstream>

#include "phrasehunter/contextreader.h"
#include "phrasehunter/tokencontext.h"
#include "phrasehunter/token.h"
#include "support/unicodehelpers.h"

namespace PhraseHunter {

ContextReader::ContextReader(const boost::filesystem::path& textDirectory):
    m_textDirectory(textDirectory)
{}

unsigned int ContextReader::removeNewlines(char* buffer, unsigned int length) const
{
    unsigned int offset = 0;
    for(unsigned int idx = 0; idx + offset < length; ++idx) {
        while(buffer[idx + offset] == '\n') {
            ++offset;
            if(idx + offset >= length)
                return offset;
        }
        buffer[idx] = buffer[idx+offset];
    }
    return offset;
}

void ContextReader::readBlock(std::ifstream& file, TokenContextPtr result) const
{
    static char buffer[BufferSize];
    IdxPos position = result->position();
    int startpos = position - BufferSize;
    int bytesRead;
    if (startpos < 0) 
        bytesRead = file.readsome(buffer, position);
    else {
        file.seekg(startpos);
        bytesRead = file.readsome(buffer, BufferSize-1);
    }
    bytesRead -= removeNewlines(buffer, bytesRead);
    UErrorCode status = U_ZERO_ERROR;

    result->setLeftContext(schma::UnicodePtr(new UnicodeString(buffer, bytesRead, schma::UTF8Converter(), status)));

    bytesRead = file.readsome(buffer, result->tokenLength());
    bytesRead -= removeNewlines(buffer, bytesRead);
    status = U_ZERO_ERROR;
    result->setToken(schma::UnicodePtr(new UnicodeString(buffer, bytesRead, schma::UTF8Converter(), status)));
    result->token()->trim();
    
    bytesRead = file.readsome(buffer, BufferSize-1);
    bytesRead -= removeNewlines(buffer, bytesRead);
    status = U_ZERO_ERROR;
    result->setRightContext(schma::UnicodePtr(new UnicodeString(buffer, bytesRead, schma::UTF8Converter(), status)));
}


TokenContextPtr ContextReader::getContextFromPosition(DocID docId, IdxPos pos, unsigned int tokenLength, unsigned int width) const
    throw(Exceptions::FileError) 
{
    TokenContextPtr output = TokenContext::emptyContext(docId, pos, tokenLength);
    fillContext(output, width);
    
    return output;
}

void ContextReader::fillContext(TokenContextPtr context, unsigned int width) const
    throw(Exceptions::FileError)
{
    std::ifstream file(documentFile(context->docID()).string().c_str(), 
                   std::ios::in | std::ios::binary);
    if(!file) {
      throw Exceptions::FileError(documentFile(context->docID()).string());
    }
    readBlock(file, context);
    context->leftContext()->remove(0, context->leftContext()->length() - width);
    context->rightContext()->truncate(width);
}

std::vector<TokenContextPtr> ContextReader::context(TokenPtr token, unsigned int width) const
    throw(Exceptions::FileError)
{
    try {
        schma::UnicodePtr unitoken;
      
        if(token->isUniform()) {
            unitoken = token->tokenString();
        }
        
        std::vector<TokenContextPtr> result;
      
        const OccurrenceMap& all_occurrences = token->allOccurrences();
        for (OccurrenceMap::const_iterator it = all_occurrences.begin();
             it != all_occurrences.end();
             ++it) {            
            std::string corpus_file = documentFile(it->first).string();
          
            for (PositionList::const_iterator it_inner = it->second.begin();
                 it_inner != it->second.end(); ++it_inner) {

                TokenContextPtr context = TokenContext::emptyContext(it->first, *it_inner, token->length());
            fillContext(context, width);
                
            if(unitoken != NULL) {
                    context->setToken(unitoken);
                }
            }
        }
        return result;
    }
    catch (Exceptions::FileError& _e) {
        throw;
    }
}

std::vector<TokenContextPtr> ContextReader::context(const TokenVector& tokens, unsigned int width) const
    throw(Exceptions::FileError)
{
    std::vector<TokenContextPtr> results;
    for (unsigned int i=0; i<tokens.size(); ++i) {
        std::vector<TokenContextPtr> oneContext = context(tokens[i], width);
        results.insert(results.end(), oneContext.begin(), oneContext.end());
    }
    return results;
}

}

---