token.cpp

Go to the documentation of this file.

/*
  Phrasehunter - index and query text corpora
  Copyright (C) 2006  Torsten Marek (shlomme@gmx.de) &
  Armin Schmidt (armin.sch@gmail.com)
  
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
  as published by the Free Software Foundation; either version 2
  of the License, or (at your option) any later version.
  
  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#include <boost/lambda/lambda.hpp>
#include <algorithm>

#include "phrasehunter/token.h"

using namespace SQLitePP;
using namespace boost::lambda;

namespace PhraseHunter {

TokenPtr EmptyToken::s_inst = TokenPtr(new EmptyToken);

std::vector<DocID> Token::documentIDs() const 
{
    std::vector<DocID> ids;
    for(OccurrenceMap::const_iterator docs = m_occurrences.begin();
        docs != m_occurrences.end(); ++docs) {
        ids.push_back(docs->first);
    }
    return ids;
}

void CorpusToken::insertPositions(ResultIterator occ_it) 
{
    for(; occ_it.hasMoreRows(); occ_it.next()) {
        DocID docID = occ_it.get<int>(0);
        Blob posblob = occ_it.get<Blob>(1);
        //const IdxPos* positions = reinterpret_cast<const IdxPos*>(posblob.first);
        unsigned int numOcc = posblob.second / sizeof(IdxPos);
        PositionList& pl = m_occurrences[docID];
        pl.resize(numOcc);
        // compute corpus frequency
        m_corpusfreq += numOcc;
        memcpy(&pl[0], posblob.first, posblob.second);
        /*for(unsigned int i = 0; i < numOcc; ++i, ++positions) {
          pl.push_back(*positions);
          }*/
    }
}

CorpusToken::CorpusToken(const char* token, TokenID id, SqliteDB& db)
    : CorpusTokenBase(token, id)
{
    init(id, db);
}

CorpusToken::CorpusToken(const char* token, TokenID id, SqliteDB& db, const std::set<DocID>& documentIDs)
    : CorpusTokenBase(token, id)
{
    init(id, db, documentIDs);
}

CorpusToken::CorpusToken(schma::UnicodePtr token, TokenID id, SqliteDB& db)
    : CorpusTokenBase(token, id)
{
    init(id, db);
}

CorpusToken::CorpusToken(schma::UnicodePtr token, TokenID id, SqliteDB& db, const std::set<DocID>& documentIDs)
    : CorpusTokenBase(token, id)
{
    init(id, db, documentIDs);
}

void CorpusToken::init(TokenID id, SqliteDB& db)
{
    Statement::Pointer occs = db.cachedStatement("SELECT docid, positions FROM occurrences WHERE wordid = ?");
    occs->bindArgs(id);
    insertPositions(occs->query());
}

void CorpusToken::init(TokenID id, SqliteDB& db, const std::set<DocID>& documentIDs)
{
    std::stringstream valueList;
    std::for_each(documentIDs.begin(), documentIDs.end(), valueList << _1 << ',');
    
    Statement::Pointer occs = 
        db.statement(("SELECT docid, positions FROM occurrences\
WHERE wordid = ? AND docid IN (" + valueList.str() + ")").c_str());
    occs->bindArgs(id);
    insertPositions(occs->query());
}

TokenPtr LightCorpusToken::search(const char* tokenstr, SQLitePP::SqliteDB& db) 
{
    Statement::Pointer getID = db.cachedStatement("SELECT id, frequency, documentfrequency FROM tokens WHERE word = ?");
    getID->bindArgs(tokenstr);
      
    ResultIterator ri = getID->query();
    return (ri.hasMoreRows())
        ? TokenPtr(new LightCorpusToken(tokenstr, ri.get<int>(0), ri.get<int>(1), ri.get<int>(2)))
        : EmptyToken::instance();
}

inline void MutableToken::removeDocument(DocID docID) 
{
    if(inDoc(docID)) {
        m_occurrences.erase(docID);
    }
}

void MutableToken::addOccurrence(DocID docID, IdxPos position)
{
    OccurrenceMap::iterator it = m_occurrences.find(docID);
    if(it == m_occurrences.end()) {
        PositionList p;
        p.push_back(position);
        m_occurrences[docID] = p;
    } else {
        if(position >= *it->second.rend()) {
            it->second.push_back(position);
        } else {
            it->second.push_back(it->second.back());
            int pos;
            for(pos = it->second.size()-1; it->second[pos]-1 > position; --pos) 
                it->second[pos] = it->second[pos]-1;
            it->second[pos] = position;
        }
    }
    ++m_totalOccurrences;
}

template<>
struct Phrase::adja<Phrase::LeftToRight>
{
    adja(TokenPtr left, TokenPtr right):
        occ_outer(left->allOccurrences()),
        occ_inner(right->allOccurrences()),
        innerOffset(left->length()),
        startOffset(0) 
    {}
    const OccurrenceMap& occ_outer, occ_inner;
    const int innerOffset;
    const int startOffset;
};

template<>
struct Phrase::adja<Phrase::RightToLeft>
{
    adja(TokenPtr left, TokenPtr right):
        occ_outer(right->allOccurrences()),
        occ_inner(left->allOccurrences()),
        innerOffset(-left->length()),
        startOffset(-left->length())
    {}
    const OccurrenceMap& occ_outer, occ_inner;
    const int innerOffset;
    const int startOffset;
};

template<Phrase::Direction d>
TokenPtr Phrase::getAdjacent(TokenPtr left, TokenPtr right)
{
    schma::UnicodePtr phrase(new UnicodeString(*(left->tokenString())));
    phrase->append(' ');
    phrase->append(*(right->tokenString()));
    
    Phrase* result = new Phrase(phrase, left->numTokens() + right->numTokens());
    
    adja<d> a(left, right);
    
    const OccurrenceMap& occ_outer = a.occ_outer;
    const OccurrenceMap& occ_inner = a.occ_inner;
    
    for (OccurrenceMap::const_iterator outer_doc = occ_outer.begin();
         outer_doc != occ_outer.end();
         ++outer_doc) {
      
        OccurrenceMap::const_iterator inner_doc =
            occ_inner.find(outer_doc->first);
      
        if (inner_doc == occ_inner.end())
            continue;
      
        for (PositionList::const_iterator outer_pos = outer_doc->second.begin();
             outer_pos != outer_doc->second.end();
             ++outer_pos) {
          
            if(binary_search(inner_doc->second.begin(), inner_doc->second.end(),
                             *outer_pos + a.innerOffset)) {
                result->addOccurrence(outer_doc->first, *outer_pos + a.startOffset);
            }
        }
    }
    return (result->isEmpty())
        ? EmptyToken::instance()
        : TokenPtr(result);
}

TokenPtr Phrase::mergeTokens(TokenPtr left, TokenPtr right)
{
    return (left->corpusFrequency() <= right->corpusFrequency())
        ? getAdjacent<LeftToRight>(left, right)
        : getAdjacent<RightToLeft>(left, right);

}

}

---