token.h

Go to the documentation of this file.

// -*- C++ -*-
/*
  Phrasehunter - index and query text corpora
  Copyright (C) 2006  Torsten Marek (shlomme@gmx.de) &
  Armin Schmidt (armin.sch@gmail.com)
  
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
  as published by the Free Software Foundation; either version 2
  of the License, or (at your option) any later version.
  
  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#ifndef TOKEN_H
#define TOKEN_H TOKEN_H

#include <iostream>
#include <sstream>
#include <set>
#include <boost/shared_ptr.hpp>

#include "support/safe_bool.hpp"
#include "support/unicodehelpers.h"

#include "sqlitepp/sqlitepp.h"

#include "ph_types.h"

namespace PhraseHunter {

/**
   \brief Virtual base class for all Token descendants.

   A Token consists of a particular string (i.e. word type) and a map of its
   occurrences, i.e. documents and offsets, in the corpus. Tokens may be used in boolean
   expressions.
*/
class Token: public safe_bool<>
{
protected:
    schma::UnicodePtr m_tokenstring;
    OccurrenceMap m_occurrences;
    Token(const char* token): m_tokenstring(new UnicodeString(token))
    {}
    Token(schma::UnicodePtr tokenstring) : m_tokenstring(tokenstring)
    {}

    static const int SPACE_BETWEEN_TWO_TOKENS = 2;
    
public:
    virtual ~Token() {}

    //! Return the real length (bytes) of a token string in the index.
    virtual size_t length() const 
    {
        return m_tokenstring->length() + SPACE_BETWEEN_TWO_TOKENS;
    }
    //! Returns true if all occurrences in all files are expected to look *exactly* the same.
    bool isUniform() const 
    {
        return false; 
    }
    //! Returns true if there are no actual occurrences of this Token in a corpus, i.e. its occurrence map is empty.
    virtual bool isEmpty() const { return m_occurrences.empty(); }

    //! Return the number of times this Token occurs in the corpus.
    virtual unsigned int corpusFrequency() const = 0;
    
    //! Returns the string to which this Token belongs as a UnicodePtr.
    schma::UnicodePtr tokenString() const { return m_tokenstring; }

    //! Returns the number of documents a Token occurs in.
    virtual unsigned int documentFrequency() const { return m_occurrences.size(); }

    //! Returns true if this Token occurs in a document.
    bool inDoc(DocID docID) const { return m_occurrences.count(docID) > 0; }

    //! Returns a reference to all offsets of a Token in a particular document as a PositionList.
    const PositionList& documentOccurrences(DocID docID) { return m_occurrences[docID]; }

    //! Return a reference to the entire OccurrenceMap of a Token.
    const OccurrenceMap& allOccurrences() const { return m_occurrences; }

    //! Returns the DocIDs of all documents, in which this Token occurs.
    virtual std::vector<DocID> documentIDs() const;
    virtual unsigned int numTokens() const { return 1; }
    //! Returns the ID for this Token.
    virtual TokenID id() const { return InvalidTokenID; }
};

//! A Token without a token string nor any occurrences.
class EmptyToken: public Token {
private:
    EmptyToken(): Token("") {}
    static TokenPtr s_inst;

protected:
    bool boolean_test() const
    { return false; }
    
public:
    //! Return instance() anywhere you would otherwise return NULL for a TokenPtr
    static TokenPtr instance() 
    { return s_inst; }

    //! Always returns 0.
    unsigned int corpusFrequency() const { return 0; }
    //! Always returns true.
    bool isEmpty() const { return true; }
    //! Always returns 0.
    unsigned int numTokens() const { return 0; }
};

//! Base class for all CorpusToken classes.
class CorpusTokenBase: public Token
{
protected:
    CorpusTokenBase(const char* token, TokenID id): 
        Token(token), m_id(id), m_corpusfreq(0) {}
    CorpusTokenBase(schma::UnicodePtr token, TokenID id): 
        Token(token), m_id(id), m_corpusfreq(0) {}
    
    bool boolean_test() const 
    { return m_corpusfreq > 0; }
    
    TokenID m_id;
    unsigned int m_corpusfreq;
    
public:
    //! Returns the ID of this Token.
    TokenID id() const { return m_id; }
    //! Return the number of times this Token occurs in the corpus.
    unsigned int corpusFrequency() const { return m_corpusfreq; }
};

    
/**
   \brief A Token of only one word.

   This is a non-phrasal Token. For reading only. A PhraseHunter::CorpusToken
   has an individual id, by which it can uniquely be identified and called from
   the database.
*/
class CorpusToken: public CorpusTokenBase
{
private:
    void insertPositions(SQLitePP::ResultIterator);
    CorpusToken(const char*, TokenID, SQLitePP::SqliteDB&);
    CorpusToken(const char*, TokenID, SQLitePP::SqliteDB&, const std::set<DocID>&);
    CorpusToken(schma::UnicodePtr, TokenID, SQLitePP::SqliteDB&);
    CorpusToken(schma::UnicodePtr, TokenID, SQLitePP::SqliteDB&, const std::set<DocID>&);

    void init(TokenID, SQLitePP::SqliteDB&);
    void init(TokenID, SQLitePP::SqliteDB&, const std::set<DocID>&);

public:
    //! Don't call this template function directly. Instead, use the non-template version below.
    template<typename T>
    static TokenPtr loadFromCorpus(T tokenstr, TokenID id, SQLitePP::SqliteDB& db)
    {
        p_assert(id != InvalidTokenID, "Invalid token id");
        return TokenPtr(new CorpusToken(tokenstr, id, db));
    }

    //! Don't call this template function directly. Instead, use the non-template version below.
    template<typename T>
    static TokenPtr loadFromCorpus(T tokenstr, TokenID id, SQLitePP::SqliteDB& db, 
                                   const std::set<DocID>& docs)
    {
        p_assert(id != InvalidTokenID, "Invalid token id");
        TokenPtr t(new CorpusToken(tokenstr, id, db, docs));
        return (t->isEmpty()) 
            ? EmptyToken::instance()
            : t;
    }
    /**
       \brief Loads a CorpusToken. For initializing the OccurrenceMap of a LightCorpusToken.
       \return The so-(re)loaded CorpusToken
    */
    static TokenPtr loadFromToken(const TokenPtr& token, SQLitePP::SqliteDB& db)
    {
        return loadFromCorpus<schma::UnicodePtr>(token->tokenString(), token->id(), db);
    }
    /**
       \brief Like above function, but ignoring all occurrences that are not in docs.
       \return The so-(re)loaded CorpusToken
    */
    static TokenPtr loadFromToken(const TokenPtr& token, SQLitePP::SqliteDB& db, const std::set<DocID>& docs)
    {
        return loadFromCorpus<schma::UnicodePtr>(token->tokenString(), token->id(), db, docs);
    }

    //! Returns true if this Token has no occurrences in the corpus.
    bool isEmpty() const 
    { return m_corpusfreq == 0; }
    
};

//! A corpus token that only carries corpus frequency information (opposed to CorpusToken)
class LightCorpusToken: public CorpusTokenBase
{
private:
    unsigned int m_documentFreq;
    LightCorpusToken(const char* tokenstr, TokenID id, 
                     unsigned int freq, unsigned int docFreq) 
        : CorpusTokenBase(tokenstr, id), m_documentFreq(docFreq) 
    {
        m_corpusfreq = freq;
    }
    
public:
    /**
       \brief Search and initialize a LightCorpusToken.
       Special search function for LightCorpusToken if you don't want to use
       SearchEngine::searchToken() which always loads the whole OccurrenceMap
    */
    static TokenPtr search(const char* tokenstr, SQLitePP::SqliteDB& db);
    //! Returns the number of documents a Token occurs in.
    unsigned int documentFrequency() const { return m_documentFreq; }
};

/**
   \brief A Token whose OccurrenceMap can be modified.
   The main purpose of MutableToken is to be a base class for phrases,
   whose occurrence matrix needs to be modified as phrases are themselve
   not indexed.
*/
class MutableToken: public Token
{
protected:
    unsigned int m_totalOccurrences;
    bool boolean_test() const 
    { return m_totalOccurrences > 0; }
    
    MutableToken(const char* tokenstring): Token(tokenstring), m_totalOccurrences(0)
    {}
    MutableToken(schma::UnicodePtr tokenstring): Token(tokenstring), m_totalOccurrences(0)
    {}
    
public:
    //! Returns the number of documents a Token occurs in.
    unsigned int corpusFrequency() const { return m_totalOccurrences; }
    //! Remove the document referred to by docID from the occurrence matrix.
    void removeDocument(DocID docID);
    /**
       \brief Add a position to the occurrence matrix of this Token.
       \param docID The document ID referring to the document of the new occurrence.
       \param position The byte offset, i.e. byte position, of the new occurrence.
    */
    void addOccurrence(DocID docID, IdxPos position);
    
};

//! A phrasal token consists of several words or tokens
class Phrase: public MutableToken
{
public:
    /**
       \brief The direction which should be used to merge a Token to this Phrase.
       When merging two Tokens into a Phrase, we need to go through the entire
       occurrence matrix of one Token and see if its position is next to an occurrence
       of the other Token. This process can be quite time consuming but may be
       considerably speed up by going through the Token with less occurrences as a
       starting point. Phrase::Direction indicates the Token to start with: If the left
       Token is less frequent, call getAdjacent<LeftToRight>(TokenPtr left, TokenPtr right),
       otherwise call getAdjacent<RightToLeft>(TokenPtr left, TokenPtr right).
    */
    enum Direction {
        LeftToRight,
        RightToLeft
    };

private:
    template<Direction d>
    struct adja
    {};
    int m_tokencount;
    
    Phrase(const char* token, int tokencount)
        : MutableToken(token), m_tokencount(tokencount) 
    {}
    Phrase(schma::UnicodePtr token, int tokencount)
        : MutableToken(token), m_tokencount(tokencount)
    {}
    
public:
    /**
       \brief Merge two Tokens and specify the merging Direction.
       Usually, you want to call mergeTokens() instead.
    */
    template<Direction d>
    static TokenPtr getAdjacent(TokenPtr left, TokenPtr right);
    
    //! Merge two Token objects into a Phrase.
    static TokenPtr mergeTokens(TokenPtr left, TokenPtr right);

    //! Returns the length of the Phrase as it is in the text repository.
    inline size_t length() const 
    {
        return m_tokenstring->length() + m_tokencount + 1;
    }

    //! Returns how often this Phrase occurs in the corpus.
    inline unsigned int numTokens() const
    {
        return m_tokencount;
    }
};

}

#endif // TOKEN_H

---