statistics.h

Go to the documentation of this file.

// -*- C++ -*-
/*
  Phrasehunter - index and query text corpora
  Copyright (C) 2006  Torsten Marek (shlomme@gmx.de) &
  Armin Schmidt (armin.sch@gmail.com)
  
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
  as published by the Free Software Foundation; either version 2
  of the License, or (at your option) any later version.
  
  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#ifndef STATISTICS_H
#define STATISTICS_H STATISTICS_H

#include "ph_types.h"
#include "searchengine.h"
#include "support/unicodehelpers.h"

namespace PhraseHunter {

//! Class that provides functionality to do statistical calculations on a corpus
class StatisticsEngine
{
public:
    //! Constructor. Should not be called directly. Instead, use CorpusManager::statisticsEngine().
    StatisticsEngine(SearchEngine*, ContextReader*, SQLitePP::SqliteDB& db);
    ~StatisticsEngine() {};

    //! Returns the rank of a Token.
    unsigned int rank(TokenPtr) const;
    //! Returns the frequency count of a Token.
    unsigned int overallFrequency(TokenPtr) const;
    /**
       \brief Returns the frequency count of the intersection of two Token.
       The size of the intersection is determined by the number of times the
       offsets of the two tokens are not further apart than getContextSize().
    */
    unsigned int frequency_x_intersection_y(TokenPtr, TokenPtr) const;
    //! Returns the relative frequency of the Token.
    double relative_frequency(TokenPtr) const;
    //! Returns the probability of the intersection of two tokens according to maximum likelyhood estimation.
    double mle_x_intersection_y(TokenPtr, TokenPtr) const;
    //! Returns the probability of one Token occuring given the other had occurred according to maximum likelyhood estimation.
    double mle_x_given_y(TokenPtr, TokenPtr) const;

    //! Returns the size of the sample space, i.e. the number of all word tokens and everything else that is considered a token.
    unsigned int getSizeOfSampleSpace() const { return m_sizeOfSampleSpace; }
    //! Returns the max number of bytes two Tokens may be apart to be considered intersecting.
    int getContextSize() const { return m_contextWindow;}
    //! Returns the average word length in bytes.
    double getAverageWordLength() const { return m_averageWordLength; }
    //! Returns the number of Documents in the corpus.
    unsigned int getNumberOfDocuments() const { return m_numberOfDocuments; }
    //! Returns the number of word types (and everything else considered a token) as opposed to word tokens.
    unsigned int getNumberOfTypes() const { return m_numberOfTypes; }

    /**
       \brief Set how large the context may be within which two tokens may be considered intersecting.
       \param words The context size in number of words.
    */
    void setNumberContextWords(int words) {
        m_contextWindow = static_cast<int>(m_averageWordLength) * words + words;
    }

    /**
       \brief Get all context strings for a particular Token.
       \return A map with all context strings as UnicodePtr and their frequencies.
       \param t A Token.
       \param contextlen The maximum size of the context in bytes.
    */
    std::map<schma::UnicodePtr, int> getContextVector(TokenPtr t, int contextlen) const;
    /**
       \brief Returns the pointwise mutual information of a Token and a particular context.
       \param token A Token.
       \param contextVec The context vector of token.
       \param context A particular context.
    */
    double mutual_information(const TokenPtr& token, const std::map<schma::UnicodePtr,int>& contextVec, schma::UnicodePtr context) const;
    /**
       \brief Calculate the similarity between two tokens.
       \param t1 First Token.
       \param t2 Second Token.
       \param contextlen The maximum size of the context in bytes.
    */
    double similarity(TokenPtr t1, TokenPtr t2, int contextlen) const;
    /**
       \brief Find all Tokens that are semantically similar to a particular Token.
       \param t The Token for which to find similar ones.
       \param contextlen The maximum size of the context in bytes.
    */
    std::multimap<double,TokenPtr> similarTokens(TokenPtr t, int contextlen) const;

    static unsigned int sizeOfDocSet(const TokenVector&);
    
private:
    static const int MAX_CANDIDATES = 1000;
    static const int MAX_DEPENDENTS = 20;
    
    unsigned int m_sizeOfSampleSpace;
    double m_averageWordLength;
    unsigned int m_numberOfDocuments;
    unsigned int m_numberOfTypes; 
    unsigned int m_contextWindow;
    
    SearchEngine* m_searcher;
    ContextReader* m_reader;
    SQLitePP::SqliteDB& m_db;

    unsigned freq_intersection_in_doc(const PositionList&, const PositionList&) const;
    TokenVector getCandidates(TokenPtr) const;
    std::pair<unsigned int, unsigned int> contextBounds(unsigned int position, int numWord) const;

};
} // namespace PhraseHunter

#endif // STATISTICS_H

---