indexer.cpp

Go to the documentation of this file.

/*
  Phrasehunter - index and query text corpora
  Copyright (C) 2006  Torsten Marek (shlomme@gmx.de) &
  Armin Schmidt (armin.sch@gmail.com)
  
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
  as published by the Free Software Foundation; either version 2
  of the License, or (at your option) any later version.
  
  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#include <fstream>
#include <tr1/unordered_map>


#include "sqlitepp/sqlitepp.h"

#include "phrasehunter/indexer.h"
#include "phrasehunter/tokenizer.h"
#include "support/unicodehelpers.h"

using namespace SQLitePP;

namespace PhraseHunter {

void IndexManager::removeFromIndex(DocID docID) 
{
    (*m_db.statement("DELETE FROM occurrences WHERE docid = ?"))
        .bindArgs(docID)
        .exec();
}


typedef std::tr1::unordered_map<std::string, std::pair<TokenID, PositionList> > idx_map;

void IndexManager::addToIndex(TextSaver& token_stream, DocID docID) 
    throw(Exceptions::FileError)
{
    char buf[1024];
    idx_map tokens;
    
    TextSaver::TokenInformation ti;
    
    m_db.begin();
    
    Statement::Pointer getid = m_db.statement("SELECT id FROM tokens WHERE word = ?");
    Statement::Pointer insertTok = m_db.statement("INSERT INTO tokens (word) VALUES (?)");
    Statement::Pointer insertPos = m_db.statement("\
INSERT INTO occurrences (wordid, docid, positions) VALUES (?,?,?)");
    
    UErrorCode status;
    
    while(token_stream.hasMoreTokens()) {
        ti = token_stream.nextToken();
      
        ti.token->toLower();
        status = U_ZERO_ERROR;

        ti.token->extract(buf, 1023, schma::UTF8Converter(), status);

        idx_map::iterator it = tokens.find(buf);

        if(it == tokens.end()) {
            int id;
            getid->bindArg(buf);
            ResultIterator ri = getid->query();
          
            if(ri.hasMoreRows()) {
                id = ri.get<int>(0);
            } else {
                id = (*insertTok)
                    .bindArg(buf)
                    .exec()
                    .lastInsertId();

                insertTok->reset();
            }
            getid->reset();
            PositionList v;
            v.push_back(ti.offset);
          
            tokens[buf] = std::make_pair(id, v);
        } else {
            it->second.second.push_back(ti.offset);
        }
    }
    
    for(idx_map::iterator it = tokens.begin();
        it != tokens.end();
        ++it) {
        (*insertPos)
            .bindArgs(it->second.first, docID, 
                      Blob(reinterpret_cast<const void*>(&it->second.second[0]), 
                           it->second.second.size() * sizeof(PositionList::value_type)))
            .exec()
            .reset();
    }
    m_db.commit();
}

IndexManager::~IndexManager() 
{
    (*m_db.statement("\
UPDATE tokens SET \
frequency = (SELECT SUM(LENGTH(positions)) / ? FROM occurrences WHERE wordid = tokens.id ), \
documentfrequency = (SELECT DISTINCT docid FROM occurrences WHERE wordid = tokens.id)"))
        .bindArgs(sizeof(IdxPos))
        .exec();
    
    
    m_db.statement("UPDATE tokens SET rank = NULL")->exec();
    
    Statement::Pointer getByFrequency = 
        m_db.statement("SELECT id FROM tokens ORDER BY frequency DESC");

    Statement::Pointer setRank = 
        m_db.statement("UPDATE tokens SET rank = ? WHERE id = ?");
    
    int rank = 1;
    for(ResultIterator ri = getByFrequency->query();
        ri.hasMoreRows();
        ri.next()) {
        (*setRank)
            .bindArgs(rank++, ri.get<int>(0))
            .exec()
            .reset();
    }
}


} // PhraseHunter

---