proto/Source/ngrams.h

Go to the documentation of this file.

#ifndef _NGRAMS_H
#define _NGRAMS_H

//
#include "config.h"

#ifdef _USE_HASHSET
// Use almost std hash_set.
#if defined(_MSC_VER)
#include <hash_set>
#elif defined(__GNUC__)
// GCC stores hash_set in ext subdirectory.
#include <ext/hash_set>
#endif
#else
//
#include "hash_vector.h"
#endif

// Some vectors are engaged.
#include <vector>
// We use std::pair here.
#include <utility>

// N-gram, of course!
#include "ngram.h"
// VoidPool class.
#include "pool.h"

namespace ace {

class NGramHashCounter {    

#ifdef _64_BIT
    // 64-bit CPU
    static const size_t _seed = 0xdeadbeeffeeddeaf;
#else

    static const size_t _seed = 0xdeadbeef;
#endif

public:
    static size_t count_hash(const NGram *gram);
    static bool check(ngram_size_t n);
};

struct NGramHasher: public std::unary_function<size_t, const NGram *> {
    inline size_t operator()(const NGram *key) const {
        return NGramHashCounter::count_hash(key);
    }
};

#ifdef _USE_HASHSET

#if defined(_MSC_VER)

struct NGramLessComparator: public std::binary_function<bool, const NGram *, const NGram *> {
    bool operator()(const NGram *lhs, const NGram *rhs) const;
};

class NGramHashCompare: public stdext::hash_compare<const NGram *> {
    static NGramHasher _hasher;
    static NGramLessComparator _comparator;
public:
    /* Double sized than default.
     */
    enum {
        bucket_size = 8,
        min_buckets = 16
    };
    inline size_t operator()(const NGram *key) const {
        //
        return _hasher(key);
    }
    inline bool operator()(const NGram *lhs, const NGram *rhs) const {
        //
        return _comparator(lhs, rhs);
    }
};

#elif defined(__GNUC__)

struct NGramEqualComparator: public std::binary_function<bool, const NGram *, const NGram *> {
    bool operator()(const NGram *lhs, const NGram *rhs) const;
};

#endif

#endif // _USE_HASHSET

class NGramStore {
public:
#ifdef _USE_HASHSET

#if defined(_MSC_VER)
    typedef stdext::hash_set<NGram *, NGramHashCompare> hash_table_t;
#elif defined(__GNUC__)
    typedef __gnu_cxx::hash_set<NGram *, NGramHasher, NGramEqualComparator> hash_table_t;
#endif
#else
    typedef HashVector<NGram *, NGramHasher> hash_table_t;
#endif

    NGram zero_ngram;

private:
    typedef std::pair<hash_table_t::const_iterator, bool> _hash_table_pair_cib_t;
    std::vector<VoidPool> _memory_pools;
    std::vector<hash_table_t> _hash_tables;

    _hash_table_pair_cib_t _find(const NGram *ngram) const;

    NGramStore(const NGramStore&);
    NGramStore& operator=(const NGramStore&);
public:
    typedef std::pair<NGram *, bool> insert_status_t;

    NGramStore(size_t memory_pool_block_size = 0);
    
    // No need for special destructor - pools destructor properly clears memory they occupy.     
    
    inline hash_table_t::const_iterator begin(ngram_size_t type) const { return _hash_tables[type].begin(); }
    /*  @return Iterator pointing to the beginning of store of full N-gram type.
     */
    inline hash_table_t::const_iterator begin(void) const { return _hash_tables[NGram::full_ngram_type()].begin(); }
    inline hash_table_t::const_iterator end(ngram_size_t type) const { return _hash_tables[type].end(); }
    /*  @return Iterator pointing to the end of store of full N-gram type.
     */
    inline hash_table_t::const_iterator end(void) const { return _hash_tables[NGram::full_ngram_type()].end(); }
    const NGram * get(const NGram *ngram) const;
    void init(size_t memory_pool_block_size);
    insert_status_t insert(const NGram *ngram);
    inline size_t size(ngram_type_t type) const { return (type == 0) ? 1 : _hash_tables[type].size(); }
    inline size_t size(void) const { return _hash_tables[NGram::full_ngram_type()].size(); }
#ifndef _USE_HASHSET

    void sort(ngram_type_t type = 0);
#endif
};

typedef std::vector<frequency_t> freq_table_t;

struct NGramFrequencies {
    const NGram *ngram;
    freq_table_t frequencies;
    NGramFrequencies(void): ngram(NULL), frequencies(NGram::types_count()) {}
};

class NGramsProcessor {
    const NGramStore& _ngram_store;
    NGramStore::hash_table_t::const_iterator _next;
    NGramStore::hash_table_t::const_iterator _end;
    NGramFrequencies _ngram_frequencies;
public:
    NGramsProcessor(NGramStore& ngrams): _ngram_store(ngrams), _next(ngrams.begin()), _end(ngrams.end()), _ngram_frequencies() {}

    bool next(void);

    inline const NGramFrequencies& ngram_frequencies(void) const { return _ngram_frequencies; }
};

} // namespace ace

#endif

---