proto/Source/ngram.h

Go to the documentation of this file.

#ifndef _NGRAM_H
#define _NGRAM_H

// We use pair here.
#include <utility>
#include <vector>

#include "config.h"
// N-gram context.
#include "context.h"
#ifdef _DEPENDENCY_MODE
// dependency::head needed.
#include "dependencies.h"
#endif
// Frequence counter overflow treatment.
#include "overflow.h"
// ith_bit(), bits_in()
#include "utils.h"
// Word(s) are processed here.
#include "word.h"

namespace ace {

#ifdef _SURFACE_MODE

typedef words_store_t::const_iterator raw_ngram_member_t;
#else

typedef std::pair<ngram_size_t, words_store_t::const_iterator> raw_ngram_member_t;
#endif

typedef std::vector<raw_ngram_member_t> raw_ngram_t;

// Forward declaration (class is declared as friend).
class NGramToken;

typedef unsigned int freq_counter_t;

typedef size_t frequency_t;

#if defined(_MSC_VER) || defined(__GNUC__)
// Save default pack value.
#pragma pack(push)
// Pack following structure data on 2 bytes boundary.
#pragma pack(1)
#endif

class NGram {
public:
    class Member {
        TaggedLemma _tagged_lemma;
#ifdef _DEPENDENCY_MODE

        ngram_size_t _parent;
        dependency_index_t _dependency;
    public:
        Member(string_index_t lemma, tag_index_t tag, ngram_size_t parent, dependency_index_t dependency): _tagged_lemma(lemma, tag), _parent(parent), _dependency(dependency) {}
        inline ngram_size_t parent(void) const { return _parent; }
        inline dependency_index_t dependency(void) const { return _dependency; }
#else
    public:
        Member(string_index_t lemma, tag_index_t tag): _tagged_lemma(lemma, tag) {}
#endif

        inline string_index_t lemma(void) const { return _tagged_lemma.lemma(); }
        inline tag_index_t tag(void) const { return _tagged_lemma.tag(); }
    };
private:
    typedef Overflow<const NGram *, frequency_t> _overflows_map_t;


    static const freq_counter_t _max_frequency;
    static const ngram_type_t _overflow_flag_mask;
    static const size_t _last_bit_index;
    static const ngram_type_t _type_mask;


    static bool _context_tracing_on;        
    static ngram_type_t _full_ngram_type;
    static std::vector<ngram_size_t>  _members_count;
    static std::vector<size_t> _ngrams_memory_sizes;
    static _overflows_map_t _overflows;
    static ngram_type_t _types_count;


    union {
        freq_counter_t _frequency;
        ngram_type_t _frequency_bits[sizeof(freq_counter_t)/sizeof(ngram_type_t)];
    };

    /* No assignments are allowed - method body must stay undefined.
     */
    NGram& operator=(const NGram& source);


    inline bool _overflow(void) const { return (_frequency_bits[_last_bit_index] & _overflow_flag_mask) != 0; }
    void _type(ngram_type_t ngram_type);
public:
    static void init(void);

    NGram(void): _frequency(0) {}
    NGram(const raw_ngram_t& raw_ngram, ngram_type_t ngram_type);
    NGram(const NGram& source);
    NGram(const NGram& source, ngram_type_t ngram_type);

    inline ngram_size_t degree() const { return _members_count[type()]; }
    static ngram_type_t full_ngram_type(void) { return _full_ngram_type; }
    inline bool is_zero(void) const { return type() == 0; }
    static ngram_size_t n(void) { return settings::n; }
    static size_t size_of(ngram_size_t degree) { return _ngrams_memory_sizes[degree]; }
    inline size_t size_of(void) const { return _ngrams_memory_sizes[degree()]; }
    inline ngram_type_t type(void) const { return _frequency_bits[_last_bit_index] & _type_mask; }
    static ngram_type_t types_count(void) { return _types_count; }

    inline Member * begin(void) { return static_cast<Member *>(static_cast<void *>(this + 1)); }
    inline const Member * begin(void) const { return static_cast<const Member *>(static_cast<const void *>(this + 1)); }
    inline Member * end(void) { return this->begin() + degree(); }
    inline const Member * end(void) const { return this->begin() + degree(); }
    inline Member * get(ngram_size_t index) { return this->begin() + index; }
    inline const Member * get(ngram_size_t index) const { return this->begin() + index; }
    frequency_t frequency(void) const {
        // A + B may oveflow, so special treatment is needed.
        // A - stored (already) partly-overflowen counter value.
        // B - actual counter value.
        return _overflows_map_t::add_to_maximum((_overflow() ? _overflows.get(this) : 0), (_frequency & _max_frequency));
    }
    void inc(void);
    inline string_index_t lemma(ngram_size_t i) const { return this->get(i)->lemma(); }
    inline tag_index_t tag(ngram_size_t i) const { return this->get(i)->tag(); }
#ifdef _DEPENDENCY_MODE

    inline ngram_size_t parent(ngram_size_t i) const { return this->get(i)->parent(); }
    inline dependency_index_t dependency(ngram_size_t i) const { return this->get(i)->dependency(); }
#endif
    // Context related stuff.

    void add_to_context(const PartOfContext& part_of_context);
    inline context_t * context(void) {
        return static_cast<context_t *>(static_cast<void *>(this->end()));
    }
    inline const context_t * context(void) const {
        return static_cast<const context_t *>(static_cast<const void *>(this->end()));
    }

    friend class NGramToken;
};

bool operator==(const NGram& lhs, const NGram& rhs);
bool operator<(const NGram& lhs, const NGram& rhs);


#if defined(_MSC_VER) || defined(__GNUC__)
// Restore default pack value.
#pragma pack(pop)
#endif

class NGramToken {
    static void *_ngram_space;
public:
    static void init(void);
    static NGram * get(const raw_ngram_t& raw_ngram, ngram_type_t ngram_type);
    static NGram * get(const NGram *source, ngram_type_t ngram_type);
};

} // namespace ace

#endif

---