proto/Source/writer.cpp

Go to the documentation of this file.

#include <algorithm>
#include <iomanip>

// Settings
#include "config.h"
#ifdef _DEPENDENCY_MODE
// Dependencies output (code -> string)
#include "dependencies.h"
#endif
// Evaluation of statistics
#include "evaluation.h"
// Output of morphologic filtration statistics
#include "filter.h"
// N-grams are to output.
#include "ngrams.h"
// Progress display.
#include "notifier.h"
// Access to persistent stores.
#include "persistent.h"
// unsigned2str
#include "utils.h"
// Self header
#include "writer.h"

namespace ace {

typedef std::vector<double> scores_t;

const char fields_separator = '\t';

// We use eol constant abudantly.
using constants::eol;

void _print_member(std::ostream& output, ngram_size_t index, const NGram::Member * member) {
    output
        << static_cast<unsigned>(index)
        << fields_separator
        << persistent::lemmas.get(member->lemma())
        << fields_separator
        << persistent::tags.get(member->tag())
#ifdef _DEPENDENCY_MODE
        << fields_separator
        << static_cast<unsigned>(member->parent())
        << fields_separator
        << dependency2cstring(member->dependency())
#endif
        << eol;
}

void _print_members(std::ostream& output, const NGram *ngram) {
    ngram_size_t index = 1;
    for ( const NGram::Member * member = ngram->begin(); member != ngram->end(); ++member, ++index ) {
        _print_member(output, index, member);
    }
}

void _print_n_members(std::ostream& output, const NGram *ngram) {
    ngram_size_t index = 1;
    // Get first member.
    const NGram::Member *member = ngram->begin();
    
    for ( ngram_type_t i = NGram::n(); i > 0; ++index /* i gets decremented inside loop body */ ) { 
        if ( ith_bit(ngram->type(), --i) ) {
            _print_member(output, index, member);
            ++member;
        } else {
            // "Empty" member.
            output << static_cast<unsigned>(index) << fields_separator << "*" << eol;
        }
    }
}

void _print_tables(std::ostream& output, const EvaluationTables& tables) {
#ifdef _DEBUG
    // Debug version is more verbose.
    static const size_t width = 18;
    output << "    " << std::right
        << std::setw(width) << "Frequency:"
        << std::setw(width) << "Contingent freq.:"
        << std::setw(width) << "Inverted freq.:"
        << std::setw(width) << "Expected c. fr.:"
        << eol;

    for ( size_t i = 0; i < tables.size(); ++i ) {
        output << std::setw(3) << i << ":"
            << std::setw(width) << tables.frequency()[i]
            << std::setw(width) << tables.contigency()[i]
            << std::setw(width) << tables.inverted_frequency()[i]
            << std::setw(width) << tables.expected_frequency()[i]
            << eol;
    }
#else

    for ( size_t i = 0; i < tables.contigency().size(); ++i ) {
        if ( i != 0 ) {
            output << fields_separator;
        }
        output << tables.contigency()[i];
    }
    output << eol;

#endif
}

void _print_stats(std::ostream& output, const scores_t& scores) {
#ifdef _DEBUG
    // Debug version is more verbose.
    static const size_t width = 38;
    output << std::right
        << std::setw(width) << "Pearson's chi square test:" 
        << std::setw(width) << "Log likelihood ratio:"
        << eol
        << std::setw(width) << scores[0]
        << std::setw(width) << scores[1]
        << eol;

    if ( scores.size() > 2 ) {
        output
            << std::setw(width) << "Pointwise mutual information:"
            << std::setw(width) << "Pearson's coefficient:"
            << eol
            << std::setw(width) << scores[2]
            << std::setw(width) << scores[3]
            << eol
            << std::setw(width) << "Student's t-test:"
            << std::setw(width) << "Z score:"
            << eol
            << std::setw(width) << scores[4]
            << std::setw(width) << scores[5]
            << eol;
    }
#else
    output << scores[0] << fields_separator << scores[1];

    if ( scores.size() > 2 ) {
        output
            << fields_separator << scores[2] << fields_separator << scores[3]
            << fields_separator << scores[4] << fields_separator << scores[5]
            ;
    }

    output << eol;
#endif
}


/*  Interface item implementation - for info see description in header file.
 */
void write(std::ofstream& output_file) {
    
    // 2 evaluation scores are available for any N in general.
    scores_t scores(2);
    scores_t thresholds;
    std::vector<double (*)(const EvaluationTables&)> evaluators;
    //
    thresholds.push_back(settings::thresholds.chi_square_test); // 0
    thresholds.push_back(settings::thresholds.log_likelihood_ratio); // 1
    //
    evaluators.push_back(&eval::chi_square_test); // 0
    evaluators.push_back(&eval::log_likelihood_ratio); // 1
    
    // For N == 2 there are 4 more evaluations available.
    if ( settings::n == 2 ) {
        // Make room for six scores.
        scores.resize(6);
        //
        thresholds.push_back(settings::thresholds.mutual_information); // 2
        thresholds.push_back(settings::thresholds.pearsons_coefficient); // 3
        thresholds.push_back(settings::thresholds.t_test); // 4
        thresholds.push_back(settings::thresholds.z_score); // 5
        //  
        evaluators.push_back(&eval::mutual_information); // 2
        evaluators.push_back(&eval::pearsons_coefficient); // 3
        evaluators.push_back(&eval::t_test); // 4
        evaluators.push_back(&eval::z_score); // 5
    }

#ifdef _BOOST_ON
    // Progress display.
    progress_display_t display(static_cast<unsigned long>(persistent::ngrams.size()), notifier::progress.output_stream(), "");
#endif

    // Set statistics precision.
    output_file << std::setprecision(static_cast<std::streamsize>(settings::precision));

    // 
    NGramsProcessor ngrams_processor(persistent::ngrams);

    while ( ngrams_processor.next() ) {

#ifdef _BOOST_ON
        // Progress bar display update.
        ++display;
#endif
        const NGramFrequencies& ngram_freq = ngrams_processor.ngram_frequencies();

        // Check frequency threshold here.
        if ( settings::sieves.frequency > ngram_freq.frequencies[NGram::full_ngram_type()]) {
            // You shall not pass!
            continue;
        }
        
        // Evaluate n-gram frequencies.
        EvaluationTables eval_tables(ngram_freq.frequencies);
        
        // Check estimated frequency threshold here.
        if ( settings::sieves.expected_frequency > eval_tables.expected_frequency()[NGram::full_ngram_type()] ) {
            // You shall not pass too!
            continue;
        }

        // Clear scores from previous loop!
        std::fill(scores.begin(), scores.end(), 0.0);

        // After the loop passed == true, in case of one of following:
        // 1. Any of thresholds will be trespassed and all_thresholds_together setting is false.
        // 2. All of thresholds will be trespassed.
        // 3. There are no score thresholds.
        // Similarly passed == false, in case of one of following:
        // 1. Any of thresholds will not be trespassed and all_thresholds_together setting is true.
        // 2. None of thresholds will be trespassed.
        bool passed = true;
        for ( size_t i = 0; i < evaluators.size(); ++i ) {
            // Count only scores needed for thresholds check.
            if ( thresholds[i] > 0.0 ) {
                //
                scores[i] = evaluators[i](eval_tables);
                if ( thresholds[i] > scores[i] ) {
                    passed = false;
                    if ( settings::all_thresholds_together ) {
                        // One is enough to fail.
                        break;
                    }
                } else {
                    passed = true;
                    if ( !settings::all_thresholds_together ) {
                        // One is enough to pass.
                        break;
                    }
                }
            }
        }

        if ( !passed ) {
            continue;
        }

        // Count missing scores.
        for ( size_t i = 0; i < scores.size(); ++i ) {
            if ( !(scores[i] > 0) ) {
                scores[i] = evaluators[i](eval_tables);
            }
        }

        // N-gram members.
        _print_members(output_file, ngram_freq.ngram);
        // Tables.
        _print_tables(output_file, eval_tables);
        // Scores.
        _print_stats(output_file, scores);
        // Separator.
        output_file << eol;
    }

    // Final flush.
    output_file << std::endl;
}

void _print_context(std::ofstream& output_file, const context_t * ctx, bool narrow = false) {
    for ( context_t::const_iterator i_ctx = ctx->begin(); i_ctx != ctx->end(); ++i_ctx ) {
        // Print narrow, temporary store wide.
        if ( (narrow && i_ctx->is_part_of_narrow()) || (!narrow && i_ctx->is_part_of_wide()) ) {
            // lemma tag frequency
            output_file
                << persistent::lemmas.get(i_ctx->lemma()) << fields_separator
                << persistent::tags.get(i_ctx->tag()) << fields_separator
                << i_ctx->frequency() << eol;
        }
    }
    output_file << eol;
}

/*  Interface item implementation - for info see description in header file.
 */
void write_contexts(std::ofstream& output_file) {
    // 
    size_t ngrams_to_process = 0;   
    for ( ngram_type_t type = NGram::full_ngram_type(); type > 0; --type ) {
        ngrams_to_process += persistent::ngrams.size(type);
    }

#ifdef _BOOST_ON
    // Progress display.
    progress_display_t display(static_cast<unsigned long>(ngrams_to_process), notifier::progress.output_stream(), "");
#endif

    // Print context.
    // Loop through all types except zero...
    for ( ngram_type_t type = NGram::full_ngram_type(); type > 0; --type ) {
        // ...then loop through all stored NGram(s) of given type...
        for ( NGramStore::hash_table_t::const_iterator i_ngram = persistent::ngrams.begin(type); i_ngram != persistent::ngrams.end(type); ++i_ngram ) {
            // Print members.
            _print_n_members(output_file, *i_ngram);
            // Space between head (N-gram definition) and body (N-gram context).
            output_file << eol;
            if ( (*i_ngram)->type() == NGram::full_ngram_type() ) {
                // Print narrow.
                _print_context(output_file, (*i_ngram)->context(), true);
            }
            // Print wide.
            _print_context(output_file, (*i_ngram)->context(), false);
            // Space between N-grams.
            output_file << eol;

#ifdef _BOOST_ON
            // Update display.
            ++display;
#endif
        }
    }
    // Final flush.
    output_file << std::endl;
}


/*  Interface item implementation - for info see description in header file.
 */
void write_stats(std::ofstream& output_file) {

    DataFileStats overall;

    for ( stats_t::const_iterator i_stats = persistent::overall_stats.begin(); i_stats != persistent::overall_stats.end(); ++i_stats) {

        output_file
            << "Statistics for datafile " << i_stats->filename << ":" << eol
            << "# of words: " << i_stats->words << eol
            << "# of sentences: " << i_stats->sentences << eol
            << "# of " << static_cast<unsigned>(settings::n) <<  "-grams extracted: " << i_stats->ngrams.extracted << eol
            ;
        if ( persistent::morphologic_filter.is_on() ) {
            output_file
                << "# of " << static_cast<unsigned>(settings::n) <<  "-grams passed: " << i_stats->ngrams.passed << eol
                << "# of " << static_cast<unsigned>(settings::n) <<  "-grams filtered: " << i_stats->ngrams.filtered << eol
                ;
        }       
        output_file << eol;
        
        // This is situation to use std::accumulate, but devil knows why it is using:
        // "init_value = init_value + elem;" instead of simply: "init_value += elem;"
        overall += *i_stats;
    }

    output_file
        << "Overall datafiles statistics:" << eol
        << "# of words: " << overall.words << eol
        << "# of sentences: " << overall.sentences << eol
        << "# of " << static_cast<unsigned>(settings::n) <<  "-grams extracted: " << overall.ngrams.extracted << eol
        ;
    if ( persistent::morphologic_filter.is_on() ) {
        output_file
            << "# of " << static_cast<unsigned>(settings::n) <<  "-grams passed: " << overall.ngrams.passed << eol
            << "# of " << static_cast<unsigned>(settings::n) <<  "-grams filtered: " << overall.ngrams.filtered << eol
            ;
    }
    output_file << eol;

    output_file
        << "Strings storage statistics:" << eol
        << "# of unique lemmas: " << persistent::lemmas.size() << eol
        << "# of unique tags: " << persistent::tags.size() << eol
        << eol
        << static_cast<unsigned>(settings::n) <<  "-grams storage statistics:" << eol;
    for ( ngram_type_t type = 0; type <= NGram::full_ngram_type(); ++type ) {
        output_file << "# of unique " << static_cast<unsigned>(settings::n) <<  "-grams of type " << static_cast<unsigned>(type) << ": " << persistent::ngrams.size(type) << eol;
    }
    
    // Flush.
    output_file << std::endl;

}

/*  Interface item implementation - for info see description in header file.
 */
void write_morphologic_filter_stats(std::ofstream& output_file) {

    const MultiRulesFilter::rules_t& rules = persistent::morphologic_filter.get_rules();

    for ( morphologic_filter_file_stats_t::const_iterator i_stats = persistent::morphologic_filter_file_stats.begin(); i_stats != persistent::morphologic_filter_file_stats.end(); ++i_stats) {

        output_file << "Passed " << static_cast<unsigned>(settings::n) << "-grams statistics for datafile " << i_stats->first << ":" << eol;

        const Filter::Stats& stats = i_stats->second;
        
        for ( size_t i = 0; i < rules.size(); ++i ) {
            // Write down the rule.
            for ( size_t j = 0; j < rules[i].size(); ++j ) {
                output_file << ' ' << rules[i][j];
            }
            // Write down appropriate stats.
            output_file << fields_separator << stats.matched[i] << eol;
        }

        output_file << eol;     

    }

    output_file << "Overall passed " << static_cast<unsigned>(settings::n) << "-grams statistics:" << eol;
    for ( size_t i = 0; i < rules.size(); ++i ) {
        // Write down the rule.
        for ( size_t j = 0; j < rules[i].size(); ++j ) {
            output_file << ' ' << rules[i][j];
        }
        // Write down appropriate stats.
        output_file << fields_separator << persistent::morphologic_filter.stats_so_far().matched[i] << eol;
    }

    // Flush.
    output_file << std::endl;

}

} // namespace ace

---