/** @file writer.cpp
 *
 *	Definition (implementation) of (output) writing module interface.
 *
 *	See writer.h for more info.
 *
 *	(C) Ceslav Przywara 2008, MFF UK Prague
 *
 */

#include <algorithm>
#include <iomanip>

// Settings
#include "config.h"
#ifdef _DEPENDENCY_MODE
// Dependencies output (code -> string)
#include "dependencies.h"
#endif
// Evaluation of statistics
#include "evaluation.h"
// Output of morphologic filtration statistics
#include "filter.h"
// N-grams are to output.
#include "ngrams.h"
// Progress display.
#include "notifier.h"
// Access to persistent stores.
#include "persistent.h"
// unsigned2str
#include "utils.h"
// Self header
#include "writer.h"

namespace ace {

/** Container for scores and statistics thresholds.
 */
typedef std::vector<double> scores_t;

/** CSV format separator.
 */
const char fields_separator = '\t';

// We use eol constant abudantly.
using constants::eol;

/** Helper procedure - prints given N-gram member to given output stream.
 *	@param output Reference to output stream.
 *	@param index Member index within the N-gram members.
 *	@param member Pointer to const instance of NGram member.
 */
void _print_member(std::ostream& output, ngram_size_t index, const NGram::Member * member) {
	output
		<< static_cast<unsigned>(index)
		<< fields_separator
		<< persistent::lemmas.get(member->lemma())
		<< fields_separator
		<< persistent::tags.get(member->tag())
#ifdef _DEPENDENCY_MODE
		<< fields_separator
		<< static_cast<unsigned>(member->parent())
		<< fields_separator
		<< dependency2cstring(member->dependency())
#endif
		<< eol;
}

/** Helper procedure - prints given N-gram members to given output stream.
 *	@param output Reference to output stream.
 *	@param ngram Pointer to const instance of NGram.
 */
void _print_members(std::ostream& output, const NGram *ngram) {
	ngram_size_t index = 1;
	for ( const NGram::Member * member = ngram->begin(); member != ngram->end(); ++member, ++index ) {
		_print_member(output, index, member);
	}
}

/** Helper procedure - prints given N-gram members to given output stream.
 *	Procedure prints all N members, even if given N-gram is not full-typed
 *	(* is printed in place of missing member).
 *	@param output Reference to output stream.
 *	@param ngram Pointer to const instance of NGram.
 */
void _print_n_members(std::ostream& output, const NGram *ngram) {
	ngram_size_t index = 1;
	// Get first member.
	const NGram::Member *member = ngram->begin();
	
	for ( ngram_type_t i = NGram::n(); i > 0; ++index /* i gets decremented inside loop body */ ) {	
		if ( ith_bit(ngram->type(), --i) ) {
			_print_member(output, index, member);
			++member;
		} else {
			// "Empty" member.
			output << static_cast<unsigned>(index) << fields_separator << "*" << eol;
		}
	}
}

/** Helper procedure - prints certain N-gram evaluation tables to given output stream.
 *	Note: Debug version prints all tables, Release version only contigency table.
 *	@param output Reference to output stream.
 *	@param tables Reference to const instance EvaluationTables.
 */
void _print_tables(std::ostream& output, const EvaluationTables& tables) {
#ifdef _DEBUG
	// Debug version is more verbose.
	static const size_t width = 18;
	output << "    " << std::right
		<< std::setw(width) << "Frequency:"
		<< std::setw(width)	<< "Contingent freq.:"
		<< std::setw(width) << "Inverted freq.:"
		<< std::setw(width) << "Expected c. fr.:"
		<< eol;

	for ( size_t i = 0; i < tables.size(); ++i ) {
		output << std::setw(3) << i << ":"
			<< std::setw(width) << tables.frequency()[i]
			<< std::setw(width) << tables.contigency()[i]
			<< std::setw(width) << tables.inverted_frequency()[i]
			<< std::setw(width) << tables.expected_frequency()[i]
			<< eol;
	}
#else

	for ( size_t i = 0; i < tables.contigency().size(); ++i ) {
		if ( i != 0 ) {
			output << fields_separator;
		}
		output << tables.contigency()[i];
	}
	output << eol;

#endif
}

/** Helper procedure - prints certain N-gram statistics to given output stream.
 *	Note: Debug version prints formatted statistics (including name). Release version
 *	prints statistics in single-row fashion.
 *	@param output Reference to output stream.
 *	@param scores Reference to const instance of statistics (scores) table.
 */
void _print_stats(std::ostream& output, const scores_t& scores) {
#ifdef _DEBUG
	// Debug version is more verbose.
	static const size_t width = 38;
	output << std::right
		<< std::setw(width) << "Pearson's chi square test:" 
		<< std::setw(width)	<< "Log likelihood ratio:"
		<< eol
		<< std::setw(width) << scores[0]
		<< std::setw(width) << scores[1]
		<< eol;

	if ( scores.size() > 2 ) {
		output
			<< std::setw(width) << "Pointwise mutual information:"
			<< std::setw(width) << "Pearson's coefficient:"
			<< eol
			<< std::setw(width) << scores[2]
			<< std::setw(width) << scores[3]
			<< eol
			<< std::setw(width) << "Student's t-test:"
			<< std::setw(width) << "Z score:"
			<< eol
			<< std::setw(width) << scores[4]
			<< std::setw(width) << scores[5]
			<< eol;
	}
#else
	output << scores[0] << fields_separator << scores[1];

	if ( scores.size() > 2 ) {
		output
			<< fields_separator << scores[2] << fields_separator << scores[3]
			<< fields_separator << scores[4] << fields_separator << scores[5]
			;
	}

	output << eol;
#endif
}

//// (header file interface implementation) ////

/*	Interface item implementation - for info see description in header file.
 */
void write(std::ofstream& output_file) {
	
	// 2 evaluation scores are available for any N in general.
	scores_t scores(2);
	scores_t thresholds;
	std::vector<double (*)(const EvaluationTables&)> evaluators;
	//
	thresholds.push_back(settings::thresholds.chi_square_test); // 0
	thresholds.push_back(settings::thresholds.log_likelihood_ratio); // 1
	//
	evaluators.push_back(&eval::chi_square_test); // 0
	evaluators.push_back(&eval::log_likelihood_ratio); // 1
	
	// For N == 2 there are 4 more evaluations available.
	if ( settings::n == 2 ) {
		// Make room for six scores.
		scores.resize(6);
		//
		thresholds.push_back(settings::thresholds.mutual_information); // 2
		thresholds.push_back(settings::thresholds.pearsons_coefficient); // 3
		thresholds.push_back(settings::thresholds.t_test); // 4
		thresholds.push_back(settings::thresholds.z_score); // 5
		//	
		evaluators.push_back(&eval::mutual_information); // 2
		evaluators.push_back(&eval::pearsons_coefficient); // 3
		evaluators.push_back(&eval::t_test); // 4
		evaluators.push_back(&eval::z_score); // 5
	}

#ifdef _BOOST_ON
	// Progress display.
	progress_display_t display(static_cast<unsigned long>(persistent::ngrams.size()), notifier::progress.output_stream(), "");
#endif

	// Set statistics precision.
	output_file << std::setprecision(static_cast<std::streamsize>(settings::precision));

	// 
	NGramsProcessor ngrams_processor(persistent::ngrams);

	while ( ngrams_processor.next() ) {

#ifdef _BOOST_ON
		// Progress bar display update.
		++display;
#endif
		const NGramFrequencies& ngram_freq = ngrams_processor.ngram_frequencies();

		// Check frequency threshold here.
		if ( settings::sieves.frequency > ngram_freq.frequencies[NGram::full_ngram_type()]) {
			// You shall not pass!
			continue;
		}
		
		// Evaluate n-gram frequencies.
		EvaluationTables eval_tables(ngram_freq.frequencies);
		
		// Check estimated frequency threshold here.
		if ( settings::sieves.expected_frequency > eval_tables.expected_frequency()[NGram::full_ngram_type()] ) {
			// You shall not pass too!
			continue;
		}

		// Clear scores from previous loop!
		std::fill(scores.begin(), scores.end(), 0.0);

		// After the loop passed == true, in case of one of following:
		// 1. Any of thresholds will be trespassed and all_thresholds_together setting is false.
		// 2. All of thresholds will be trespassed.
		// 3. There are no score thresholds.
		// Similarly passed == false, in case of one of following:
		// 1. Any of thresholds will not be trespassed and all_thresholds_together setting is true.
		// 2. None of thresholds will be trespassed.
		bool passed = true;
		for ( size_t i = 0; i < evaluators.size(); ++i ) {
			// Count only scores needed for thresholds check.
			if ( thresholds[i] > 0.0 ) {
				//
				scores[i] = evaluators[i](eval_tables);
				if ( thresholds[i] > scores[i] ) {
					passed = false;
					if ( settings::all_thresholds_together ) {
						// One is enough to fail.
						break;
					}
				} else {
					passed = true;
					if ( !settings::all_thresholds_together ) {
						// One is enough to pass.
						break;
					}
				}
			}
		}

		if ( !passed ) {
			continue;
		}

		// Count missing scores.
		for ( size_t i = 0; i < scores.size(); ++i ) {
			if ( !(scores[i] > 0) ) {
				scores[i] = evaluators[i](eval_tables);
			}
		}

		// N-gram members.
		_print_members(output_file, ngram_freq.ngram);
		// Tables.
		_print_tables(output_file, eval_tables);
		// Scores.
		_print_stats(output_file, scores);
		// Separator.
		output_file << eol;
	}

	// Final flush.
	output_file << std::endl;
}

void _print_context(std::ofstream& output_file, const context_t * ctx, bool narrow = false) {
	for ( context_t::const_iterator i_ctx = ctx->begin(); i_ctx != ctx->end(); ++i_ctx ) {
		// Print narrow, temporary store wide.
		if ( (narrow && i_ctx->is_part_of_narrow()) || (!narrow && i_ctx->is_part_of_wide()) ) {
			// lemma tag frequency
			output_file
				<< persistent::lemmas.get(i_ctx->lemma()) << fields_separator
				<< persistent::tags.get(i_ctx->tag()) << fields_separator
				<< i_ctx->frequency() << eol;
		}
	}
	output_file << eol;
}

/*	Interface item implementation - for info see description in header file.
 */
void write_contexts(std::ofstream& output_file) {
	// 
	size_t ngrams_to_process = 0;	
	for ( ngram_type_t type = NGram::full_ngram_type(); type > 0; --type ) {
		ngrams_to_process += persistent::ngrams.size(type);
	}

#ifdef _BOOST_ON
	// Progress display.
	progress_display_t display(static_cast<unsigned long>(ngrams_to_process), notifier::progress.output_stream(), "");
#endif

	// Print context.
	// Loop through all types except zero...
	for ( ngram_type_t type = NGram::full_ngram_type(); type > 0; --type ) {
		// ...then loop through all stored NGram(s) of given type...
		for ( NGramStore::hash_table_t::const_iterator i_ngram = persistent::ngrams.begin(type); i_ngram != persistent::ngrams.end(type); ++i_ngram ) {
			// Print members.
			_print_n_members(output_file, *i_ngram);
			// Space between head (N-gram definition) and body (N-gram context).
			output_file << eol;
			if ( (*i_ngram)->type() == NGram::full_ngram_type() ) {
				// Print narrow.
				_print_context(output_file, (*i_ngram)->context(), true);
			}
			// Print wide.
			_print_context(output_file, (*i_ngram)->context(), false);
			// Space between N-grams.
			output_file << eol;

#ifdef _BOOST_ON
			// Update display.
			++display;
#endif
		}
	}
	// Final flush.
	output_file << std::endl;
}


/*	Interface item implementation - for info see description in header file.
 */
void write_stats(std::ofstream& output_file) {

	DataFileStats overall;

	for ( stats_t::const_iterator i_stats = persistent::overall_stats.begin(); i_stats != persistent::overall_stats.end(); ++i_stats) {

		output_file
			<< "Statistics for datafile " << i_stats->filename << ":" << eol
			<< "# of words: " << i_stats->words << eol
			<< "# of sentences: " << i_stats->sentences << eol
			<< "# of " << static_cast<unsigned>(settings::n) <<  "-grams extracted: " << i_stats->ngrams.extracted << eol
			;
		if ( persistent::morphologic_filter.is_on() ) {
			output_file
				<< "# of " << static_cast<unsigned>(settings::n) <<  "-grams passed: " << i_stats->ngrams.passed << eol
				<< "# of " << static_cast<unsigned>(settings::n) <<  "-grams filtered: " << i_stats->ngrams.filtered << eol
				;
		}		
		output_file	<< eol;
		
		// This is situation to use std::accumulate, but devil knows why it is using:
		// "init_value = init_value + elem;" instead of simply: "init_value += elem;"
		overall += *i_stats;
	}

	output_file
		<< "Overall datafiles statistics:" << eol
		<< "# of words: " << overall.words << eol
		<< "# of sentences: " << overall.sentences << eol
		<< "# of " << static_cast<unsigned>(settings::n) <<  "-grams extracted: " << overall.ngrams.extracted << eol
		;
	if ( persistent::morphologic_filter.is_on() ) {
		output_file
			<< "# of " << static_cast<unsigned>(settings::n) <<  "-grams passed: " << overall.ngrams.passed << eol
			<< "# of " << static_cast<unsigned>(settings::n) <<  "-grams filtered: " << overall.ngrams.filtered << eol
			;
	}
	output_file << eol;

	output_file
		<< "Strings storage statistics:" << eol
		<< "# of unique lemmas: " << persistent::lemmas.size() << eol
		<< "# of unique tags: " << persistent::tags.size() << eol
		<< eol
		<< static_cast<unsigned>(settings::n) <<  "-grams storage statistics:" << eol;
	for ( ngram_type_t type = 0; type <= NGram::full_ngram_type(); ++type ) {
		output_file << "# of unique " << static_cast<unsigned>(settings::n) <<  "-grams of type " << static_cast<unsigned>(type) << ": " << persistent::ngrams.size(type) << eol;
	}
	
	// Flush.
	output_file << std::endl;

}

/*	Interface item implementation - for info see description in header file.
 */
void write_morphologic_filter_stats(std::ofstream& output_file) {

	const MultiRulesFilter::rules_t& rules = persistent::morphologic_filter.get_rules();

	for ( morphologic_filter_file_stats_t::const_iterator i_stats = persistent::morphologic_filter_file_stats.begin(); i_stats != persistent::morphologic_filter_file_stats.end(); ++i_stats) {

		output_file << "Passed " << static_cast<unsigned>(settings::n) << "-grams statistics for datafile " << i_stats->first << ":" << eol;

		const Filter::Stats& stats = i_stats->second;
		
		for ( size_t i = 0; i < rules.size(); ++i ) {
			// Write down the rule.
			for ( size_t j = 0; j < rules[i].size(); ++j ) {
				output_file << ' ' << rules[i][j];
			}
			// Write down appropriate stats.
			output_file << fields_separator << stats.matched[i] << eol;
		}

		output_file	<< eol;		

	}

	output_file << "Overall passed " << static_cast<unsigned>(settings::n) << "-grams statistics:" << eol;
	for ( size_t i = 0; i < rules.size(); ++i ) {
		// Write down the rule.
		for ( size_t j = 0; j < rules[i].size(); ++j ) {
			output_file << ' ' << rules[i][j];
		}
		// Write down appropriate stats.
		output_file << fields_separator << persistent::morphologic_filter.stats_so_far().matched[i] << eol;
	}

	// Flush.
	output_file << std::endl;

}

} // namespace ace
