/** @file processor.cpp
 *
 *	Definition (implementation) of processing module interface.
 *
 *	See processor.h for more info.
 *
 *	(C) Ceslav Przywara 2008, MFF UK Prague
 *
 */

#include <fstream>
#include <string>

// Extractor interface
#include "extractor.h"
// Notification
#include "notifier.h"
// Overall stats storage
#include "persistent.h"
// Slef header file
#include "processor.h"
// Stats for files
#include "stats.h"

namespace ace {

unsigned processing_progress_precision = 4;

/*	Interface item implementation - for info see description in header file.
 */
void process(filenames_t& files_to_process, size_t files_total_size) {
	
	//
	std::ifstream input_file;

	// 
	size_t processed_so_far = 0, file_size = 0;

	//
	notifier::progress.info("Processing input datafiles.");

	for ( filenames_t::const_iterator i_file = files_to_process.begin(); i_file != files_to_process.end(); ++i_file ) {
		
		//// File opening ////
		
		input_file.open(i_file->c_str());

		if ( !input_file ) {
			notifier::error.warning("Input datafile: " + *i_file + " cannot be opened for reading! Skipping...");
			continue;
		}

		//// File parsing and extraction ////

		// Stats for current file.
		NamedDataFileStats file_stats(*i_file);

		// Show time elapsed.
		notifier::progress.info("Processing datafile: " + *i_file);

		// Error message. If empty -> no error.
		std::string error_message;
		
		try {
			// Extract.
			file_size = extract(input_file, file_stats);
		} catch ( std::bad_alloc& ) {
			error_message = "Program has met with out-of-memory condition while processing datafile: " + *i_file + "!";
		} catch ( index_overflow& ) {
			if ( persistent::lemmas.is_full() ) {
				error_message = std::string("Number of unique strings in lemmas store has overflowen numeric index type range!");
			} else if ( persistent::tags.is_full() ) {
				error_message = std::string("Number of unique strings in morphologic tags store has overflowen numeric index type range!");
			} else {
				error_message = std::string("Undetermined index overflow exception!");
			}
		}
		
		// Close the file!
		input_file.close();

		if ( !error_message.empty() ) {
			// What a pity.
			throw fatal_error(error_message);
		}

		// Store the stats.
		persistent::overall_stats.push_back(file_stats);

		if ( persistent::morphologic_filter.is_on() ) {
			// Store the morphologic filter stats...
			persistent::morphologic_filter_file_stats.insert(morphologic_filter_file_stats_t::value_type(*i_file, persistent::morphologic_filter.recent_stats()));
			// ...and reset (make clear for next file).
			persistent::morphologic_filter.reset_recent_stats();
		}

		// Show time elapsed.
		notifier::progress.elapsed("Finished processing datafile: " + *i_file + " (" + unsigned2str(file_size) + " bytes)");

		if ( files_total_size ) {
			// Show file processing progress.
			processed_so_far += file_size;
			notifier::progress.info("Overall processing progress: " + double2str(static_cast<double>(processed_so_far)/static_cast<double>(files_total_size) * 100, processing_progress_precision) + "%");
		}

	}

	// Elapsed:
	notifier::progress.elapsed("Finished processing input datafiles.");

#ifndef _USE_HASHSET
	if ( settings::sort ) {
		notifier::progress.info("Sorting N-grams by their frequency.");
		// Sort all hash tables except the one for full N-gram type (each full N-gram is accessed only one).
		for ( ngram_type_t type = 1; type < NGram::full_ngram_type(); ++type ) {
			ace::persistent::ngrams.sort(type);
		}
		notifier::progress.elapsed("Finished sorting N-grams by their frequency.");
	}
#endif

}

} // namespace ace
