proto/Source/processor.cpp

Go to the documentation of this file.

#include <fstream>
#include <string>

// Extractor interface
#include "extractor.h"
// Notification
#include "notifier.h"
// Overall stats storage
#include "persistent.h"
// Slef header file
#include "processor.h"
// Stats for files
#include "stats.h"

namespace ace {

unsigned processing_progress_precision = 4;

/*  Interface item implementation - for info see description in header file.
 */
void process(filenames_t& files_to_process, size_t files_total_size) {
    
    //
    std::ifstream input_file;

    // 
    size_t processed_so_far = 0, file_size = 0;

    //
    notifier::progress.info("Processing input datafiles.");

    for ( filenames_t::const_iterator i_file = files_to_process.begin(); i_file != files_to_process.end(); ++i_file ) {
        
        
        input_file.open(i_file->c_str());

        if ( !input_file ) {
            notifier::error.warning("Input datafile: " + *i_file + " cannot be opened for reading! Skipping...");
            continue;
        }


        // Stats for current file.
        NamedDataFileStats file_stats(*i_file);

        // Show time elapsed.
        notifier::progress.info("Processing datafile: " + *i_file);

        // Error message. If empty -> no error.
        std::string error_message;
        
        try {
            // Extract.
            file_size = extract(input_file, file_stats);
        } catch ( std::bad_alloc& ) {
            error_message = "Program has met with out-of-memory condition while processing datafile: " + *i_file + "!";
        } catch ( index_overflow& ) {
            if ( persistent::lemmas.is_full() ) {
                error_message = std::string("Number of unique strings in lemmas store has overflowen numeric index type range!");
            } else if ( persistent::tags.is_full() ) {
                error_message = std::string("Number of unique strings in morphologic tags store has overflowen numeric index type range!");
            } else {
                error_message = std::string("Undetermined index overflow exception!");
            }
        }
        
        // Close the file!
        input_file.close();

        if ( !error_message.empty() ) {
            // What a pity.
            throw fatal_error(error_message);
        }

        // Store the stats.
        persistent::overall_stats.push_back(file_stats);

        if ( persistent::morphologic_filter.is_on() ) {
            // Store the morphologic filter stats...
            persistent::morphologic_filter_file_stats.insert(morphologic_filter_file_stats_t::value_type(*i_file, persistent::morphologic_filter.recent_stats()));
            // ...and reset (make clear for next file).
            persistent::morphologic_filter.reset_recent_stats();
        }

        // Show time elapsed.
        notifier::progress.elapsed("Finished processing datafile: " + *i_file + " (" + unsigned2str(file_size) + " bytes)");

        if ( files_total_size ) {
            // Show file processing progress.
            processed_so_far += file_size;
            notifier::progress.info("Overall processing progress: " + double2str(static_cast<double>(processed_so_far)/static_cast<double>(files_total_size) * 100, processing_progress_precision) + "%");
        }

    }

    // Elapsed:
    notifier::progress.elapsed("Finished processing input datafiles.");

#ifndef _USE_HASHSET
    if ( settings::sort ) {
        notifier::progress.info("Sorting N-grams by their frequency.");
        // Sort all hash tables except the one for full N-gram type (each full N-gram is accessed only one).
        for ( ngram_type_t type = 1; type < NGram::full_ngram_type(); ++type ) {
            ace::persistent::ngrams.sort(type);
        }
        notifier::progress.elapsed("Finished sorting N-grams by their frequency.");
    }
#endif

}

} // namespace ace

---