proto/Source/parser.cpp

Go to the documentation of this file.

// Base settings.
#include "config.h"
#ifdef _DEPENDENCY_MODE
// Need of string2dependency()
#include "dependencies.h"
#endif
// Progress display, warnings display.
#include "notifier.h"
// Self header.
#include "parser.h"
// Persistent store access (strings).
#include "persistent.h"
// Need of unsigned2str()
#include "utils.h"

namespace ace {

/*  Interface item implementation - for info see description in header file.
 */
bool Parser::next(size_t count, ContextWindow::Mode mode, words_store_t& store) {
    // Items, which don't need to be defined within declaration may safely be static.
    // To read (check) first char of the line.
    static int c;
    // To gather parsed information following variables are needed.
    static std::string lemma;
    static std::string tag;
    static std::string context_tag;
#ifdef _DEPENDENCY_MODE
    static std::string occurrence;
    static std::string dependency;
    // Must be unsigned int, because text input passed to unsigned char is not interpreted as number by operator >>.
    static unsigned int order;  
    static unsigned int parent;
#endif
    // Counters of words parsed: for last sentence and for single function run.
    size_t words_recently_parsed = 0, words_totally_parsed = 0;
    bool incorrect_input_format_detected = false;   

    // Clear last run results
    _parsed_counter.clear();

    while ( _input_file ) {
        
        // Get first char.
        c = _input_file.get();
        
        if ( c < 0 ) { // end-of-file
            // Jump out.
            break;
        }

        // Count line.
        ++_line_no;

        if ( c != constants::eol ) {
            // Not the separator line, unget previously retrieved char, will be needed.
            _input_file.unget();
#ifdef _DEPENDENCY_MODE
            // Format of the line is as following:
            _input_file >> order >> occurrence >> lemma >> tag >> parent >> dependency;
#else
            // Format of the line is much simpler than above:
            _input_file >> lemma >> tag;
#endif
            if ( _input_file.fail() && !_input_file.eof() ) {
                // Some error occured.
                incorrect_input_format_detected = true;
                _input_file.clear();
            }

            // Skip all characters until the end-of-line.
            // If the input file is in correct format, the end-of-line character will be
            // the right next one in stream buffer.
            _input_file.ignore(1, constants::eol);
#ifdef _DEPENDENCY_MODE
            if ( incorrect_input_format_detected || (order != (words_recently_parsed + 1)) || (words_recently_parsed == constants::max_sentence_length) ) {
#else
            if ( incorrect_input_format_detected || (words_recently_parsed == constants::max_sentence_length) ) {
#endif
                // Some error occured, print message.
                if ( incorrect_input_format_detected ) {                
                    notifier::error.warning("Incorrect input format detected at line " + unsigned2str(_line_no) + ". Skipping current sentence...");
                    // Reset flag.
                    incorrect_input_format_detected = false;
                } else if ( words_recently_parsed == constants::max_sentence_length ) {
                    // Word order check overflow.
                    notifier::error.warning("Word order overflow detected at line " + unsigned2str(_line_no) + ". Got word ordered as " + unsigned2str(words_recently_parsed + 1) + ", while maximum allowed value for word order is " + unsigned2str(constants::max_sentence_length) + ". Skipping current sentence...");
#ifdef _DEPENDENCY_MODE
                } else {                
                    // Mismatch. 
                    notifier::error.warning("Word order mismatch detected at line " + unsigned2str(_line_no) + ". Got word ordered as " + unsigned2str(order) + ", while expecting " + unsigned2str(words_recently_parsed + 1) + ". Skipping current sentence...");
#endif
                }

                // Roll back current sentence.
                for ( ; words_recently_parsed > 0; --words_recently_parsed ) {
                    store.pop_back();
                }
                
                // Roll through to the end of sentence.
                while ( _input_file &&(_input_file.get() != constants::eol) ) {
                    // Loop until we get to end of sentence.
                    _input_file.ignore(100000, constants::eol);
                    ++_line_no;
                }
                // Empty line needs to be counted too.
                ++_line_no;             
                if ( _input_file ) {
                    continue;
                } else {
                    // EOF.
                    break;
                }
            }

            // Process lemma.
            for ( size_t i = 0; i < lemma.length(); ++i ) {
                // TODO: Get rid off of literals!
                switch ( lemma[i] ) {
                    case ':':
                    case '^':
                    case ';':
                    case '\'':
                    case '_':
                        lemma.erase(i);
                        goto jump;
                }
            }
jump:
            // Process tag and context tag.
            if ( settings::morphologic_tag_mask != 0 ) {
                // 
                unsigned i = 0;
                //
                for ( ; (settings::morphologic_tag_mask[i] != 0) && (i < tag.size()); ++i ) {
                    if ( settings::morphologic_tag_mask[i] != constants::significant_position_mark ) {
                        // "Null" the i-th mark.
                        tag[i] = constants::insignificant_position_mark;
                    }
                }
                // Chop off the rest of tag if longer or adjust with '-' chars if shorter.
                tag.resize(strlen(settings::morphologic_tag_mask), constants::insignificant_position_mark);
                // 
            } else {
                // Leave tags empty.
                tag = std::string();                
            }           
#ifdef _DEPENDENCY_MODE
            // Both numeric variables must be cast to proper type.
            // The strings are stored in string-stores.
            store.push_back(Word(static_cast<word_order_t>(order), static_cast<word_order_t>(parent), persistent::lemmas.store(lemma.c_str()), persistent::tags.store(tag.c_str()), string2dependency(dependency)));
#else
            // The lemma and tag are stored within string-stores.
            store.push_back(Word(persistent::lemmas.store(lemma.c_str()), persistent::tags.store(tag.c_str())));
#endif
            // Increase parsed words counter.
            ++words_recently_parsed;
        } else {
            // Empty line.
            if ( words_recently_parsed ) {
                // Sentence must be counted.                
                _parsed_counter.push_back(words_recently_parsed);
                words_totally_parsed += words_recently_parsed;
                // Reset counter.
                words_recently_parsed = 0;
            }
            if ( (mode == ContextWindow::WORD_MODE) && (words_totally_parsed >= count) || (mode == ContextWindow::SENTENCE_MODE) && (_parsed_counter.size() >= count) ) {
                // We've parsed at least as much as was required.
                break;
            }
        }
    }
        
    if ( words_recently_parsed ) {
        // Last sentence needs to be counted too.
        _parsed_counter.push_back(words_recently_parsed);
    }

    return !_parsed_counter.empty();

}

} // namespace ace

---