- Main Page
- Namespaces
- Classes
- Files

# proto/Source/parser.cpp

Go to the documentation of this file.

    00001
    00011 // Base settings.
    00012 #include "config.h"
    00013 #ifdef _DEPENDENCY_MODE
    00014 // Need of string2dependency()
    00015 #include "dependencies.h"
    00016 #endif
    00017 // Progress display, warnings display.
    00018 #include "notifier.h"
    00019 // Self header.
    00020 #include "parser.h"
    00021 // Persistent store access (strings).
    00022 #include "persistent.h"
    00023 // Need of unsigned2str()
    00024 #include "utils.h"
    00025
    00026 namespace ace {
    00027
    00028 /*  Interface item implementation - for info see description in header file.
    00029  */
    00030 bool Parser::next(size_t count, ContextWindow::Mode mode, words_store_t& store) {
    00031     // Items, which don't need to be defined within declaration may safely be static.
    00032     // To read (check) first char of the line.
    00033     static int c;
    00034     // To gather parsed information following variables are needed.
    00035     static std::string lemma;
    00036     static std::string tag;
    00037     static std::string context_tag;
    00038 #ifdef _DEPENDENCY_MODE
    00039     static std::string occurrence;
    00040     static std::string dependency;
    00041     // Must be unsigned int, because text input passed to unsigned char is not interpreted as number by operator >>.
    00042     static unsigned int order;
    00043     static unsigned int parent;
    00044 #endif
    00045     // Counters of words parsed: for last sentence and for single function run.
    00046     size_t words_recently_parsed = 0, words_totally_parsed = 0;
    00047     bool incorrect_input_format_detected = false;
    00048
    00049     // Clear last run results
    00050     _parsed_counter.clear();
    00051
    00052     while ( _input_file ) {
    00053
    00054         // Get first char.
    00055         c = _input_file.get();
    00056
    00057         if ( c < 0 ) { // end-of-file
    00058             // Jump out.
    00059             break;
    00060         }
    00061
    00062         // Count line.
    00063         ++_line_no;
    00064
    00065         if ( c != constants::eol ) {
    00066             // Not the separator line, unget previously retrieved char, will be needed.
    00067             _input_file.unget();
    00068 #ifdef _DEPENDENCY_MODE
    00069             // Format of the line is as following:
    00070             _input_file >> order >> occurrence >> lemma >> tag >> parent >> dependency;
    00071 #else
    00072             // Format of the line is much simpler than above:
    00073             _input_file >> lemma >> tag;
    00074 #endif
    00075             if ( _input_file.fail() && !_input_file.eof() ) {
    00076                 // Some error occured.
    00077                 incorrect_input_format_detected = true;
    00078                 _input_file.clear();
    00079             }
    00080
    00081             // Skip all characters until the end-of-line.
    00082             // If the input file is in correct format, the end-of-line character will be
    00083             // the right next one in stream buffer.
    00084             _input_file.ignore(1, constants::eol);
    00085 #ifdef _DEPENDENCY_MODE
    00086             if ( incorrect_input_format_detected || (order != (words_recently_parsed + 1)) || (words_recently_parsed == constants::max_sentence_length) ) {
    00087 #else
    00088             if ( incorrect_input_format_detected || (words_recently_parsed == constants::max_sentence_length) ) {
    00089 #endif
    00090                 // Some error occured, print message.
    00091                 if ( incorrect_input_format_detected ) {
    00092                     notifier::error.warning("Incorrect input format detected at line " + unsigned2str(_line_no) + ". Skipping current sentence...");
    00093                     // Reset flag.
    00094                     incorrect_input_format_detected = false;
    00095                 } else if ( words_recently_parsed == constants::max_sentence_length ) {
    00096                     // Word order check overflow.
    00097                     notifier::error.warning("Word order overflow detected at line " + unsigned2str(_line_no) + ". Got word ordered as " + unsigned2str(words_recently_parsed + 1) + ", while maximum allowed value for word order is " + unsigned2str(constants::max_sentence_length) + ". Skipping current sentence...");
    00098 #ifdef _DEPENDENCY_MODE
    00099                 } else {
    00100                     // Mismatch.
    00101                     notifier::error.warning("Word order mismatch detected at line " + unsigned2str(_line_no) + ". Got word ordered as " + unsigned2str(order) + ", while expecting " + unsigned2str(words_recently_parsed + 1) + ". Skipping current sentence...");
    00102 #endif
    00103                 }
    00104
    00105                 // Roll back current sentence.
    00106                 for ( ; words_recently_parsed > 0; --words_recently_parsed ) {
    00107                     store.pop_back();
    00108                 }
    00109
    00110                 // Roll through to the end of sentence.
    00111                 while ( _input_file &&(_input_file.get() != constants::eol) ) {
    00112                     // Loop until we get to end of sentence.
    00113                     _input_file.ignore(100000, constants::eol);
    00114                     ++_line_no;
    00115                 }
    00116                 // Empty line needs to be counted too.
    00117                 ++_line_no;
    00118                 if ( _input_file ) {
    00119                     continue;
    00120                 } else {
    00121                     // EOF.
    00122                     break;
    00123                 }
    00124             }
    00125
    00126             // Process lemma.
    00127             for ( size_t i = 0; i < lemma.length(); ++i ) {
    00128                 // TODO: Get rid off of literals!
    00129                 switch ( lemma[i] ) {
    00130                     case ':':
    00131                     case '^':
    00132                     case ';':
    00133                     case '\'':
    00134                     case '_':
    00135                         lemma.erase(i);
    00136                         goto jump;
    00137                 }
    00138             }
    00139 jump:
    00140             // Process tag and context tag.
    00141             if ( settings::morphologic_tag_mask != 0 ) {
    00142                 //
    00143                 unsigned i = 0;
    00144                 //
    00145                 for ( ; (settings::morphologic_tag_mask[i] != 0) && (i < tag.size()); ++i ) {
    00146                     if ( settings::morphologic_tag_mask[i] != constants::significant_position_mark ) {
    00147                         // "Null" the i-th mark.
    00148                         tag[i] = constants::insignificant_position_mark;
    00149                     }
    00150                 }
    00151                 // Chop off the rest of tag if longer or adjust with '-' chars if shorter.
    00152                 tag.resize(strlen(settings::morphologic_tag_mask), constants::insignificant_position_mark);
    00153                 //
    00154             } else {
    00155                 // Leave tags empty.
    00156                 tag = std::string();
    00157             }
    00158 #ifdef _DEPENDENCY_MODE
    00159             // Both numeric variables must be cast to proper type.
    00160             // The strings are stored in string-stores.
    00161             store.push_back(Word(static_cast<word_order_t>(order), static_cast<word_order_t>(parent), persistent::lemmas.store(lemma.c_str()), persistent::tags.store(tag.c_str()), string2dependency(dependency)));
    00162 #else
    00163             // The lemma and tag are stored within string-stores.
    00164             store.push_back(Word(persistent::lemmas.store(lemma.c_str()), persistent::tags.store(tag.c_str())));
    00165 #endif
    00166             // Increase parsed words counter.
    00167             ++words_recently_parsed;
    00168         } else {
    00169             // Empty line.
    00170             if ( words_recently_parsed ) {
    00171                 // Sentence must be counted.
    00172                 _parsed_counter.push_back(words_recently_parsed);
    00173                 words_totally_parsed += words_recently_parsed;
    00174                 // Reset counter.
    00175                 words_recently_parsed = 0;
    00176             }
    00177             if ( (mode == ContextWindow::WORD_MODE) && (words_totally_parsed >= count) || (mode == ContextWindow::SENTENCE_MODE) && (_parsed_counter.size() >= count) ) {
    00178                 // We've parsed at least as much as was required.
    00179                 break;
    00180             }
    00181         }
    00182     }
    00183
    00184     if ( words_recently_parsed ) {
    00185         // Last sentence needs to be counted too.
    00186         _parsed_counter.push_back(words_recently_parsed);
    00187     }
    00188
    00189     return !_parsed_counter.empty();
    00190
    00191 }
    00192
    00193 } // namespace ace