MessagePattern & MessagePattern::parse(const UnicodeString &pattern, UParseError *parseError, UErrorCode &errorCode) { preParse(pattern, parseError, errorCode); parseMessage(0, 0, 0, UMSGPAT_ARG_TYPE_NONE, parseError, errorCode); postParse(); return *this; }
void WordNgrams::addTokens() { // get token string from input file string & inFileName = getInFileName(); FILE * fp = inFileName.length() > 0 ? fopen( inFileName.c_str(), "r" ) : stdin; if ( !fp ) { printf("Can not find file %s, use stdio as input.\n", inFileName.c_str() ); fp = stdin; } int count = 0; char c; bool isSpecialChar = false; string token; token.reserve(256); while ( ( c = (char) fgetc( fp ) ) != EOF ) { if ( isDelimiter( c ) || isStopChar ( c ) ) { if ( !isSpecialChar && token.length() >0 ) { addToken( token ); token.empty(); ++count; isSpecialChar = true; } else { isSpecialChar = false; } } else { token.append( c ); isSpecialChar = false; } } if ( token.length() > 0 ) { ++count; addToken( token ); } // special processing need to be done, if less than NGRAM_N tokens in the whole input text. if ( count < this->getN() ) { preParse( count ); } /* int padding = ngramN - count % ngramN; for ( int i=0; i< padding; i++) { addToken( "_" ); } */ fclose( fp ); }
MessagePattern & MessagePattern::parseSelectStyle(const UnicodeString &pattern, UParseError *parseError, UErrorCode &errorCode) { preParse(pattern, parseError, errorCode); parsePluralOrSelectStyle(UMSGPAT_ARG_TYPE_SELECT, 0, 0, parseError, errorCode); postParse(); return *this; }
MessagePattern & MessagePattern::parseChoiceStyle(const UnicodeString &pattern, UParseError *parseError, UErrorCode &errorCode) { preParse(pattern, parseError, errorCode); parseChoiceStyle(0, 0, parseError, errorCode); postParse(); return *this; }
void Ngrams::addToken ( const string & token ) { int count = pushQueue( token.c_str() ); if ( count == this->ngramN ) { parse(); popQueue(); } else if ( count == this->ngramN - 1 ) { preParse( count ); } }
ValidationList ConfigSection::parse(const INIConfigSection& section) { section_name = section.getName(); preParse(section, validation); auto entries = section.getEntries(); for (auto entry_it = entries.begin(); entry_it != entries.end(); ++entry_it) { std::string key = entry_it->first; std::string value = entry_it->second; if (!parseField(key, value, validation)) validation.warning("Unknown configuration option '" + key + "'!"); } postParse(section, validation); return validation; }