MessagePattern &
MessagePattern::parse(const UnicodeString &pattern, UParseError *parseError, UErrorCode &errorCode) {
preParse(pattern, parseError, errorCode);
parseMessage(0, 0, 0, UMSGPAT_ARG_TYPE_NONE, parseError, errorCode);
postParse();
return *this;
}
void WordNgrams::addTokens()
{
// get token string from input file
string & inFileName = getInFileName();
FILE * fp = inFileName.length() > 0 ? fopen( inFileName.c_str(), "r" ) : stdin;
if ( !fp )
{
printf("Can not find file %s, use stdio as input.\n", inFileName.c_str() );
fp = stdin;
}
int count = 0;
char c;
bool isSpecialChar = false;
string token;
token.reserve(256);
while ( ( c = (char) fgetc( fp ) ) != EOF )
{
if ( isDelimiter( c ) || isStopChar ( c ) )
{
if ( !isSpecialChar && token.length() >0 )
{
addToken( token );
token.empty();
++count;
isSpecialChar = true;
}
else
{
isSpecialChar = false;
}
}
else
{
token.append( c );
isSpecialChar = false;
}
}
if ( token.length() > 0 )
{
++count;
addToken( token );
}
// special processing need to be done, if less than NGRAM_N tokens in the whole input text.
if ( count < this->getN() )
{
preParse( count );
}
/* int padding = ngramN - count % ngramN;
for ( int i=0; i< padding; i++)
{
addToken( "_" );
}
*/
fclose( fp );
}
MessagePattern &
MessagePattern::parseSelectStyle(const UnicodeString &pattern,
UParseError *parseError, UErrorCode &errorCode) {
preParse(pattern, parseError, errorCode);
parsePluralOrSelectStyle(UMSGPAT_ARG_TYPE_SELECT, 0, 0, parseError, errorCode);
postParse();
return *this;
}
MessagePattern &
MessagePattern::parseChoiceStyle(const UnicodeString &pattern,
UParseError *parseError, UErrorCode &errorCode) {
preParse(pattern, parseError, errorCode);
parseChoiceStyle(0, 0, parseError, errorCode);
postParse();
return *this;
}
void Ngrams::addToken ( const string & token )
{
int count = pushQueue( token.c_str() );
if ( count == this->ngramN )
{
parse();
popQueue();
}
else if ( count == this->ngramN - 1 )
{
preParse( count );
}
}
ValidationList ConfigSection::parse(const INIConfigSection& section) {
section_name = section.getName();
preParse(section, validation);
auto entries = section.getEntries();
for (auto entry_it = entries.begin(); entry_it != entries.end(); ++entry_it) {
std::string key = entry_it->first;
std::string value = entry_it->second;
if (!parseField(key, value, validation))
validation.warning("Unknown configuration option '" + key + "'!");
}
postParse(section, validation);
return validation;
}
