bool TGztParser::ParseArticleDefinition(TArticleDescriptorProto* article)
{
RecordLocation(article, NProtoBuf::DescriptorPool::ErrorCollector::OTHER);
DO(ConsumeType(article->mutable_type(), "Expected article type."));
if (LookingAt('{'))
article->mutable_name()->clear(); // allow articles without names
else {
if (LookingAtType(Tokenizer::TYPE_STRING)) {
DO(ConsumeString(article->mutable_name(), "Expected article name."));
}
else
DO(ConsumeIdentifier(article->mutable_name(), "Expected article name."));
}
DO(ParseArticleBlock(article->mutable_field()));
return true;
}
bool HTMLLexer::AdvanceOneToken( int &outToken, TokenList *outTokens )
{
outToken = -1;
if (!fLexerInput)
return false;
// There can be any combination of newlines and whitespaces preceeding semantic tokens,
// so we're going to loop until we don't find either.
while (fLexerInput->HasMoreChars())
{
bool consumedWhitespace = false;
bool consumedNewLine = false;
if (!SkipWhitespaces( consumedWhitespace, outTokens ))
return false;
// HTML also treats newlines as a whitespace
while (fLexerInput->HasMoreChars() && IsLineEnding( fLexerInput->PeekAtNextChar() ))
{
// Eat the line ending
ConsumeLineEnding( fLexerInput->MoveToNextChar() );
consumedNewLine = true;
}
// If we're done consuming newlines and whitespaces, then we're done with this loop
if (!consumedWhitespace && !consumedNewLine)
break;
}
if (!fLexerInput->HasMoreChars())
return false;
// Take a peek at what sort of token we're about to deal with.
UniChar uChar = fLexerInput->PeekAtNextChar();
sLONG stringType;
sLONG stringValue;
if( (outToken = ConsumePossiblePunctuation(uChar, outTokens)) != 0 )
{
}
else if (IsStringStart( uChar, stringType, stringValue ))
{
VString vstrQuoted;
if (!ConsumeString( &vstrQuoted, outTokens, stringType, stringValue ))
{
return false;
}
outToken = stringValue;
fLastTokenText = vstrQuoted;
}
else if (IsIdentifierStart( uChar ))
{
// The base class assumes we've consumed the first character already for this call. We should
// rectify this some day, as it's very confusing.
fLexerInput->MoveToNextChar();
VString *vstrNAME = ConsumeIdentifier();
if (!vstrNAME)
{
return false;
}
outToken = HTMLLexemes::TEXT;
if (outTokens) outTokens->push_back( new HTMLLexerToken( ILexerToken::TT_NAME, fLexerInput->GetCurrentPosition() - vstrNAME->GetLength(), vstrNAME->GetLength(), *vstrNAME, outToken ) );
fLastTokenText = *vstrNAME;
delete vstrNAME;
}
else
{
return false;
}
SetLastToken( outToken );
return true;
}
bool TGztParser::ParseSingleFieldValue(TFieldValueDescriptorProto* value)
{
bool is_negative = TryConsume('-');
switch (CurrentToken().type) {
case Tokenizer::TYPE_START:
AddError("Trying to read value before any tokens have been read.");
return false;
case Tokenizer::TYPE_END:
AddError("Unexpected end of stream while parsing field value.");
return false;
case Tokenizer::TYPE_IDENTIFIER:
if (is_negative) {
AddError("Invalid '-' symbol before identifier.");
return false;
}
value->set_type(TFieldValueDescriptorProto::TYPE_IDENTIFIER);
DO(ConsumeIdentifier(value->mutable_string_or_identifier(), "Expected identifier."));
return true;
case Tokenizer::TYPE_INTEGER:
ui64 number, max_value;
max_value = is_negative ? static_cast<ui64>(NProtoBuf::kint64max) + 1 : NProtoBuf::kuint64max;
DO(ConsumeInteger64(max_value, &number, "Expected integer."));
value->set_type(TFieldValueDescriptorProto::TYPE_INTEGER);
value->set_int_number(is_negative ? -(i64)number : number);
return true;
case Tokenizer::TYPE_FLOAT:
value->set_type(TFieldValueDescriptorProto::TYPE_FLOAT);
value->set_float_number(Tokenizer::ParseFloat(CurrentToken().text) * (is_negative ? -1 : 1));
NextToken();
return true;
case Tokenizer::TYPE_STRING:
if (is_negative) {
AddError("Invalid '-' symbol before string.");
return false;
}
value->set_type(TFieldValueDescriptorProto::TYPE_STRING);
Tokenizer::ParseString(CurrentToken().text, value->mutable_string_or_identifier());
NextToken();
return true;
case Tokenizer::TYPE_SYMBOL:
if (LookingAt('{')) {
if (is_negative) {
AddError("Invalid '-' symbol before inner block.");
return false;
}
//parse sub-article
value->set_type(TFieldValueDescriptorProto::TYPE_BLOCK);
return ParseArticleBlock(value->mutable_sub_field());
} else if (LookingAt('[')) {
if (is_negative) {
AddError("Invalid '-' symbol before list.");
return false;
}
return ParseBracketedValuesList(value);
}
}
AddError("Expected field value.");
return false;
}
