// Reads a new-style value
//@TODO: UCREMOVAL: Needs a better name
FString FBaseParser::ReadNewStyleValue(const FString& TypeOfSpecifier)
{
FToken ValueToken;
if (!GetToken( ValueToken, false ))
FError::Throwf(TEXT("Expected a value when handling a "), *TypeOfSpecifier);
switch (ValueToken.TokenType)
{
case TOKEN_Identifier:
case TOKEN_Symbol:
{
FString Result = ValueToken.Identifier;
if (MatchSymbol(TEXT("=")))
{
Result += TEXT("=");
Result += ReadNewStyleValue(TypeOfSpecifier);
}
return Result;
}
case TOKEN_Const:
return ValueToken.GetConstantValue();
default:
return TEXT("");
}
}
/**
* Put all text from the current position up to either EOL or the StopToken
* into Token. Advances the compiler's current position.
*
* @param Token [out] will contain the text that was parsed
* @param StopChar stop processing when this character is reached
*
* @return the number of character parsed
*/
bool FBaseParser::GetRawToken( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ )
{
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
while( !IsEOL(c) && c != StopChar )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
}
c = GetChar(true);
}
UngetChar();
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
Token.SetConstString(Temp);
return Length>0;
}
bool FBaseParser::GetConstInt(int32& Result, const TCHAR* Tag)
{
FToken Token;
if (GetToken(Token))
{
if (Token.GetConstInt(Result))
{
return true;
}
else
{
UngetToken(Token);
}
}
if (Tag != NULL)
{
FError::Throwf(TEXT("%s: Missing constant integer"), Tag );
}
return false;
}
bool FBaseParser::GetRawTokenRespectingQuotes( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ )
{
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
bool bInQuote = false;
while( !IsEOL(c) && ((c != StopChar) || bInQuote) )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
if (c == '"')
{
bInQuote = !bInQuote;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
c = GetChar(true);
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
break;
}
c = GetChar(true);
}
UngetChar();
if (bInQuote)
{
FError::Throwf(TEXT("Unterminated quoted string"));
}
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
Token.SetConstString(Temp);
return Length>0;
}
// Reads a set of specifiers (with optional values) inside the () of a new-style metadata macro like UPROPERTY or UFUNCTION
void FBaseParser::ReadSpecifierSetInsideMacro(TArray<FPropertySpecifier>& SpecifiersFound, const FString& TypeOfSpecifier, TMap<FName, FString>& MetaData)
{
int32 FoundSpecifierCount = 0;
FString ErrorMessage = FString::Printf(TEXT("%s declaration specifier"), *TypeOfSpecifier);
RequireSymbol(TEXT("("), *ErrorMessage);
while (!MatchSymbol(TEXT(")")))
{
if (FoundSpecifierCount > 0)
{
RequireSymbol(TEXT(","), *ErrorMessage);
}
++FoundSpecifierCount;
// Read the specifier key
FToken Specifier;
if (!GetToken(Specifier))
{
FError::Throwf(TEXT("Expected %s"), *ErrorMessage);
}
if (Specifier.Matches(TEXT("meta")))
{
RequireSymbol(TEXT("="), *ErrorMessage);
RequireSymbol(TEXT("("), *ErrorMessage);
// Keep reading comma-separated metadata pairs
do
{
// Read a key
FToken MetaKeyToken;
if (!GetIdentifier(MetaKeyToken))
{
FError::Throwf(TEXT("Expected a metadata key"));
}
FString Key = MetaKeyToken.Identifier;
// Potentially read a value
FString Value;
if (MatchSymbol(TEXT("=")))
{
Value = ReadNewStyleValue(TypeOfSpecifier);
}
// Validate the value is a valid type for the key and insert it into the map
InsertMetaDataPair(MetaData, Key, Value);
} while ( MatchSymbol(TEXT(",")) );
RequireSymbol(TEXT(")"), *ErrorMessage);
}
// Look up specifier in metadata dictionary
else if (FMetadataKeyword* MetadataKeyword = GetMetadataKeyword(Specifier.Identifier))
{
if (MatchSymbol(TEXT("=")))
{
if (MetadataKeyword->ValueArgument == EMetadataValueArgument::None)
{
FError::Throwf(TEXT("Incorrect = after metadata specifier '%s'"), Specifier.Identifier);
}
FString Value = ReadNewStyleValue(TypeOfSpecifier);
MetadataKeyword->ApplyToMetadata(MetaData, &Value);
}
else
{
if (MetadataKeyword->ValueArgument == EMetadataValueArgument::Required)
{
FError::Throwf(TEXT("Missing = after metadata specifier '%s'"), Specifier.Identifier);
}
MetadataKeyword->ApplyToMetadata(MetaData);
}
}
else
{
// Creating a new specifier
SpecifiersFound.Emplace(Specifier.Identifier);
// Look for a value for this specifier
if (MatchSymbol(TEXT("=")) || PeekSymbol(TEXT("(")))
{
TArray<FString>& NewPairValues = SpecifiersFound.Last().Values;
if (!ReadOptionalCommaSeparatedListInParens(NewPairValues, TypeOfSpecifier))
{
FString Value = ReadNewStyleValue(TypeOfSpecifier);
NewPairValues.Add(Value);
}
}
}
}
}
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line.
bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/, ESymbolParseOption bParseTemplateClosingBracket/*=ESymbolParseOption::Normal*/ )
{
Token.TokenName = NAME_None;
TCHAR c = GetLeadingChar();
TCHAR p = PeekChar();
if( c == 0 )
{
UngetChar();
return 0;
}
Token.StartPos = PrevPos;
Token.StartLine = PrevLine;
if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') )
{
// Alphanumeric token.
int32 Length=0;
do
{
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE);
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = GetChar();
} while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') );
UngetChar();
Token.Identifier[Length]=0;
// Assume this is an identifier unless we find otherwise.
Token.TokenType = TOKEN_Identifier;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
// If const values are allowed, determine whether the identifier represents a constant
if ( !bNoConsts )
{
// See if the identifier is part of a vector, rotation or other struct constant.
// boolean true/false
if( Token.Matches(TEXT("true")) )
{
Token.SetConstBool(true);
return true;
}
else if( Token.Matches(TEXT("false")) )
{
Token.SetConstBool(false);
return true;
}
}
return true;
}
// if const values are allowed, determine whether the non-identifier token represents a const
else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) )
{
// Integer or floating point constant.
bool bIsFloat = 0;
int32 Length = 0;
bool bIsHex = 0;
do
{
if( c==TEXT('.') )
{
bIsFloat = true;
}
if( c==TEXT('X') || c == TEXT('x') )
{
bIsHex = true;
}
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE );
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = FChar::ToUpper(GetChar());
} while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F')));
Token.Identifier[Length]=0;
if (!bIsFloat || c != 'F')
{
UngetChar();
}
if (bIsFloat)
{
Token.SetConstFloat( FCString::Atof(Token.Identifier) );
}
else if (bIsHex)
{
TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier);
Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) );
}
else
{
Token.SetConstInt( FCString::Atoi(Token.Identifier) );
}
return true;
}
else if (c == '\'')
{
TCHAR ActualCharLiteral = GetChar(/*bLiteral=*/ true);
if (ActualCharLiteral == '\\')
{
ActualCharLiteral = GetChar(/*bLiteral=*/ true);
switch (ActualCharLiteral)
{
case TCHAR('t'):
ActualCharLiteral = '\t';
break;
case TCHAR('n'):
ActualCharLiteral = '\n';
break;
case TCHAR('r'):
ActualCharLiteral = '\r';
break;
}
}
c = GetChar(/*bLiteral=*/ true);
if (c != '\'')
{
FError::Throwf(TEXT("Unterminated character constant"));
UngetChar();
}
Token.SetConstChar(ActualCharLiteral);
return true;
}
else if (c == '"')
{
// String constant.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
c = GetChar(/*bLiteral=*/ true);
while( (c!='"') && !IsEOL(c) )
{
if( c=='\\' )
{
c = GetChar(/*bLiteral=*/ true);
if( IsEOL(c) )
{
break;
}
else if(c == 'n')
{
// Newline escape sequence.
c = '\n';
}
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
c = TEXT('\"');
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
break;
}
c = GetChar(/*bLiteral=*/ true);
}
Temp[Length]=0;
if( c != '"' )
{
FError::Throwf(TEXT("Unterminated string constant: %s"), Temp);
UngetChar();
}
Token.SetConstString(Temp);
return true;
}
else
{
// Symbol.
int32 Length=0;
Token.Identifier[Length++] = c;
// Handle special 2-character symbols.
#define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */
TCHAR d = GetChar();
if
( PAIR('<','<')
|| (PAIR('>','>') && (bParseTemplateClosingBracket != ESymbolParseOption::CloseTemplateBracket))
|| PAIR('!','=')
|| PAIR('<','=')
|| PAIR('>','=')
|| PAIR('+','+')
|| PAIR('-','-')
|| PAIR('+','=')
|| PAIR('-','=')
|| PAIR('*','=')
|| PAIR('/','=')
|| PAIR('&','&')
|| PAIR('|','|')
|| PAIR('^','^')
|| PAIR('=','=')
|| PAIR('*','*')
|| PAIR('~','=')
|| PAIR(':',':')
)
{
Token.Identifier[Length++] = d;
if( c=='>' && d=='>' )
{
if( GetChar()=='>' )
Token.Identifier[Length++] = '>';
else
UngetChar();
}
}
else UngetChar();
#undef PAIR
Token.Identifier[Length] = 0;
Token.TokenType = TOKEN_Symbol;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
return true;
}
}
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line.
bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/ )
{
Token.TokenName = NAME_None;
TCHAR c = GetLeadingChar();
TCHAR p = PeekChar();
if( c == 0 )
{
UngetChar();
return 0;
}
Token.StartPos = PrevPos;
Token.StartLine = PrevLine;
if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') )
{
// Alphanumeric token.
int32 Length=0;
do
{
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE);
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = GetChar();
} while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') );
UngetChar();
Token.Identifier[Length]=0;
// Assume this is an identifier unless we find otherwise.
Token.TokenType = TOKEN_Identifier;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
// If const values are allowed, determine whether the identifier represents a constant
if ( !bNoConsts )
{
// See if the identifier is part of a vector, rotation or other struct constant.
// boolean true/false
if( Token.Matches(TEXT("true")) )
{
Token.SetConstBool(true);
return true;
}
else if( Token.Matches(TEXT("false")) )
{
Token.SetConstBool(false);
return true;
}
}
return true;
}
// if const values are allowed, determine whether the non-identifier token represents a const
else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) )
{
// Integer or floating point constant.
bool bIsFloat = 0;
int32 Length = 0;
bool bIsHex = 0;
do
{
if( c==TEXT('.') )
{
bIsFloat = true;
}
if( c==TEXT('X') || c == TEXT('x') )
{
bIsHex = true;
}
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE );
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = FChar::ToUpper(GetChar());
} while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F')));
Token.Identifier[Length]=0;
if (!bIsFloat || c != 'F')
{
UngetChar();
}
if (bIsFloat)
{
Token.SetConstFloat( FCString::Atof(Token.Identifier) );
}
else if (bIsHex)
{
TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier);
Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) );
}
else
{
Token.SetConstInt( FCString::Atoi(Token.Identifier) );
}
return true;
}
//@TODO: 'z' is a character literal in C++, not a FName like it was in UnrealScript - do we need this code?
// else if( !bNoConsts && c=='\'' )
// {
// // Name constant.
// int32 Length=0;
// c = GetChar();
// while( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c>='0' && c<='9') || (c=='_') || (c=='-') || (c==' ') ) //@FIXME: space in names should be illegal!
// {
// Token.Identifier[Length++] = c;
// if( Length >= NAME_SIZE )
// {
// FError::Throwf(TEXT("Name length exceeds maximum of %i"), (int32)NAME_SIZE );
// // trick the error a few lines down
// c = TEXT('\'');
// Length = ((int32)NAME_SIZE) - 1;
// break;
// }
// c = GetChar();
// }
// if( c != '\'' )
// {
// UngetChar();
// FError::Throwf(TEXT("Illegal character in name") );
// }
// Token.Identifier[Length]=0;
//
// // Make constant name.
// Token.SetConstName( FName(Token.Identifier) );
// return true;
// }
else if( c=='"' )
{
// String constant.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
c = GetChar(1);
while( (c!='"') && !IsEOL(c) )
{
if( c=='\\' )
{
c = GetChar(1);
if( IsEOL(c) )
{
break;
}
else if(c == 'n')
{
// Newline escape sequence.
c = '\n';
}
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
c = TEXT('\"');
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
break;
}
c = GetChar(1);
}
Temp[Length]=0;
if( c != '"' )
{
FError::Throwf(TEXT("Unterminated string constant: %s"), Temp);
UngetChar();
}
Token.SetConstString(Temp);
return true;
}
else
{
// Symbol.
int32 Length=0;
Token.Identifier[Length++] = c;
// Handle special 2-character symbols.
#define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */
TCHAR d = GetChar();
if
( PAIR('<','<')
|| PAIR('>','>')
|| PAIR('!','=')
|| PAIR('<','=')
|| PAIR('>','=')
|| PAIR('+','+')
|| PAIR('-','-')
|| PAIR('+','=')
|| PAIR('-','=')
|| PAIR('*','=')
|| PAIR('/','=')
|| PAIR('&','&')
|| PAIR('|','|')
|| PAIR('^','^')
|| PAIR('=','=')
|| PAIR('*','*')
|| PAIR('~','=')
|| PAIR(':',':')
)
{
Token.Identifier[Length++] = d;
if( c=='>' && d=='>' )
{
if( GetChar()=='>' )
Token.Identifier[Length++] = '>';
else
UngetChar();
}
}
else UngetChar();
#undef PAIR
Token.Identifier[Length] = 0;
Token.TokenType = TOKEN_Symbol;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
return true;
}
}
