//------------------------------------------------------------------------------
bool FBasicTokenParser::GetRawTokenRespectingQuotes(FBasicToken& Token, TCHAR StopChar/* = TCHAR('\n')*/)
{
// if the parser is in a bad state, then don't continue parsing (who
// knows what will happen!?)
if (!IsValid())
{
return false;
}
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
bool bInQuote = false;
while( !IsEOL(c) && ((c != StopChar) || bInQuote) )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
if (c == '"')
{
bInQuote = !bInQuote;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
Temp[Length]=0; // needs to happen for the error description below
FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
c = GetChar(true);
break;
}
c = GetChar(true);
}
UngetChar();
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
if (bInQuote)
{
FText ErrorDesc = FText::Format(LOCTEXT("NoClosingQuote", "Unterminated quoted string ({0})"), FText::FromString(Temp));
SetError(FErrorState::ParseError, ErrorDesc);
}
Token.SetConstString(Temp);
return Length>0 && IsValid();
}
/**
* Put all text from the current position up to either EOL or the StopToken
* into Token. Advances the compiler's current position.
*
* @param Token [out] will contain the text that was parsed
* @param StopChar stop processing when this character is reached
*
* @return the number of character parsed
*/
bool FBaseParser::GetRawToken( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ )
{
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
while( !IsEOL(c) && c != StopChar )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
}
c = GetChar(true);
}
UngetChar();
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
Token.SetConstString(Temp);
return Length>0;
}
bool FBaseParser::GetRawTokenRespectingQuotes( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ )
{
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
bool bInQuote = false;
while( !IsEOL(c) && ((c != StopChar) || bInQuote) )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
if (c == '"')
{
bInQuote = !bInQuote;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
c = GetChar(true);
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
break;
}
c = GetChar(true);
}
UngetChar();
if (bInQuote)
{
FError::Throwf(TEXT("Unterminated quoted string"));
}
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
Token.SetConstString(Temp);
return Length>0;
}
//------------------------------------------------------------------------------
bool FBasicTokenParser::GetRawToken(FBasicToken& Token, TCHAR StopChar/* = TCHAR('\n')*/)
{
// if the parser is in a bad state, then don't continue parsing (who
// knows what will happen!?)
if (!IsValid())
{
return false;
}
// Get token after whitespace.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
TCHAR c = GetLeadingChar();
while( !IsEOL(c) && c != StopChar )
{
if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') )
{
break;
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
Temp[Length] = 0;
FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
}
c = GetChar(true);
}
UngetChar();
// Get rid of trailing whitespace.
while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) )
{
Length--;
}
Temp[Length]=0;
Token.SetConstString(Temp);
return Length>0;
}
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line.
bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/, ESymbolParseOption bParseTemplateClosingBracket/*=ESymbolParseOption::Normal*/ )
{
Token.TokenName = NAME_None;
TCHAR c = GetLeadingChar();
TCHAR p = PeekChar();
if( c == 0 )
{
UngetChar();
return 0;
}
Token.StartPos = PrevPos;
Token.StartLine = PrevLine;
if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') )
{
// Alphanumeric token.
int32 Length=0;
do
{
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE);
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = GetChar();
} while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') );
UngetChar();
Token.Identifier[Length]=0;
// Assume this is an identifier unless we find otherwise.
Token.TokenType = TOKEN_Identifier;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
// If const values are allowed, determine whether the identifier represents a constant
if ( !bNoConsts )
{
// See if the identifier is part of a vector, rotation or other struct constant.
// boolean true/false
if( Token.Matches(TEXT("true")) )
{
Token.SetConstBool(true);
return true;
}
else if( Token.Matches(TEXT("false")) )
{
Token.SetConstBool(false);
return true;
}
}
return true;
}
// if const values are allowed, determine whether the non-identifier token represents a const
else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) )
{
// Integer or floating point constant.
bool bIsFloat = 0;
int32 Length = 0;
bool bIsHex = 0;
do
{
if( c==TEXT('.') )
{
bIsFloat = true;
}
if( c==TEXT('X') || c == TEXT('x') )
{
bIsHex = true;
}
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE );
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = FChar::ToUpper(GetChar());
} while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F')));
Token.Identifier[Length]=0;
if (!bIsFloat || c != 'F')
{
UngetChar();
}
if (bIsFloat)
{
Token.SetConstFloat( FCString::Atof(Token.Identifier) );
}
else if (bIsHex)
{
TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier);
Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) );
}
else
{
Token.SetConstInt( FCString::Atoi(Token.Identifier) );
}
return true;
}
else if (c == '\'')
{
TCHAR ActualCharLiteral = GetChar(/*bLiteral=*/ true);
if (ActualCharLiteral == '\\')
{
ActualCharLiteral = GetChar(/*bLiteral=*/ true);
switch (ActualCharLiteral)
{
case TCHAR('t'):
ActualCharLiteral = '\t';
break;
case TCHAR('n'):
ActualCharLiteral = '\n';
break;
case TCHAR('r'):
ActualCharLiteral = '\r';
break;
}
}
c = GetChar(/*bLiteral=*/ true);
if (c != '\'')
{
FError::Throwf(TEXT("Unterminated character constant"));
UngetChar();
}
Token.SetConstChar(ActualCharLiteral);
return true;
}
else if (c == '"')
{
// String constant.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
c = GetChar(/*bLiteral=*/ true);
while( (c!='"') && !IsEOL(c) )
{
if( c=='\\' )
{
c = GetChar(/*bLiteral=*/ true);
if( IsEOL(c) )
{
break;
}
else if(c == 'n')
{
// Newline escape sequence.
c = '\n';
}
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
c = TEXT('\"');
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
break;
}
c = GetChar(/*bLiteral=*/ true);
}
Temp[Length]=0;
if( c != '"' )
{
FError::Throwf(TEXT("Unterminated string constant: %s"), Temp);
UngetChar();
}
Token.SetConstString(Temp);
return true;
}
else
{
// Symbol.
int32 Length=0;
Token.Identifier[Length++] = c;
// Handle special 2-character symbols.
#define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */
TCHAR d = GetChar();
if
( PAIR('<','<')
|| (PAIR('>','>') && (bParseTemplateClosingBracket != ESymbolParseOption::CloseTemplateBracket))
|| PAIR('!','=')
|| PAIR('<','=')
|| PAIR('>','=')
|| PAIR('+','+')
|| PAIR('-','-')
|| PAIR('+','=')
|| PAIR('-','=')
|| PAIR('*','=')
|| PAIR('/','=')
|| PAIR('&','&')
|| PAIR('|','|')
|| PAIR('^','^')
|| PAIR('=','=')
|| PAIR('*','*')
|| PAIR('~','=')
|| PAIR(':',':')
)
{
Token.Identifier[Length++] = d;
if( c=='>' && d=='>' )
{
if( GetChar()=='>' )
Token.Identifier[Length++] = '>';
else
UngetChar();
}
}
else UngetChar();
#undef PAIR
Token.Identifier[Length] = 0;
Token.TokenType = TOKEN_Symbol;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
return true;
}
}
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line.
bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/ )
{
Token.TokenName = NAME_None;
TCHAR c = GetLeadingChar();
TCHAR p = PeekChar();
if( c == 0 )
{
UngetChar();
return 0;
}
Token.StartPos = PrevPos;
Token.StartLine = PrevLine;
if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') )
{
// Alphanumeric token.
int32 Length=0;
do
{
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE);
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = GetChar();
} while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') );
UngetChar();
Token.Identifier[Length]=0;
// Assume this is an identifier unless we find otherwise.
Token.TokenType = TOKEN_Identifier;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
// If const values are allowed, determine whether the identifier represents a constant
if ( !bNoConsts )
{
// See if the identifier is part of a vector, rotation or other struct constant.
// boolean true/false
if( Token.Matches(TEXT("true")) )
{
Token.SetConstBool(true);
return true;
}
else if( Token.Matches(TEXT("false")) )
{
Token.SetConstBool(false);
return true;
}
}
return true;
}
// if const values are allowed, determine whether the non-identifier token represents a const
else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) )
{
// Integer or floating point constant.
bool bIsFloat = 0;
int32 Length = 0;
bool bIsHex = 0;
do
{
if( c==TEXT('.') )
{
bIsFloat = true;
}
if( c==TEXT('X') || c == TEXT('x') )
{
bIsHex = true;
}
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE );
Length = ((int32)NAME_SIZE) - 1;
break;
}
c = FChar::ToUpper(GetChar());
} while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F')));
Token.Identifier[Length]=0;
if (!bIsFloat || c != 'F')
{
UngetChar();
}
if (bIsFloat)
{
Token.SetConstFloat( FCString::Atof(Token.Identifier) );
}
else if (bIsHex)
{
TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier);
Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) );
}
else
{
Token.SetConstInt( FCString::Atoi(Token.Identifier) );
}
return true;
}
//@TODO: 'z' is a character literal in C++, not a FName like it was in UnrealScript - do we need this code?
// else if( !bNoConsts && c=='\'' )
// {
// // Name constant.
// int32 Length=0;
// c = GetChar();
// while( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c>='0' && c<='9') || (c=='_') || (c=='-') || (c==' ') ) //@FIXME: space in names should be illegal!
// {
// Token.Identifier[Length++] = c;
// if( Length >= NAME_SIZE )
// {
// FError::Throwf(TEXT("Name length exceeds maximum of %i"), (int32)NAME_SIZE );
// // trick the error a few lines down
// c = TEXT('\'');
// Length = ((int32)NAME_SIZE) - 1;
// break;
// }
// c = GetChar();
// }
// if( c != '\'' )
// {
// UngetChar();
// FError::Throwf(TEXT("Illegal character in name") );
// }
// Token.Identifier[Length]=0;
//
// // Make constant name.
// Token.SetConstName( FName(Token.Identifier) );
// return true;
// }
else if( c=='"' )
{
// String constant.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
c = GetChar(1);
while( (c!='"') && !IsEOL(c) )
{
if( c=='\\' )
{
c = GetChar(1);
if( IsEOL(c) )
{
break;
}
else if(c == 'n')
{
// Newline escape sequence.
c = '\n';
}
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE );
c = TEXT('\"');
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
break;
}
c = GetChar(1);
}
Temp[Length]=0;
if( c != '"' )
{
FError::Throwf(TEXT("Unterminated string constant: %s"), Temp);
UngetChar();
}
Token.SetConstString(Temp);
return true;
}
else
{
// Symbol.
int32 Length=0;
Token.Identifier[Length++] = c;
// Handle special 2-character symbols.
#define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */
TCHAR d = GetChar();
if
( PAIR('<','<')
|| PAIR('>','>')
|| PAIR('!','=')
|| PAIR('<','=')
|| PAIR('>','=')
|| PAIR('+','+')
|| PAIR('-','-')
|| PAIR('+','=')
|| PAIR('-','=')
|| PAIR('*','=')
|| PAIR('/','=')
|| PAIR('&','&')
|| PAIR('|','|')
|| PAIR('^','^')
|| PAIR('=','=')
|| PAIR('*','*')
|| PAIR('~','=')
|| PAIR(':',':')
)
{
Token.Identifier[Length++] = d;
if( c=='>' && d=='>' )
{
if( GetChar()=='>' )
Token.Identifier[Length++] = '>';
else
UngetChar();
}
}
else UngetChar();
#undef PAIR
Token.Identifier[Length] = 0;
Token.TokenType = TOKEN_Symbol;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
return true;
}
}
//------------------------------------------------------------------------------
bool FBasicTokenParser::GetToken(FBasicToken& Token, bool bNoConsts/* = false*/)
{
// if the parser is in a bad state, then don't continue parsing (who
// knows what will happen!?)
if (!IsValid())
{
return false;
}
Token.TokenName = NAME_None;
TCHAR c = GetLeadingChar();
TCHAR p = PeekChar();
if( c == 0 )
{
UngetChar();
return 0;
}
Token.StartPos = PrevPos;
Token.StartLine = PrevLine;
if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') )
{
// Alphanumeric token.
int32 Length=0;
do
{
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
Length = ((int32)NAME_SIZE) - 1;
Token.Identifier[Length]=0; // need this for the error description
FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Token.Identifier), FText::AsNumber((int32)NAME_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
break;
}
c = GetChar();
} while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') );
UngetChar();
Token.Identifier[Length]=0;
// Assume this is an identifier unless we find otherwise.
Token.TokenType = FBasicToken::TOKEN_Identifier;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
// If const values are allowed, determine whether the identifier represents a constant
if ( !bNoConsts )
{
// See if the identifier is part of a vector, rotation or other struct constant.
// boolean true/false
if( Token.Matches(TEXT("true")) )
{
Token.SetConstBool(true);
return true;
}
else if( Token.Matches(TEXT("false")) )
{
Token.SetConstBool(false);
return true;
}
}
return IsValid();
}
// if const values are allowed, determine whether the non-identifier token represents a const
else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) )
{
// Integer or floating point constant.
bool bIsFloat = 0;
int32 Length = 0;
bool bIsHex = 0;
do
{
if( c==TEXT('.') )
{
bIsFloat = true;
}
if( c==TEXT('X') || c == TEXT('x') )
{
bIsHex = true;
}
Token.Identifier[Length++] = c;
if( Length >= NAME_SIZE )
{
Length = ((int32)NAME_SIZE) - 1;
Token.Identifier[Length]=0; // need this for the error description
FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Token.Identifier), FText::AsNumber((int32)NAME_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
break;
}
c = FChar::ToUpper(GetChar());
} while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F')));
Token.Identifier[Length]=0;
if (!bIsFloat || c != 'F')
{
UngetChar();
}
if (bIsFloat)
{
Token.SetConstFloat( FCString::Atof(Token.Identifier) );
}
else if (bIsHex)
{
TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier);
Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) );
}
else
{
Token.SetConstInt( FCString::Atoi(Token.Identifier) );
}
return IsValid();
}
else if( c=='"' )
{
// String constant.
TCHAR Temp[MAX_STRING_CONST_SIZE];
int32 Length=0;
c = GetChar(1);
while( (c!='"') && !IsEOL(c) )
{
if( c=='\\' )
{
c = GetChar(1);
if( IsEOL(c) )
{
break;
}
else if(c == 'n')
{
// Newline escape sequence.
c = '\n';
}
}
Temp[Length++] = c;
if( Length >= MAX_STRING_CONST_SIZE )
{
Length = ((int32)MAX_STRING_CONST_SIZE) - 1;
Temp[Length]=0; // need this for the error description
FText ErrorDesc = FText::Format(LOCTEXT("StringConstTooLong", "String constant ({0}...) exceeds maximum of {1} characters"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE));
SetError(FErrorState::ParseError, ErrorDesc);
c = TEXT('\"');
break;
}
c = GetChar(1);
}
Temp[Length]=0;
if( c != '"' )
{
FText ErrorDesc = FText::Format(LOCTEXT("NoClosingQuote", "Unterminated quoted string ({0})"), FText::FromString(Temp));
SetError(FErrorState::ParseError, ErrorDesc);
UngetChar();
}
Token.SetConstString(Temp);
return IsValid();
}
else
{
// Symbol.
int32 Length=0;
Token.Identifier[Length++] = c;
// Handle special 2-character symbols.
#define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */
TCHAR d = GetChar();
if
( PAIR('<','<')
|| PAIR('>','>')
|| PAIR('!','=')
|| PAIR('<','=')
|| PAIR('>','=')
|| PAIR('+','+')
|| PAIR('-','-')
|| PAIR('+','=')
|| PAIR('-','=')
|| PAIR('*','=')
|| PAIR('/','=')
|| PAIR('&','&')
|| PAIR('|','|')
|| PAIR('^','^')
|| PAIR('=','=')
|| PAIR('*','*')
|| PAIR('~','=')
|| PAIR(':',':')
)
{
Token.Identifier[Length++] = d;
if( c=='>' && d=='>' )
{
if( GetChar()=='>' )
Token.Identifier[Length++] = '>';
else
UngetChar();
}
}
else UngetChar();
#undef PAIR
Token.Identifier[Length] = 0;
Token.TokenType = FBasicToken::TOKEN_Symbol;
// Lookup the token's global name.
Token.TokenName = FName( Token.Identifier, FNAME_Find, true );
return true;
}
}