// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line. bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/, ESymbolParseOption bParseTemplateClosingBracket/*=ESymbolParseOption::Normal*/ ) { Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE); Length = ((int32)NAME_SIZE) - 1; break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return true; } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE ); Length = ((int32)NAME_SIZE) - 1; break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return true; } else if (c == '\'') { TCHAR ActualCharLiteral = GetChar(/*bLiteral=*/ true); if (ActualCharLiteral == '\\') { ActualCharLiteral = GetChar(/*bLiteral=*/ true); switch (ActualCharLiteral) { case TCHAR('t'): ActualCharLiteral = '\t'; break; case TCHAR('n'): ActualCharLiteral = '\n'; break; case TCHAR('r'): ActualCharLiteral = '\r'; break; } } c = GetChar(/*bLiteral=*/ true); if (c != '\'') { FError::Throwf(TEXT("Unterminated character constant")); UngetChar(); } Token.SetConstChar(ActualCharLiteral); return true; } else if (c == '"') { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(/*bLiteral=*/ true); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(/*bLiteral=*/ true); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = TEXT('\"'); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(/*bLiteral=*/ true); } Temp[Length]=0; if( c != '"' ) { FError::Throwf(TEXT("Unterminated string constant: %s"), Temp); UngetChar(); } Token.SetConstString(Temp); return true; } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || (PAIR('>','>') && (bParseTemplateClosingBracket != ESymbolParseOption::CloseTemplateBracket)) || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line. bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/ ) { Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE); Length = ((int32)NAME_SIZE) - 1; break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return true; } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE ); Length = ((int32)NAME_SIZE) - 1; break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return true; } //@TODO: 'z' is a character literal in C++, not a FName like it was in UnrealScript - do we need this code? // else if( !bNoConsts && c=='\'' ) // { // // Name constant. // int32 Length=0; // c = GetChar(); // while( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c>='0' && c<='9') || (c=='_') || (c=='-') || (c==' ') ) //@FIXME: space in names should be illegal! // { // Token.Identifier[Length++] = c; // if( Length >= NAME_SIZE ) // { // FError::Throwf(TEXT("Name length exceeds maximum of %i"), (int32)NAME_SIZE ); // // trick the error a few lines down // c = TEXT('\''); // Length = ((int32)NAME_SIZE) - 1; // break; // } // c = GetChar(); // } // if( c != '\'' ) // { // UngetChar(); // FError::Throwf(TEXT("Illegal character in name") ); // } // Token.Identifier[Length]=0; // // // Make constant name. // Token.SetConstName( FName(Token.Identifier) ); // return true; // } else if( c=='"' ) { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(1); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(1); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = TEXT('\"'); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(1); } Temp[Length]=0; if( c != '"' ) { FError::Throwf(TEXT("Unterminated string constant: %s"), Temp); UngetChar(); } Token.SetConstString(Temp); return true; } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || PAIR('>','>') || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }