//------------------------------------------------------------------------------ bool FBasicTokenParser::GetRawTokenRespectingQuotes(FBasicToken& Token, TCHAR StopChar/* = TCHAR('\n')*/) { // if the parser is in a bad state, then don't continue parsing (who // knows what will happen!?) if (!IsValid()) { return false; } // Get token after whitespace. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; TCHAR c = GetLeadingChar(); bool bInQuote = false; while( !IsEOL(c) && ((c != StopChar) || bInQuote) ) { if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') ) { break; } if (c == '"') { bInQuote = !bInQuote; } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { Length = ((int32)MAX_STRING_CONST_SIZE) - 1; Temp[Length]=0; // needs to happen for the error description below FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); c = GetChar(true); break; } c = GetChar(true); } UngetChar(); // Get rid of trailing whitespace. while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) ) { Length--; } Temp[Length]=0; if (bInQuote) { FText ErrorDesc = FText::Format(LOCTEXT("NoClosingQuote", "Unterminated quoted string ({0})"), FText::FromString(Temp)); SetError(FErrorState::ParseError, ErrorDesc); } Token.SetConstString(Temp); return Length>0 && IsValid(); }
/** * Put all text from the current position up to either EOL or the StopToken * into Token. Advances the compiler's current position. * * @param Token [out] will contain the text that was parsed * @param StopChar stop processing when this character is reached * * @return the number of character parsed */ bool FBaseParser::GetRawToken( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ ) { // Get token after whitespace. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; TCHAR c = GetLeadingChar(); while( !IsEOL(c) && c != StopChar ) { if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') ) { break; } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); } c = GetChar(true); } UngetChar(); // Get rid of trailing whitespace. while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) ) { Length--; } Temp[Length]=0; Token.SetConstString(Temp); return Length>0; }
bool FBaseParser::GetRawTokenRespectingQuotes( FToken& Token, TCHAR StopChar /* = TCHAR('\n') */ ) { // Get token after whitespace. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; TCHAR c = GetLeadingChar(); bool bInQuote = false; while( !IsEOL(c) && ((c != StopChar) || bInQuote) ) { if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') ) { break; } if (c == '"') { bInQuote = !bInQuote; } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("Identifier exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = GetChar(true); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(true); } UngetChar(); if (bInQuote) { FError::Throwf(TEXT("Unterminated quoted string")); } // Get rid of trailing whitespace. while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) ) { Length--; } Temp[Length]=0; Token.SetConstString(Temp); return Length>0; }
//------------------------------------------------------------------------------ bool FBasicTokenParser::GetRawToken(FBasicToken& Token, TCHAR StopChar/* = TCHAR('\n')*/) { // if the parser is in a bad state, then don't continue parsing (who // knows what will happen!?) if (!IsValid()) { return false; } // Get token after whitespace. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; TCHAR c = GetLeadingChar(); while( !IsEOL(c) && c != StopChar ) { if( (c=='/' && PeekChar()=='/') || (c=='/' && PeekChar()=='*') ) { break; } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { Temp[Length] = 0; FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); } c = GetChar(true); } UngetChar(); // Get rid of trailing whitespace. while( Length>0 && (Temp[Length-1]==' ' || Temp[Length-1]==9 ) ) { Length--; } Temp[Length]=0; Token.SetConstString(Temp); return Length>0; }
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line. bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/, ESymbolParseOption bParseTemplateClosingBracket/*=ESymbolParseOption::Normal*/ ) { Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE); Length = ((int32)NAME_SIZE) - 1; break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return true; } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE ); Length = ((int32)NAME_SIZE) - 1; break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return true; } else if (c == '\'') { TCHAR ActualCharLiteral = GetChar(/*bLiteral=*/ true); if (ActualCharLiteral == '\\') { ActualCharLiteral = GetChar(/*bLiteral=*/ true); switch (ActualCharLiteral) { case TCHAR('t'): ActualCharLiteral = '\t'; break; case TCHAR('n'): ActualCharLiteral = '\n'; break; case TCHAR('r'): ActualCharLiteral = '\r'; break; } } c = GetChar(/*bLiteral=*/ true); if (c != '\'') { FError::Throwf(TEXT("Unterminated character constant")); UngetChar(); } Token.SetConstChar(ActualCharLiteral); return true; } else if (c == '"') { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(/*bLiteral=*/ true); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(/*bLiteral=*/ true); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = TEXT('\"'); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(/*bLiteral=*/ true); } Temp[Length]=0; if( c != '"' ) { FError::Throwf(TEXT("Unterminated string constant: %s"), Temp); UngetChar(); } Token.SetConstString(Temp); return true; } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || (PAIR('>','>') && (bParseTemplateClosingBracket != ESymbolParseOption::CloseTemplateBracket)) || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line. bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/ ) { Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE); Length = ((int32)NAME_SIZE) - 1; break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return true; } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE ); Length = ((int32)NAME_SIZE) - 1; break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return true; } //@TODO: 'z' is a character literal in C++, not a FName like it was in UnrealScript - do we need this code? // else if( !bNoConsts && c=='\'' ) // { // // Name constant. // int32 Length=0; // c = GetChar(); // while( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c>='0' && c<='9') || (c=='_') || (c=='-') || (c==' ') ) //@FIXME: space in names should be illegal! // { // Token.Identifier[Length++] = c; // if( Length >= NAME_SIZE ) // { // FError::Throwf(TEXT("Name length exceeds maximum of %i"), (int32)NAME_SIZE ); // // trick the error a few lines down // c = TEXT('\''); // Length = ((int32)NAME_SIZE) - 1; // break; // } // c = GetChar(); // } // if( c != '\'' ) // { // UngetChar(); // FError::Throwf(TEXT("Illegal character in name") ); // } // Token.Identifier[Length]=0; // // // Make constant name. // Token.SetConstName( FName(Token.Identifier) ); // return true; // } else if( c=='"' ) { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(1); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(1); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = TEXT('\"'); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(1); } Temp[Length]=0; if( c != '"' ) { FError::Throwf(TEXT("Unterminated string constant: %s"), Temp); UngetChar(); } Token.SetConstString(Temp); return true; } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || PAIR('>','>') || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }
//------------------------------------------------------------------------------ bool FBasicTokenParser::GetToken(FBasicToken& Token, bool bNoConsts/* = false*/) { // if the parser is in a bad state, then don't continue parsing (who // knows what will happen!?) if (!IsValid()) { return false; } Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { Length = ((int32)NAME_SIZE) - 1; Token.Identifier[Length]=0; // need this for the error description FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Token.Identifier), FText::AsNumber((int32)NAME_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = FBasicToken::TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return IsValid(); } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { Length = ((int32)NAME_SIZE) - 1; Token.Identifier[Length]=0; // need this for the error description FText ErrorDesc = FText::Format(LOCTEXT("IdTooLong", "Identifer ({0}...) exceeds maximum length of {1}"), FText::FromString(Token.Identifier), FText::AsNumber((int32)NAME_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return IsValid(); } else if( c=='"' ) { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(1); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(1); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { Length = ((int32)MAX_STRING_CONST_SIZE) - 1; Temp[Length]=0; // need this for the error description FText ErrorDesc = FText::Format(LOCTEXT("StringConstTooLong", "String constant ({0}...) exceeds maximum of {1} characters"), FText::FromString(Temp), FText::AsNumber((int32)MAX_STRING_CONST_SIZE)); SetError(FErrorState::ParseError, ErrorDesc); c = TEXT('\"'); break; } c = GetChar(1); } Temp[Length]=0; if( c != '"' ) { FText ErrorDesc = FText::Format(LOCTEXT("NoClosingQuote", "Unterminated quoted string ({0})"), FText::FromString(Temp)); SetError(FErrorState::ParseError, ErrorDesc); UngetChar(); } Token.SetConstString(Temp); return IsValid(); } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || PAIR('>','>') || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = FBasicToken::TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }