// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line. bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/, ESymbolParseOption bParseTemplateClosingBracket/*=ESymbolParseOption::Normal*/ ) { Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE); Length = ((int32)NAME_SIZE) - 1; break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return true; } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE ); Length = ((int32)NAME_SIZE) - 1; break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return true; } else if (c == '\'') { TCHAR ActualCharLiteral = GetChar(/*bLiteral=*/ true); if (ActualCharLiteral == '\\') { ActualCharLiteral = GetChar(/*bLiteral=*/ true); switch (ActualCharLiteral) { case TCHAR('t'): ActualCharLiteral = '\t'; break; case TCHAR('n'): ActualCharLiteral = '\n'; break; case TCHAR('r'): ActualCharLiteral = '\r'; break; } } c = GetChar(/*bLiteral=*/ true); if (c != '\'') { FError::Throwf(TEXT("Unterminated character constant")); UngetChar(); } Token.SetConstChar(ActualCharLiteral); return true; } else if (c == '"') { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(/*bLiteral=*/ true); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(/*bLiteral=*/ true); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = TEXT('\"'); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(/*bLiteral=*/ true); } Temp[Length]=0; if( c != '"' ) { FError::Throwf(TEXT("Unterminated string constant: %s"), Temp); UngetChar(); } Token.SetConstString(Temp); return true; } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || (PAIR('>','>') && (bParseTemplateClosingBracket != ESymbolParseOption::CloseTemplateBracket)) || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }
// Reads a set of specifiers (with optional values) inside the () of a new-style metadata macro like UPROPERTY or UFUNCTION void FBaseParser::ReadSpecifierSetInsideMacro(TArray<FPropertySpecifier>& SpecifiersFound, const FString& TypeOfSpecifier, TMap<FName, FString>& MetaData) { int32 FoundSpecifierCount = 0; FString ErrorMessage = FString::Printf(TEXT("%s declaration specifier"), *TypeOfSpecifier); RequireSymbol(TEXT("("), *ErrorMessage); while (!MatchSymbol(TEXT(")"))) { if (FoundSpecifierCount > 0) { RequireSymbol(TEXT(","), *ErrorMessage); } ++FoundSpecifierCount; // Read the specifier key FToken Specifier; if (!GetToken(Specifier)) { FError::Throwf(TEXT("Expected %s"), *ErrorMessage); } if (Specifier.Matches(TEXT("meta"))) { RequireSymbol(TEXT("="), *ErrorMessage); RequireSymbol(TEXT("("), *ErrorMessage); // Keep reading comma-separated metadata pairs do { // Read a key FToken MetaKeyToken; if (!GetIdentifier(MetaKeyToken)) { FError::Throwf(TEXT("Expected a metadata key")); } FString Key = MetaKeyToken.Identifier; // Potentially read a value FString Value; if (MatchSymbol(TEXT("="))) { Value = ReadNewStyleValue(TypeOfSpecifier); } // Validate the value is a valid type for the key and insert it into the map InsertMetaDataPair(MetaData, Key, Value); } while ( MatchSymbol(TEXT(",")) ); RequireSymbol(TEXT(")"), *ErrorMessage); } // Look up specifier in metadata dictionary else if (FMetadataKeyword* MetadataKeyword = GetMetadataKeyword(Specifier.Identifier)) { if (MatchSymbol(TEXT("="))) { if (MetadataKeyword->ValueArgument == EMetadataValueArgument::None) { FError::Throwf(TEXT("Incorrect = after metadata specifier '%s'"), Specifier.Identifier); } FString Value = ReadNewStyleValue(TypeOfSpecifier); MetadataKeyword->ApplyToMetadata(MetaData, &Value); } else { if (MetadataKeyword->ValueArgument == EMetadataValueArgument::Required) { FError::Throwf(TEXT("Missing = after metadata specifier '%s'"), Specifier.Identifier); } MetadataKeyword->ApplyToMetadata(MetaData); } } else { // Creating a new specifier SpecifiersFound.Emplace(Specifier.Identifier); // Look for a value for this specifier if (MatchSymbol(TEXT("=")) || PeekSymbol(TEXT("("))) { TArray<FString>& NewPairValues = SpecifiersFound.Last().Values; if (!ReadOptionalCommaSeparatedListInParens(NewPairValues, TypeOfSpecifier)) { FString Value = ReadNewStyleValue(TypeOfSpecifier); NewPairValues.Add(Value); } } } } }
// Gets the next token from the input stream, advancing the variables which keep track of the current input position and line. bool FBaseParser::GetToken( FToken& Token, bool bNoConsts/*=false*/ ) { Token.TokenName = NAME_None; TCHAR c = GetLeadingChar(); TCHAR p = PeekChar(); if( c == 0 ) { UngetChar(); return 0; } Token.StartPos = PrevPos; Token.StartLine = PrevLine; if( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c=='_') ) { // Alphanumeric token. int32 Length=0; do { Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Identifer length exceeds maximum of %i"), (int32)NAME_SIZE); Length = ((int32)NAME_SIZE) - 1; break; } c = GetChar(); } while( ((c>='A')&&(c<='Z')) || ((c>='a')&&(c<='z')) || ((c>='0')&&(c<='9')) || (c=='_') ); UngetChar(); Token.Identifier[Length]=0; // Assume this is an identifier unless we find otherwise. Token.TokenType = TOKEN_Identifier; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); // If const values are allowed, determine whether the identifier represents a constant if ( !bNoConsts ) { // See if the identifier is part of a vector, rotation or other struct constant. // boolean true/false if( Token.Matches(TEXT("true")) ) { Token.SetConstBool(true); return true; } else if( Token.Matches(TEXT("false")) ) { Token.SetConstBool(false); return true; } } return true; } // if const values are allowed, determine whether the non-identifier token represents a const else if ( !bNoConsts && ((c>='0' && c<='9') || ((c=='+' || c=='-') && (p>='0' && p<='9'))) ) { // Integer or floating point constant. bool bIsFloat = 0; int32 Length = 0; bool bIsHex = 0; do { if( c==TEXT('.') ) { bIsFloat = true; } if( c==TEXT('X') || c == TEXT('x') ) { bIsHex = true; } Token.Identifier[Length++] = c; if( Length >= NAME_SIZE ) { FError::Throwf(TEXT("Number length exceeds maximum of %i "), (int32)NAME_SIZE ); Length = ((int32)NAME_SIZE) - 1; break; } c = FChar::ToUpper(GetChar()); } while ((c >= TEXT('0') && c <= TEXT('9')) || (!bIsFloat && c == TEXT('.')) || (!bIsHex && c == TEXT('X')) || (bIsHex && c >= TEXT('A') && c <= TEXT('F'))); Token.Identifier[Length]=0; if (!bIsFloat || c != 'F') { UngetChar(); } if (bIsFloat) { Token.SetConstFloat( FCString::Atof(Token.Identifier) ); } else if (bIsHex) { TCHAR* End = Token.Identifier + FCString::Strlen(Token.Identifier); Token.SetConstInt( FCString::Strtoi(Token.Identifier,&End,0) ); } else { Token.SetConstInt( FCString::Atoi(Token.Identifier) ); } return true; } //@TODO: 'z' is a character literal in C++, not a FName like it was in UnrealScript - do we need this code? // else if( !bNoConsts && c=='\'' ) // { // // Name constant. // int32 Length=0; // c = GetChar(); // while( (c>='A' && c<='Z') || (c>='a' && c<='z') || (c>='0' && c<='9') || (c=='_') || (c=='-') || (c==' ') ) //@FIXME: space in names should be illegal! // { // Token.Identifier[Length++] = c; // if( Length >= NAME_SIZE ) // { // FError::Throwf(TEXT("Name length exceeds maximum of %i"), (int32)NAME_SIZE ); // // trick the error a few lines down // c = TEXT('\''); // Length = ((int32)NAME_SIZE) - 1; // break; // } // c = GetChar(); // } // if( c != '\'' ) // { // UngetChar(); // FError::Throwf(TEXT("Illegal character in name") ); // } // Token.Identifier[Length]=0; // // // Make constant name. // Token.SetConstName( FName(Token.Identifier) ); // return true; // } else if( c=='"' ) { // String constant. TCHAR Temp[MAX_STRING_CONST_SIZE]; int32 Length=0; c = GetChar(1); while( (c!='"') && !IsEOL(c) ) { if( c=='\\' ) { c = GetChar(1); if( IsEOL(c) ) { break; } else if(c == 'n') { // Newline escape sequence. c = '\n'; } } Temp[Length++] = c; if( Length >= MAX_STRING_CONST_SIZE ) { FError::Throwf(TEXT("String constant exceeds maximum of %i characters"), (int32)MAX_STRING_CONST_SIZE ); c = TEXT('\"'); Length = ((int32)MAX_STRING_CONST_SIZE) - 1; break; } c = GetChar(1); } Temp[Length]=0; if( c != '"' ) { FError::Throwf(TEXT("Unterminated string constant: %s"), Temp); UngetChar(); } Token.SetConstString(Temp); return true; } else { // Symbol. int32 Length=0; Token.Identifier[Length++] = c; // Handle special 2-character symbols. #define PAIR(cc,dd) ((c==cc)&&(d==dd)) /* Comparison macro for convenience */ TCHAR d = GetChar(); if ( PAIR('<','<') || PAIR('>','>') || PAIR('!','=') || PAIR('<','=') || PAIR('>','=') || PAIR('+','+') || PAIR('-','-') || PAIR('+','=') || PAIR('-','=') || PAIR('*','=') || PAIR('/','=') || PAIR('&','&') || PAIR('|','|') || PAIR('^','^') || PAIR('=','=') || PAIR('*','*') || PAIR('~','=') || PAIR(':',':') ) { Token.Identifier[Length++] = d; if( c=='>' && d=='>' ) { if( GetChar()=='>' ) Token.Identifier[Length++] = '>'; else UngetChar(); } } else UngetChar(); #undef PAIR Token.Identifier[Length] = 0; Token.TokenType = TOKEN_Symbol; // Lookup the token's global name. Token.TokenName = FName( Token.Identifier, FNAME_Find, true ); return true; } }