TOptional<FExpressionError> FTokenDefinitions::ConsumeToken(FExpressionTokenConsumer& Consumer) const
{
auto& Stream = Consumer.GetStream();
// Skip over whitespace
if (bIgnoreWhitespace)
{
TOptional<FStringToken> Whitespace = Stream.ParseWhitespace();
if (Whitespace.IsSet())
{
Stream.SetReadPos(Whitespace.GetValue());
}
}
if (Stream.IsEmpty())
{
// Trailing whitespace in the expression.
return TOptional<FExpressionError>();
}
const auto* Pos = Stream.GetRead();
// Try each token in turn. First come first served.
for (const auto& Def : Definitions)
{
// Call the token definition
auto Error = Def(Consumer);
if (Error.IsSet())
{
return Error;
}
// If the stream has moved on, the definition added one or more tokens, so
else if (Stream.GetRead() != Pos)
{
return TOptional<FExpressionError>();
}
}
// No token definition matched the stream at its current position - fatal error
FFormatOrderedArguments Args;
Args.Add(FText::FromString(Consumer.GetStream().GetErrorContext()));
Args.Add(Consumer.GetStream().GetPosition());
return FExpressionError(FText::Format(LOCTEXT("LexicalError", "Unrecognized token '{0}' at character {1}"), Args));
}
TOptional<FExpressionError> CompileGroup(const FExpressionToken* GroupStart, const FGuid* StopAt)
{
enum class EState { PreUnary, PostUnary, Binary };
TArray<FWrappedOperator> OperatorStack;
OperatorStack.Reserve(Tokens.Num() - CurrentTokenIndex);
bool bFoundEndOfGroup = StopAt == nullptr;
// Start off looking for a unary operator
EState State = EState::PreUnary;
for (; CurrentTokenIndex < Tokens.Num(); ++CurrentTokenIndex)
{
auto& Token = Tokens[CurrentTokenIndex];
const auto& TypeId = Token.Node.GetTypeId();
if (const FGuid* GroupingEnd = Grammar.GetGrouping(TypeId))
{
// Ignore this token
CurrentTokenIndex++;
// Start of group - recurse
auto Error = CompileGroup(&Token, GroupingEnd);
if (Error.IsSet())
{
return Error;
}
State = EState::PostUnary;
}
else if (StopAt && TypeId == *StopAt)
{
// End of group
bFoundEndOfGroup = true;
break;
}
else if (State == EState::PreUnary)
{
if (Grammar.HasPreUnaryOperator(TypeId))
{
// Make this a unary op
OperatorStack.Emplace(FCompiledToken(FCompiledToken::PreUnaryOperator, MoveTemp(Token)));
}
else if (Grammar.GetBinaryOperatorPrecedence(TypeId))
{
return FExpressionError(FText::Format(LOCTEXT("SyntaxError_NoBinaryOperand", "Syntax error: No operand specified for operator '{0}'"), FText::FromString(Token.Context.GetString())));
}
else if (Grammar.HasPostUnaryOperator(TypeId))
{
// Found a post-unary operator for the preceeding token
State = EState::PostUnary;
// Pop off any pending unary operators
while (OperatorStack.Num() > 0 && OperatorStack.Last().Precedence <= 0)
{
Commands.Add(OperatorStack.Pop(false).Steal());
}
// Make this a post-unary op
OperatorStack.Emplace(FCompiledToken(FCompiledToken::PostUnaryOperator, MoveTemp(Token)));
}
else
{
// Not an operator, so treat it as an ordinary token
Commands.Add(FCompiledToken(FCompiledToken::Operand, MoveTemp(Token)));
State = EState::PostUnary;
}
}
else if (State == EState::PostUnary)
{
if (Grammar.HasPostUnaryOperator(TypeId))
{
// Pop off any pending unary operators
while (OperatorStack.Num() > 0 && OperatorStack.Last().Precedence <= 0)
{
Commands.Add(OperatorStack.Pop(false).Steal());
}
// Make this a post-unary op
OperatorStack.Emplace(FCompiledToken(FCompiledToken::PostUnaryOperator, MoveTemp(Token)));
}
else
{
// Checking for binary operators
if (const int32* Prec = Grammar.GetBinaryOperatorPrecedence(TypeId))
{
// Pop off anything of higher precedence than this one onto the command stack
while (OperatorStack.Num() > 0 && OperatorStack.Last().Precedence < *Prec)
{
Commands.Add(OperatorStack.Pop(false).Steal());
}
// Add the operator itself to the op stack
OperatorStack.Emplace(FCompiledToken(FCompiledToken::BinaryOperator, MoveTemp(Token)), *Prec);
// Check for a unary op again
State = EState::PreUnary;
}
else
{
// Just add the token. It's possible that this is a syntax error (there's no binary operator specified between two tokens),
// But we don't have enough information at this point to say whether or not it is an error
Commands.Add(FCompiledToken(FCompiledToken::Operand, MoveTemp(Token)));
State = EState::PreUnary;
}
}
}
}
if (!bFoundEndOfGroup)
{
return FExpressionError(FText::Format(LOCTEXT("SyntaxError_UnmatchedGroup", "Syntax error: Reached end of expression before matching end of group '{0}' at line {1}:{2}"),
FText::FromString(GroupStart->Context.GetString()),
FText::AsNumber(GroupStart->Context.GetLineNumber()),
FText::AsNumber(GroupStart->Context.GetCharacterIndex())
));
}
// Pop everything off the operator stack, onto the command stack
while (OperatorStack.Num() > 0)
{
Commands.Add(OperatorStack.Pop(false).Token);
}
return TOptional<FExpressionError>();
}
void FTextFormatData::Compile_NoLock()
{
LexedExpression.Reset();
if (SourceType == ESourceType::Text)
{
SourceExpression = SourceText.ToString();
CompiledTextSnapshot = FTextSnapshot(SourceText);
}
CompiledExpressionType = FTextFormat::EExpressionType::Simple;
BaseFormatStringLength = 0;
FormatArgumentEstimateMultiplier = 1;
TValueOrError<TArray<FExpressionToken>, FExpressionError> Result = ExpressionParser::Lex(*SourceExpression, FTextFormatter::Get().GetTextFormatDefinitions());
bool bValidExpression = Result.IsValid();
if (bValidExpression)
{
LexedExpression = Result.StealValue();
// Quickly make sure the tokens are valid (argument modifiers may only follow an argument token)
for (int32 TokenIndex = 0; TokenIndex < LexedExpression.Num(); ++TokenIndex)
{
const FExpressionToken& Token = LexedExpression[TokenIndex];
if (const auto* Literal = Token.Node.Cast<TextFormatTokens::FStringLiteral>())
{
BaseFormatStringLength += Literal->StringLen;
}
else if (auto* Escaped = Token.Node.Cast<TextFormatTokens::FEscapedCharacter>())
{
BaseFormatStringLength += 1;
}
else if (const auto* ArgumentToken = Token.Node.Cast<TextFormatTokens::FArgumentTokenSpecifier>())
{
CompiledExpressionType = FTextFormat::EExpressionType::Complex;
if (LexedExpression.IsValidIndex(TokenIndex + 1))
{
const FExpressionToken& NextToken = LexedExpression[TokenIndex + 1];
// Peek to see if the next token is an argument modifier
if (const auto* ArgumentModifierToken = NextToken.Node.Cast<TextFormatTokens::FArgumentModifierTokenSpecifier>())
{
int32 ArgModLength = 0;
bool ArgModUsesFormatArgs = false;
ArgumentModifierToken->TextFormatArgumentModifier->EstimateLength(ArgModLength, ArgModUsesFormatArgs);
BaseFormatStringLength += ArgModLength;
FormatArgumentEstimateMultiplier += (ArgModUsesFormatArgs) ? 1 : 0;
++TokenIndex; // walk over the argument token so that the next iteration will skip over the argument modifier
continue;
}
}
}
else if (Token.Node.Cast<TextFormatTokens::FArgumentModifierTokenSpecifier>())
{
// Unexpected argument modifier token!
const FText ErrorSourceText = FText::FromString(Token.Context.GetString());
Result = MakeError(FExpressionError(FText::Format(LOCTEXT("UnexpectedArgumentModifierToken", "Unexpected 'argument modifier' token: {0} (token started at index {1})"), ErrorSourceText, Token.Context.GetCharacterIndex())));
bValidExpression = false;
break;
}
}
}
if (!bValidExpression)
{
LexedExpression.Reset();
CompiledExpressionType = FTextFormat::EExpressionType::Invalid;
UE_LOG(LogTextFormatter, Warning, TEXT("Failed to compile text format string '%s': %s"), *SourceExpression, *Result.GetError().Text.ToString());
}
}
