void FICUCamelCaseBreakIterator::TokenizeString(TArray<FToken>& OutTokens)
{
OutTokens.Empty(String.Len());
FICUTextCharacterIterator CharIter(String);
for(CharIter.setToStart(); CharIter.current32() != FICUTextCharacterIterator::DONE; CharIter.next32PostInc())
{
const UChar32 CurrentChar = CharIter.current32();
ETokenType TokenType = ETokenType::Other;
if(u_isULowercase(CurrentChar))
{
TokenType = ETokenType::Lowercase;
}
else if(u_isUUppercase(CurrentChar))
{
TokenType = ETokenType::Uppercase;
}
else if(u_isdigit(CurrentChar))
{
TokenType = ETokenType::Digit;
}
const int32 CharIndex = CharIter.InternalIndexToSourceIndex(CharIter.getIndex());
OutTokens.Emplace(FToken(TokenType, CharIndex));
}
OutTokens.Emplace(FToken(ETokenType::Null, String.Len()));
// There should always be at least one token for the end of the string
check(OutTokens.Num());
}
void FLegacyCamelCaseBreakIterator::TokenizeString(TArray<FToken>& OutTokens)
{
OutTokens.Empty(String.Len());
for(int32 CurrentCharIndex = 0; CurrentCharIndex < String.Len(); ++CurrentCharIndex)
{
const TCHAR CurrentChar = String[CurrentCharIndex];
ETokenType TokenType = ETokenType::Other;
if(FChar::IsLower(CurrentChar))
{
TokenType = ETokenType::Lowercase;
}
else if(FChar::IsUpper(CurrentChar))
{
TokenType = ETokenType::Uppercase;
}
else if(FChar::IsDigit(CurrentChar))
{
TokenType = ETokenType::Digit;
}
OutTokens.Emplace(FToken(TokenType, CurrentCharIndex));
}
OutTokens.Emplace(FToken(ETokenType::Null, String.Len()));
// There should always be at least one token for the end of the string
check(OutTokens.Num());
}
