void ParseStringLiteral()
{
StartToken();
lastSourceToken.tokenType = TokenType::StringLiteral;
buffer.clear();
while (index < length && errorMessage.empty())
{
++lastSourceToken.length;
if (Current() == '\\')
{
ParseEscapeSequence();
continue;
}
else if (Current() == '"')
{
Advance();
break;
}
else if (!IsStringLiteralSafe(Current()))
{
errorPosition = lastSourceToken.textPosition;
errorMessage = "invalid character; expected \"";
return;
}
else
{
buffer += Current();
}
Advance();
}
}
void ParseIntegerLiteral()
{
literal.asUInt64 = Current() - '0';
StartToken();
lastSourceToken.tokenType = TokenType::UnsignedIntegerLiteral;
while (index < length)
{
if (IsDigit(Current()))
{
++lastSourceToken.length;
literal.asUInt64 = literal.asUInt64 * 10 + (Current() - '0');
Advance();
}
else if (Current() == '.')
{
buffer.assign(source + lastSourceToken.offset, lastSourceToken.length++);
Advance();
ParseFloatLiteral();
break;
}
else
{
break;
}
}
}
TToken ASNLexer::LookupToken(void)
{
char c = Char();
switch ( c ) {
case ':':
if ( Char(1) == ':' && Char(2) == '=' ) {
StartToken();
AddChars(3);
return T_DEFINE;
}
return T_SYMBOL;
case '-':
case '+':
if ( IsDigit(Char(1)) ) {
StartToken();
AddChar();
return LookupNumber();
}
return T_SYMBOL;
case '\"':
StartToken();
AddChar();
StartString();
LookupString();
return T_STRING;
case '\'':
StartToken();
AddChar();
return LookupBinHexString();
case '[':
StartToken();
AddChar();
LookupTag();
return T_TAG;
default:
if ( IsDigit(c) ) {
StartToken();
AddChar();
return LookupNumber();
}
else if ( c >= 'a' && c <= 'z' ) {
StartToken();
AddChar();
LookupIdentifier();
return T_IDENTIFIER;
}
else if ( c >= 'A' && c <= 'Z' ) {
StartToken();
AddChar();
LookupIdentifier();
return LookupKeyword();
}
return T_SYMBOL;
}
}
void ParseSymbols()
{
StartToken();
switch (Previous())
{
case '/':
if (Current() == '/')
{
ParseLineComment();
break;
}
else if (Current() == '*')
{
ParseBlockComment();
break;
}
// Yes. No break. This is deliberate.
default:
{
String4 tinySource = {};
tinySource.Append(Previous());
lastSourceToken.tokenType = TokenType::Reserved;
lastSourceToken.tokenIndex = FindOperator(tinySource);
for (int i = index; i < length && IsSymbol(source[i]); ++i)
{
tinySource.Append(source[i]);
auto tokenIndex = FindOperator(tinySource);
if (tokenIndex != -1)
{
lastSourceToken.tokenIndex = tokenIndex;
lastSourceToken.length = i - index + 2;
}
}
int offset = lastSourceToken.length - 1;
index += offset;
position.column += offset;
break;
}
}
}
void ParseIdentifier()
{
StartToken();
lastSourceToken.tokenType = TokenType::Identifier;
while (index < length && IsIdentifierSafe(Current()))
{
++lastSourceToken.length;
Advance();
}
buffer.assign(source + lastSourceToken.offset, lastSourceToken.length);
auto tokenIndex = FindKeyword(buffer.data());
if (tokenIndex != -1)
{
lastSourceToken.tokenType = TokenType::Reserved;
lastSourceToken.tokenIndex = tokenIndex;
}
}
void ParseCodePointLiteral()
{
StartToken();
lastSourceToken.tokenType = TokenType::CodePointLiteral;
buffer.clear();
while (index < length && errorMessage.empty())
{
++lastSourceToken.length;
if (Current() == '\'')
{
if (buffer.size() > 1)
{
errorMessage = "character literal may not contain multiple characters";
errorPosition = lastSourceToken.textPosition;
}
Advance();
break;
}
else if (Current() == '\\')
{
ParseEscapeSequence();
continue;
}
else
{
buffer += Current();
Advance();
}
}
if (buffer.size() < 1)
literal.asCodePoint = 0;
else
literal.asCodePoint = buffer[0];
}
