Ejemplo n.º 1
0
Int Lexer_ParseDigit(Lexer lexer, Bool isFirstContentOnLine)
{
	const Byte *src = lexer->src;
	const Byte *start, *digitsEnd;
	Token token = lexer->token;
	UInt64 value;
	String suffix;

	START_TOKEN(src - 1);
	start = src - 1;

	// Decimal integer, or possibly a real value (if we find a '.').
	if (!ParseDecimalInteger(lexer, &value)) {
		lexer->token->text = IllegalDecimalIntegerMessage;
		return END_TOKEN(TOKEN_ERROR);
	}
	digitsEnd = src = lexer->src;
	if (src < lexer->end && *src == '.' && (src + 1 == lexer->end || src[1] != '.')) {
		// Found a '.' (and it's not part of a ".."), so rewind back and re-parse this as a real or float value.
		lexer->src = start;
		return Lexer_ParseReal(lexer, isFirstContentOnLine);
	}
	else {
		// Collected a whole octal value, so finish it.
		suffix = CollectAlphanumericSuffix(lexer);
		if (!EnsureEndOfNumber(lexer)) return END_TOKEN(TOKEN_ERROR);
		END_TOKEN(TOKEN_INTEGER32);
		return ProcessIntegerValue(lexer, value, String_Create(start, digitsEnd - start), suffix);
	}
}
Ejemplo n.º 2
0
/// <summary>
/// Having seen a dot '.' in the input, parse one of the following four kinds of tokens:
///
///   .     (dot)
///   ..    (range)
///   ...   (ellipsis)
///   .123  (real value)
///
/// We need to figure out which of these four cases we're dealing with, and decode
/// the input accordingly into the correct kind of token.
/// </summary>
/// <param name="lexer">The lexical analyzer.</param>
/// <param name="isFirstContentOnLine">Whether this token is the first non-whitespace non-comment content on its line.</param>
/// <returns>The next token that was found in the input.</returns>
Int Lexer_ParseDot(Lexer lexer, Bool isFirstContentOnLine)
{
	const Byte *src = lexer->src;
	const Byte *end = lexer->end;
	Token token = lexer->token;
	Byte ch;

	if (src >= end)
		return SIMPLE_TOKEN(src - 1, TOKEN_DOT);

	if ((ch = *src) == '.') {
		src++;
		// Two dots, so range or ellipsis.
		if (src < end && (ch = *src) == '.') {
			src++;
			// Three dots, so ellipsis.
			return SIMPLE_TOKEN(src - 3, TOKEN_DOTDOTDOT);
		}
		else return SIMPLE_TOKEN(src - 2, TOKEN_DOTDOT);
	}

	if (ch >= '0' && ch <= '9') {
		// We got numeric digits after the dot, so this is actually a real value, not a dot operator.
		lexer->src--;
		return Lexer_ParseReal(lexer, isFirstContentOnLine);
	}

	return SIMPLE_TOKEN(src - 1, TOKEN_DOT);
}
Ejemplo n.º 3
0
Int Lexer_ParseZero(Lexer lexer, Bool isFirstContentOnLine)
{
	const Byte *src = lexer->src;
	const Byte *end = lexer->end;
	const Byte *start, *digitsEnd;
	Token token = lexer->token;
	Byte ch;
	UInt64 value;
	String suffix;

	START_TOKEN(src - 1);
	start = src - 1;

	if (src < end && ((ch = *src) == 'x' || ch == 'X')) {
		// Hexadecimal integer, or possibly a zero byte.
		src++;

		if (src >= end || !(((ch = *src) >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'))) {
			// Not an error; this is decimal zero, as a byte.
			lexer->src = src;
			if (!EnsureEndOfNumber(lexer)) {
				lexer->token->text = String_FormatString(IllegalNumericSuffixMessage, String_Format("x%c", ch));
				return END_TOKEN(TOKEN_ERROR);
			}
			END_TOKEN(TOKEN_INTEGER32);
			return ProcessIntegerValue(lexer, 0, ZeroString, LowercaseXString);
		}
		else {
			// This is a hexadecimal integer.
			lexer->src = src;
			if (!ParseHexadecimalInteger(lexer, &value)) {
				lexer->token->text = IllegalHexadecimalIntegerMessage;
				return END_TOKEN(TOKEN_ERROR);
			}
			digitsEnd = src = lexer->src;
			suffix = CollectAlphanumericSuffix(lexer);
			if (!EnsureEndOfNumber(lexer)) {
				lexer->token->text = String_FormatString(IllegalNumericSuffixMessage, String_Format("%c", *lexer->src));
				return END_TOKEN(TOKEN_ERROR);
			}
			END_TOKEN(TOKEN_INTEGER32);
			return ProcessIntegerValue(lexer, value, String_Create(start, digitsEnd - start), suffix);
		}
	}
	else {
		// Octal integer, or possibly a real value (if we find a '.').
		if (!ParseOctalInteger(lexer, &value)) {
			lexer->token->text = IllegalOctalIntegerMessage;
			return END_TOKEN(TOKEN_ERROR);
		}
		digitsEnd = src = lexer->src;
		if (src < lexer->end && *src == '.' && (src+1 >= lexer->end || src[1] != '.')) {
			// Found a '.' (and it's not part of a ".."), so rewind back and re-parse this as a real or float value.
			lexer->src = start;
			return Lexer_ParseReal(lexer, isFirstContentOnLine);
		}
		else {
			// Collected a whole octal value, so finish it.
			suffix = CollectAlphanumericSuffix(lexer);
			if (!EnsureEndOfNumber(lexer)) {
				lexer->token->text = String_FormatString(IllegalNumericSuffixMessage, String_Format("%c", *lexer->src));
				return END_TOKEN(TOKEN_ERROR);
			}
			END_TOKEN(TOKEN_INTEGER32);
			return ProcessIntegerValue(lexer, value, String_Create(start, digitsEnd - start), suffix);
		}
	}
}