Beispiel #1
0
static void readToken (tokenInfo *const token)
{
	int c;

	token->type			= TOKEN_UNDEFINED;
	token->keyword		= KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:
	do
	{
		c = fileGetc ();
	}
	while (c == '\t'  ||  c == ' ' ||  c == '\n');

	token->lineNumber   = getSourceLineNumber ();
	token->filePosition = getInputFilePosition ();

	switch (c)
	{
		case EOF: longjmp (Exception, (int)ExceptionEOF);	break;
		case '(': token->type = TOKEN_OPEN_PAREN;			break;
		case ')': token->type = TOKEN_CLOSE_PAREN;			break;
		case ';': token->type = TOKEN_SEMICOLON;			break;
		case ',': token->type = TOKEN_COMMA;				break;
		case '.': token->type = TOKEN_PERIOD;				break;
		case ':': token->type = TOKEN_COLON;				break;
		case '{': token->type = TOKEN_OPEN_CURLY;			break;
		case '}': token->type = TOKEN_CLOSE_CURLY;			break;
		case '=': token->type = TOKEN_EQUAL_SIGN;			break;
		case '[': token->type = TOKEN_OPEN_SQUARE;			break;
		case ']': token->type = TOKEN_CLOSE_SQUARE;			break;

		case '\'':
		case '"':
				  token->type = TOKEN_STRING;
				  parseString (token->string, c);
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  break;

		case '\\':
				  c = fileGetc ();
				  if (c != '\\'  && c != '"'  &&  !isspace (c))
					  fileUngetc (c);
				  token->type = TOKEN_CHARACTER;
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  break;

		case '/':
				  {
					  int d = fileGetc ();
					  if ( (d != '*') &&		/* is this the start of a comment? */
							  (d != '/') )		/* is a one line comment? */
					  {
						  fileUngetc (d);
						  switch (LastTokenType)
						  {
							  case TOKEN_CHARACTER:
							  case TOKEN_KEYWORD:
							  case TOKEN_IDENTIFIER:
							  case TOKEN_STRING:
							  case TOKEN_CLOSE_CURLY:
							  case TOKEN_CLOSE_PAREN:
							  case TOKEN_CLOSE_SQUARE:
								  token->type = TOKEN_FORWARD_SLASH;
								  break;

							  default:
								  token->type = TOKEN_REGEXP;
								  parseRegExp ();
								  token->lineNumber = getSourceLineNumber ();
								  token->filePosition = getInputFilePosition ();
								  break;
						  }
					  }
					  else
					  {
						  if (d == '*')
						  {
							  do
							  {
								  skipToCharacter ('*');
								  c = fileGetc ();
								  if (c == '/')
									  break;
								  else
									  fileUngetc (c);
							  } while (c != EOF && c != '\0');
							  goto getNextChar;
						  }
						  else if (d == '/')	/* is this the start of a comment?  */
						  {
							  skipToCharacter ('\n');
							  goto getNextChar;
						  }
					  }
					  break;
				  }

		default:
				  if (! isIdentChar (c))
					  token->type = TOKEN_UNDEFINED;
				  else
				  {
					  parseIdentifier (token->string, c);
					  token->lineNumber = getSourceLineNumber ();
					  token->filePosition = getInputFilePosition ();
					  token->keyword = analyzeToken (token->string);
					  if (isKeyword (token, KEYWORD_NONE))
						  token->type = TOKEN_IDENTIFIER;
					  else
						  token->type = TOKEN_KEYWORD;
				  }
				  break;
	}

	LastTokenType = token->type;
}
Beispiel #2
0
static void readToken (tokenInfo * const token)
{
	int c;

	token->type = TOKEN_NONE;
	token->keyword = KEYWORD_NONE;
	vStringClear (token->string);

  getNextChar:
	do
	{
		c = fileGetc ();
		token->lineNumber = getSourceLineNumber ();
		token->filePosition = getInputFilePosition ();
	}
	while (c == '\t' || c == ' ' || c == '\n');

	switch (c)
	{
	case EOF:
		token->type = TOKEN_EOF;
		break;
	case '(':
		token->type = TOKEN_OPEN_PAREN;
		break;
	case ')':
		token->type = TOKEN_CLOSE_PAREN;
		break;
	case ';':
		token->type = TOKEN_SEMICOLON;
		break;
	case '.':
		token->type = TOKEN_PERIOD;
		break;
	case ',':
		token->type = TOKEN_COMMA;
		break;
	case '\'':	/* only single char are inside simple quotes */
		break;	/* or it is for attributes so we don't care */
	case '"':
		token->type = TOKEN_STRING;
		parseString (token->string, c);
		token->lineNumber = getSourceLineNumber ();
		token->filePosition = getInputFilePosition ();
		break;
	case '-':
		c = fileGetc ();
		if (c == '-')	/* start of a comment */
		{
			fileSkipToCharacter ('\n');
			goto getNextChar;
		}
		else
		{
			if (!isspace (c))
				fileUngetc (c);
			token->type = TOKEN_OPERATOR;
		}
		break;
	default:
		if (!isIdentChar1 (c))
			token->type = TOKEN_NONE;
		else
		{
			parseIdentifier (token->string, c);
			token->lineNumber = getSourceLineNumber ();
			token->filePosition = getInputFilePosition ();
			token->keyword = analyzeToken (token->string, Lang_vhdl);
			if (isKeyword (token, KEYWORD_NONE))
				token->type = TOKEN_IDENTIFIER;
			else
				token->type = TOKEN_KEYWORD;
		}
		break;
	}
}
Beispiel #3
0
static void readToken (tokenInfo *const token)
{
	int c;

	token->type		= TOKEN_UNDEFINED;
	token->keyword	= KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:

	if (! InPhp)
	{
		c = findPhpStart ();
		if (c != EOF)
			InPhp = TRUE;
	}
	else
		c = getcFromInputFile ();

	c = skipWhitespaces (c);

	token->lineNumber   = getInputLineNumber ();
	token->filePosition = getInputFilePosition ();

	switch (c)
	{
		case EOF: token->type = TOKEN_EOF;					break;
		case '(': token->type = TOKEN_OPEN_PAREN;			break;
		case ')': token->type = TOKEN_CLOSE_PAREN;			break;
		case ';': token->type = TOKEN_SEMICOLON;			break;
		case ',': token->type = TOKEN_COMMA;				break;
		case '.': token->type = TOKEN_PERIOD;				break;
		case ':': token->type = TOKEN_COLON;				break;
		case '{': token->type = TOKEN_OPEN_CURLY;			break;
		case '}': token->type = TOKEN_CLOSE_CURLY;			break;
		case '[': token->type = TOKEN_OPEN_SQUARE;			break;
		case ']': token->type = TOKEN_CLOSE_SQUARE;			break;
		case '&': token->type = TOKEN_AMPERSAND;			break;
		case '\\': token->type = TOKEN_BACKSLASH;			break;

		case '=':
		{
			int d = getcFromInputFile ();
			if (d == '=' || d == '>')
				token->type = TOKEN_OPERATOR;
			else
			{
				ungetcToInputFile (d);
				token->type = TOKEN_EQUAL_SIGN;
			}
			break;
		}

		case '\'':
		case '"':
			token->type = TOKEN_STRING;
			parseString (token->string, c);
			token->lineNumber = getInputLineNumber ();
			token->filePosition = getInputFilePosition ();
			break;

		case '<':
		{
			int d = getcFromInputFile ();
			if (d == '/')
			{
				/* </script[:white:]*> */
				if (tolower ((d = getcFromInputFile ())) == 's' &&
					tolower ((d = getcFromInputFile ())) == 'c' &&
					tolower ((d = getcFromInputFile ())) == 'r' &&
					tolower ((d = getcFromInputFile ())) == 'i' &&
					tolower ((d = getcFromInputFile ())) == 'p' &&
					tolower ((d = getcFromInputFile ())) == 't' &&
					(d = skipWhitespaces (getcFromInputFile ())) == '>')
				{
					InPhp = FALSE;
					goto getNextChar;
				}
				else
				{
					ungetcToInputFile (d);
					token->type = TOKEN_UNDEFINED;
				}
			}
			else if (d == '<' && (d = getcFromInputFile ()) == '<')
			{
				token->type = TOKEN_STRING;
				parseHeredoc (token->string);
			}
			else
			{
				ungetcToInputFile (d);
				token->type = TOKEN_UNDEFINED;
			}
			break;
		}

		case '#': /* comment */
			skipSingleComment ();
			goto getNextChar;
			break;

		case '+':
		case '-':
		case '*':
		case '%':
		{
			int d = getcFromInputFile ();
			if (d != '=' && ! (c == '-' && d == '>'))
				ungetcToInputFile (d);
			token->type = TOKEN_OPERATOR;
			break;
		}

		case '/': /* division or comment start */
		{
			int d = getcFromInputFile ();
			if (d == '/') /* single-line comment */
			{
				skipSingleComment ();
				goto getNextChar;
			}
			else if (d == '*')
			{
				do
				{
					c = skipToCharacter ('*');
					if (c != EOF)
					{
						c = getcFromInputFile ();
						if (c == '/')
							break;
						else
							ungetcToInputFile (c);
					}
				} while (c != EOF && c != '\0');
				goto getNextChar;
			}
			else
			{
				if (d != '=')
					ungetcToInputFile (d);
				token->type = TOKEN_OPERATOR;
			}
			break;
		}

		case '$': /* variable start */
		{
			int d = getcFromInputFile ();
			if (! isIdentChar (d))
			{
				ungetcToInputFile (d);
				token->type = TOKEN_UNDEFINED;
			}
			else
			{
				parseIdentifier (token->string, d);
				token->type = TOKEN_VARIABLE;
			}
			break;
		}

		case '?': /* maybe the end of the PHP chunk */
		{
			int d = getcFromInputFile ();
			if (d == '>')
			{
				InPhp = FALSE;
				goto getNextChar;
			}
			else
			{
				ungetcToInputFile (d);
				token->type = TOKEN_UNDEFINED;
			}
			break;
		}

		default:
			if (! isIdentChar (c))
				token->type = TOKEN_UNDEFINED;
			else
			{
				parseIdentifier (token->string, c);
				token->keyword = analyzeToken (token->string, getInputLanguage ());
				if (token->keyword == KEYWORD_NONE)
					token->type = TOKEN_IDENTIFIER;
				else
					token->type = TOKEN_KEYWORD;
			}
			break;
	}

	if (token->type == TOKEN_SEMICOLON ||
		token->type == TOKEN_OPEN_CURLY ||
		token->type == TOKEN_CLOSE_CURLY)
	{
		/* reset current statement details on statement end, and when entering
		 * a deeper scope.
		 * it is a bit ugly to do this in readToken(), but it makes everything
		 * a lot simpler. */
		CurrentStatement.access = ACCESS_UNDEFINED;
		CurrentStatement.impl = IMPL_UNDEFINED;
	}
}
Beispiel #4
0
static void readTokenFull (tokenInfo *const token, boolean include_newlines, vString *const repr)
{
	int c;
	int i;

	token->type			= TOKEN_UNDEFINED;
	token->keyword		= KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:
	i = 0;
	do
	{
		c = fileGetc ();
		i++;
	}
	while (c == '\t'  ||  c == ' ' ||
		   ((c == '\r' || c == '\n') && ! include_newlines));

	token->lineNumber   = getSourceLineNumber ();
	token->filePosition = getInputFilePosition ();

	if (repr)
	{
		if (i > 1)
			vStringPut (repr, ' ');
		vStringPut (repr, c);
	}

	switch (c)
	{
		case EOF: token->type = TOKEN_EOF;					break;
		case '(': token->type = TOKEN_OPEN_PAREN;			break;
		case ')': token->type = TOKEN_CLOSE_PAREN;			break;
		case ';': token->type = TOKEN_SEMICOLON;			break;
		case ',': token->type = TOKEN_COMMA;				break;
		case '.': token->type = TOKEN_PERIOD;				break;
		case ':': token->type = TOKEN_COLON;				break;
		case '{': token->type = TOKEN_OPEN_CURLY;			break;
		case '}': token->type = TOKEN_CLOSE_CURLY;			break;
		case '=': token->type = TOKEN_EQUAL_SIGN;			break;
		case '[': token->type = TOKEN_OPEN_SQUARE;			break;
		case ']': token->type = TOKEN_CLOSE_SQUARE;			break;

		case '+':
		case '-':
			{
				int d = fileGetc ();
				if (d == c) /* ++ or -- */
					token->type = TOKEN_POSTFIX_OPERATOR;
				else
				{
					fileUngetc (d);
					token->type = TOKEN_BINARY_OPERATOR;
				}
				break;
			}

		case '*':
		case '%':
		case '?':
		case '>':
		case '<':
		case '^':
		case '|':
		case '&':
			token->type = TOKEN_BINARY_OPERATOR;
			break;

		case '\r':
		case '\n':
			/* This isn't strictly correct per the standard, but following the
			 * real rules means understanding all statements, and that's not
			 * what the parser currently does.  What we do here is a guess, by
			 * avoiding inserting semicolons that would make the statement on
			 * the left invalid.  Hopefully this should not have false negatives
			 * (e.g. should not miss insertion of a semicolon) but might have
			 * false positives (e.g. it will wrongfully emit a semicolon for the
			 * newline in "foo\n+bar").
			 * This should however be mostly harmless as we only deal with
			 * newlines in specific situations where we know a false positive
			 * wouldn't hurt too bad. */
			switch (LastTokenType)
			{
				/* these cannot be the end of a statement, so hold the newline */
				case TOKEN_EQUAL_SIGN:
				case TOKEN_COLON:
				case TOKEN_PERIOD:
				case TOKEN_FORWARD_SLASH:
				case TOKEN_BINARY_OPERATOR:
				/* and these already end one, no need to duplicate it */
				case TOKEN_SEMICOLON:
				case TOKEN_COMMA:
				case TOKEN_CLOSE_CURLY:
				case TOKEN_OPEN_CURLY:
					include_newlines = FALSE; /* no need to recheck */
					goto getNextChar;
					break;
				default:
					token->type = TOKEN_SEMICOLON;
			}
			break;

		case '\'':
		case '"':
				  token->type = TOKEN_STRING;
				  parseString (token->string, c);
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  if (repr)
				  {
					  vStringCat (repr, token->string);
					  vStringPut (repr, c);
				  }
				  break;

		case '\\':
				  c = fileGetc ();
				  if (c != '\\'  && c != '"'  &&  !isspace (c))
					  fileUngetc (c);
				  token->type = TOKEN_CHARACTER;
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  break;

		case '/':
				  {
					  int d = fileGetc ();
					  if ( (d != '*') &&		/* is this the start of a comment? */
							  (d != '/') )		/* is a one line comment? */
					  {
						  fileUngetc (d);
						  switch (LastTokenType)
						  {
							  case TOKEN_CHARACTER:
							  case TOKEN_IDENTIFIER:
							  case TOKEN_STRING:
							  case TOKEN_CLOSE_CURLY:
							  case TOKEN_CLOSE_PAREN:
							  case TOKEN_CLOSE_SQUARE:
								  token->type = TOKEN_FORWARD_SLASH;
								  break;

							  default:
								  token->type = TOKEN_REGEXP;
								  parseRegExp ();
								  token->lineNumber = getSourceLineNumber ();
								  token->filePosition = getInputFilePosition ();
								  break;
						  }
					  }
					  else
					  {
						  if (repr) /* remove the / we added */
							  repr->buffer[--repr->length] = 0;
						  if (d == '*')
						  {
							  do
							  {
								  skipToCharacter ('*');
								  c = fileGetc ();
								  if (c == '/')
									  break;
								  else
									  fileUngetc (c);
							  } while (c != EOF && c != '\0');
							  goto getNextChar;
						  }
						  else if (d == '/')	/* is this the start of a comment?  */
						  {
							  skipToCharacter ('\n');
							  /* if we care about newlines, put it back so it is seen */
							  if (include_newlines)
								  fileUngetc ('\n');
							  goto getNextChar;
						  }
					  }
					  break;
				  }

		case '#':
				  /* skip shebang in case of e.g. Node.js scripts */
				  if (token->lineNumber > 1)
					  token->type = TOKEN_UNDEFINED;
				  else if ((c = fileGetc ()) != '!')
				  {
					  fileUngetc (c);
					  token->type = TOKEN_UNDEFINED;
				  }
				  else
				  {
					  skipToCharacter ('\n');
					  goto getNextChar;
				  }
				  break;

		default:
				  if (! isIdentChar (c))
					  token->type = TOKEN_UNDEFINED;
				  else
				  {
					  parseIdentifier (token->string, c);
					  token->lineNumber = getSourceLineNumber ();
					  token->filePosition = getInputFilePosition ();
					  token->keyword = analyzeToken (token->string);
					  if (isKeyword (token, KEYWORD_NONE))
						  token->type = TOKEN_IDENTIFIER;
					  else
						  token->type = TOKEN_KEYWORD;
					  if (repr && vStringLength (token->string) > 1)
						  vStringCatS (repr, vStringValue (token->string) + 1);
				  }
				  break;
	}

	LastTokenType = token->type;
}
Beispiel #5
0
static void readToken (tokenInfo *const token)
{
	int c;

	token->type    = TOKEN_UNDEFINED;
	token->keyword = KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:

	do
		c = getcFromInputFile ();
	while (c == '\t'  ||  c == ' '  ||  c == '\n');

	switch (c)
	{
		case EOF:  token->type = TOKEN_EOF;                break;
		case ';':  token->type = TOKEN_SEMICOLON;          break;
		case '!':  token->type = TOKEN_BANG;               break;
		case '}':  token->type = TOKEN_CLOSE_BRACE;        break;
		case ']':  token->type = TOKEN_CLOSE_BRACKET;      break;
		case ')':  token->type = TOKEN_CLOSE_PAREN;        break;
		case ',':  token->type = TOKEN_COMMA;              break;
		case '$':  token->type = TOKEN_DOLLAR;             break;
		case '.':  token->type = TOKEN_DOT;                break;
		case '{':  token->type = TOKEN_OPEN_BRACE;         break;
		case '[':  token->type = TOKEN_OPEN_BRACKET;       break;
		case '(':  token->type = TOKEN_OPEN_PAREN;         break;
		case '~':  token->type = TOKEN_TILDE;              break;


		case '+':
		case '*':
		case '^':
		case '=':  token->type = TOKEN_OPERATOR;           break;

		case '-':
			c = getcFromInputFile ();
			if (c == '>')
				token->type = TOKEN_CONSTRAINT;
			else if (c == '-')  /* is this the start of a comment? */
			{
				skipToCharacter ('\n');
				goto getNextChar;
			}
			else
			{
				if (!isspace (c))
					ungetcToInputFile (c);
				token->type = TOKEN_OPERATOR;
			}
			break;

		case '?':
		case ':':
		{
			int c2 = getcFromInputFile ();
			if (c2 == '=')
				token->type = TOKEN_OPERATOR;
			else
			{
				if (!isspace (c2))
					ungetcToInputFile (c2);
				if (c == ':')
					token->type = TOKEN_COLON;
				else
					token->type = TOKEN_QUESTION;
			}
			break;
		}

		case '<':
			c = getcFromInputFile ();
			if (c != '='  &&  c != '>'  &&  !isspace (c))
				ungetcToInputFile (c);
			token->type = TOKEN_OPERATOR;
			break;

		case '>':
			c = getcFromInputFile ();
			if (c != '='  &&  c != '>'  &&  !isspace (c))
				ungetcToInputFile (c);
			token->type = TOKEN_OPERATOR;
			break;

		case '/':
			c = getcFromInputFile ();
			if (c != '/'  &&  c != '='  &&  !isspace (c))
				ungetcToInputFile (c);
			token->type = TOKEN_OPERATOR;
			break;

		case '\\':
			c = getcFromInputFile ();
			if (c != '\\'  &&  !isspace (c))
				ungetcToInputFile (c);
			token->type = TOKEN_OPERATOR;
			break;

		case '"':
			token->type = TOKEN_STRING;
			parseString (token->string);
			break;

		case '\'':
			token->type = TOKEN_CHARACTER;
			parseCharacter ();
			break;

		default:
			if (isalpha (c))
			{
				parseIdentifier (token->string, c);
				token->keyword = analyzeToken (token->string, Lang_eiffel);
				if (isKeyword (token, KEYWORD_NONE))
					token->type = TOKEN_IDENTIFIER;
				else
					token->type = TOKEN_KEYWORD;
			}
			else if (isdigit (c))
			{
				vString* numeric = parseNumeric (c);
				vStringCat (token->string, numeric);
				vStringDelete (numeric);
				token->type = TOKEN_NUMERIC;
			}
			else if (isFreeOperatorChar (c))
			{
				parseFreeOperator (token->string, c);
				token->type = TOKEN_OPERATOR;
			}
			else
				token->type = TOKEN_UNDEFINED;
			break;
	}
}
Beispiel #6
0
static void readToken (tokenInfo *const token)
{
    int c;

    token->type         = TOKEN_UNDEFINED;
    token->keyword      = KEYWORD_NONE;
    vStringClear (token->string);

getNextChar:
    do
    {
        c = fileGetc ();
        token->lineNumber   = getSourceLineNumber ();
        token->filePosition = getInputFilePosition ();
    }
    while (c == '\t'  ||  c == ' ' ||  c == '\n');

    switch (c)
    {
        case EOF: longjmp (Exception, (int)ExceptionEOF);   break;
        case '(': token->type = TOKEN_OPEN_PAREN;           break;
        case ')': token->type = TOKEN_CLOSE_PAREN;          break;
        case ';': token->type = TOKEN_SEMICOLON;            break;
        case ',': token->type = TOKEN_COMMA;                break;
        case '.': token->type = TOKEN_PERIOD;               break;
        case ':': token->type = TOKEN_COLON;                break;
        case '{': token->type = TOKEN_OPEN_CURLY;           break;
        case '}': token->type = TOKEN_CLOSE_CURLY;          break;
        case '=': token->type = TOKEN_EQUAL_SIGN;           break;
        case '[': token->type = TOKEN_OPEN_SQUARE;          break;
        case ']': token->type = TOKEN_CLOSE_SQUARE;         break;
        case '?': token->type = TOKEN_QUESTION_MARK;        break;
        case '*': token->type = TOKEN_STAR;                 break;

        case '\'':
        case '"':
                  token->type = TOKEN_STRING;
                  parseString (token->string, c);
                  token->lineNumber = getSourceLineNumber ();
                  token->filePosition = getInputFilePosition ();
                  break;

        case '\\':
                  /*
                   * All Tex tags start with a backslash.
                   * Check if the next character is an alpha character
                   * else it is not a potential tex tag.
                   */
                  c = fileGetc ();
                  if (! isalpha (c))
                      fileUngetc (c);
                  else
                  {
                      parseIdentifier (token->string, c);
                      token->lineNumber = getSourceLineNumber ();
                      token->filePosition = getInputFilePosition ();
                      token->keyword = analyzeToken (token->string, Lang_js);
                      if (isKeyword (token, KEYWORD_NONE))
                          token->type = TOKEN_IDENTIFIER;
                      else
                          token->type = TOKEN_KEYWORD;
                  }
                  break;

        case '%':
                  fileSkipToCharacter ('\n'); /* % are single line comments */
                  goto getNextChar;
                  break;

        default:
                  if (! isIdentChar (c))
                      token->type = TOKEN_UNDEFINED;
                  else
                  {
                      parseIdentifier (token->string, c);
                      token->lineNumber = getSourceLineNumber ();
                      token->filePosition = getInputFilePosition ();
                      token->type = TOKEN_IDENTIFIER;
                  }
                  break;
    }
}
Beispiel #7
0
static void readToken (tokenInfo *const token)
{
    int c;

    token->type    = TOKEN_UNDEFINED;
    token->keyword = KEYWORD_NONE;
    vStringClear (token->string);

getNextChar:

    do
        c = fileGetc ();
    while (c == '\t'  ||  c == ' '  ||  c == '\n');

    switch (c)
    {
    case EOF:
        longjmp (Exception, (int)ExceptionEOF);
        break;
    case '!':
        token->type = TOKEN_BANG;
        break;
    case '$':
        token->type = TOKEN_DOLLAR;
        break;
    case '(':
        token->type = TOKEN_OPEN_PAREN;
        break;
    case ')':
        token->type = TOKEN_CLOSE_PAREN;
        break;
    case ',':
        token->type = TOKEN_COMMA;
        break;
    case '.':
        token->type = TOKEN_DOT;
        break;
    case ';':
        goto getNextChar;
    case '[':
        token->type = TOKEN_OPEN_BRACKET;
        break;
    case ']':
        token->type = TOKEN_CLOSE_BRACKET;
        break;
    case '{':
        token->type = TOKEN_OPEN_BRACE;
        break;
    case '}':
        token->type = TOKEN_CLOSE_BRACE;
        break;
    case '~':
        token->type = TOKEN_TILDE;
        break;


    case '+':
    case '*':
    case '^':
    case '=':
        token->type = TOKEN_OPERATOR;
        break;

    case '-':
        c = fileGetc ();
        if (c == '>')
            token->type = TOKEN_CONSTRAINT;
        else if (c == '-')  /* is this the start of a comment? */
        {
            skipToCharacter ('\n');
            goto getNextChar;
        }
        else
        {
            if (!isspace (c))
                fileUngetc (c);
            token->type = TOKEN_OPERATOR;
        }
        break;

    case '?':
    case ':':
        c = fileGetc ();
        if (c == '=')
            token->type = TOKEN_OPERATOR;
        else
        {
            token->type = TOKEN_COLON;
            if (!isspace (c))
                fileUngetc (c);
        }
        break;

    case '<':
        c = fileGetc ();
        if (c != '='  &&  c != '>'  &&  !isspace (c))
            fileUngetc (c);
        token->type = TOKEN_OPERATOR;
        break;

    case '>':
        c = fileGetc ();
        if (c != '='  &&  c != '>'  &&  !isspace (c))
            fileUngetc (c);
        token->type = TOKEN_OPERATOR;
        break;

    case '/':
        c = fileGetc ();
        if (c != '/'  &&  c != '='  &&  !isspace (c))
            fileUngetc (c);
        token->type = TOKEN_OPERATOR;
        break;

    case '\\':
        c = fileGetc ();
        if (c != '\\'  &&  !isspace (c))
            fileUngetc (c);
        token->type = TOKEN_OPERATOR;
        break;

    case '"':
        token->type = TOKEN_STRING;
        parseString (token->string);
        break;

    case '\'':
        token->type = TOKEN_CHARACTER;
        parseCharacter ();
        break;

    default:
        if (isalpha (c))
        {
            parseIdentifier (token->string, c);
            token->keyword = analyzeToken (token->string);
            if (isKeyword (token, KEYWORD_NONE))
                token->type = TOKEN_IDENTIFIER;
            else
                token->type = TOKEN_KEYWORD;
        }
        else if (isdigit (c))
        {
            vStringCat (token->string, parseNumeric (c));
            token->type = TOKEN_NUMERIC;
        }
        else if (isFreeOperatorChar (c))
        {
            parseFreeOperator (token->string, c);
            token->type = TOKEN_OPERATOR;
        }
        else
        {
            token->type = TOKEN_UNDEFINED;
            Assert (! isType (token, TOKEN_UNDEFINED));
        }
        break;
    }
}
Beispiel #8
0
static void readTokenFull (tokenInfo *const token, boolean include_newlines, vString *const repr)
{
	int c;
	int i;
	boolean newline_encountered = FALSE;

	/* if we've got a token held back, emit it */
	if (NextToken)
	{
		copyToken (token, NextToken, FALSE);
		deleteToken (NextToken);
		NextToken = NULL;
		return;
	}

	token->type			= TOKEN_UNDEFINED;
	token->keyword		= KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:
	i = 0;
	do
	{
		c = fileGetc ();
		if (include_newlines && (c == '\r' || c == '\n'))
			newline_encountered = TRUE;
		i++;
	}
	while (c == '\t' || c == ' ' || c == '\r' || c == '\n');

	token->lineNumber   = getSourceLineNumber ();
	token->filePosition = getInputFilePosition ();

	if (repr)
	{
		if (i > 1)
			vStringPut (repr, ' ');
		vStringPut (repr, c);
	}

	switch (c)
	{
		case EOF: token->type = TOKEN_EOF;					break;
		case '(': token->type = TOKEN_OPEN_PAREN;			break;
		case ')': token->type = TOKEN_CLOSE_PAREN;			break;
		case ';': token->type = TOKEN_SEMICOLON;			break;
		case ',': token->type = TOKEN_COMMA;				break;
		case '.': token->type = TOKEN_PERIOD;				break;
		case ':': token->type = TOKEN_COLON;				break;
		case '{': token->type = TOKEN_OPEN_CURLY;			break;
		case '}': token->type = TOKEN_CLOSE_CURLY;			break;
		case '=': token->type = TOKEN_EQUAL_SIGN;			break;
		case '[': token->type = TOKEN_OPEN_SQUARE;			break;
		case ']': token->type = TOKEN_CLOSE_SQUARE;			break;

		case '+':
		case '-':
			{
				int d = fileGetc ();
				if (d == c) /* ++ or -- */
					token->type = TOKEN_POSTFIX_OPERATOR;
				else
				{
					fileUngetc (d);
					token->type = TOKEN_BINARY_OPERATOR;
				}
				break;
			}

		case '*':
		case '%':
		case '?':
		case '>':
		case '<':
		case '^':
		case '|':
		case '&':
			token->type = TOKEN_BINARY_OPERATOR;
			break;

		case '\'':
		case '"':
				  token->type = TOKEN_STRING;
				  parseString (token->string, c);
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  if (repr)
				  {
					  vStringCat (repr, token->string);
					  vStringPut (repr, c);
				  }
				  break;

		case '`':
				  token->type = TOKEN_TEMPLATE_STRING;
				  parseTemplateString (token->string);
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  if (repr)
				  {
					  vStringCat (repr, token->string);
					  vStringPut (repr, c);
				  }
				  break;

		case '\\':
				  c = fileGetc ();
				  if (c != '\\'  && c != '"'  &&  !isspace (c))
					  fileUngetc (c);
				  token->type = TOKEN_CHARACTER;
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  break;

		case '/':
				  {
					  int d = fileGetc ();
					  if ( (d != '*') &&		/* is this the start of a comment? */
							  (d != '/') )		/* is a one line comment? */
					  {
						  fileUngetc (d);
						  switch (LastTokenType)
						  {
							  case TOKEN_CHARACTER:
							  case TOKEN_IDENTIFIER:
							  case TOKEN_STRING:
							  case TOKEN_TEMPLATE_STRING:
							  case TOKEN_CLOSE_CURLY:
							  case TOKEN_CLOSE_PAREN:
							  case TOKEN_CLOSE_SQUARE:
								  token->type = TOKEN_BINARY_OPERATOR;
								  break;

							  default:
								  token->type = TOKEN_REGEXP;
								  parseRegExp ();
								  token->lineNumber = getSourceLineNumber ();
								  token->filePosition = getInputFilePosition ();
								  break;
						  }
					  }
					  else
					  {
						  if (repr) /* remove the / we added */
							  repr->buffer[--repr->length] = 0;
						  if (d == '*')
						  {
							  do
							  {
								  fileSkipToCharacter ('*');
								  c = fileGetc ();
								  if (c == '/')
									  break;
								  else
									  fileUngetc (c);
							  } while (c != EOF && c != '\0');
							  goto getNextChar;
						  }
						  else if (d == '/')	/* is this the start of a comment?  */
						  {
							  fileSkipToCharacter ('\n');
							  /* if we care about newlines, put it back so it is seen */
							  if (include_newlines)
								  fileUngetc ('\n');
							  goto getNextChar;
						  }
					  }
					  break;
				  }

		case '#':
				  /* skip shebang in case of e.g. Node.js scripts */
				  if (token->lineNumber > 1)
					  token->type = TOKEN_UNDEFINED;
				  else if ((c = fileGetc ()) != '!')
				  {
					  fileUngetc (c);
					  token->type = TOKEN_UNDEFINED;
				  }
				  else
				  {
					  fileSkipToCharacter ('\n');
					  goto getNextChar;
				  }
				  break;

		default:
				  if (! isIdentChar (c))
					  token->type = TOKEN_UNDEFINED;
				  else
				  {
					  parseIdentifier (token->string, c);
					  token->lineNumber = getSourceLineNumber ();
					  token->filePosition = getInputFilePosition ();
					  token->keyword = analyzeToken (token->string, Lang_js);
					  if (isKeyword (token, KEYWORD_NONE))
						  token->type = TOKEN_IDENTIFIER;
					  else
						  token->type = TOKEN_KEYWORD;
					  if (repr && vStringLength (token->string) > 1)
						  vStringCatS (repr, vStringValue (token->string) + 1);
				  }
				  break;
	}

	if (include_newlines && newline_encountered)
	{
		/* This isn't strictly correct per the standard, but following the
		 * real rules means understanding all statements, and that's not
		 * what the parser currently does.  What we do here is a guess, by
		 * avoiding inserting semicolons that would make the statement on
		 * the left or right obviously invalid.  Hopefully this should not
		 * have false negatives (e.g. should not miss insertion of a semicolon)
		 * but might have false positives (e.g. it will wrongfully emit a
		 * semicolon sometimes, i.e. for the newline in "foo\n(bar)").
		 * This should however be mostly harmless as we only deal with
		 * newlines in specific situations where we know a false positive
		 * wouldn't hurt too bad. */

		/* these already end a statement, so no need to duplicate it */
		#define IS_STMT_SEPARATOR(t) ((t) == TOKEN_SEMICOLON    || \
		                              (t) == TOKEN_EOF          || \
		                              (t) == TOKEN_COMMA        || \
		                              (t) == TOKEN_CLOSE_CURLY  || \
		                              (t) == TOKEN_OPEN_CURLY)
		/* these cannot be the start or end of a statement */
		#define IS_BINARY_OPERATOR(t) ((t) == TOKEN_EQUAL_SIGN      || \
		                               (t) == TOKEN_COLON           || \
		                               (t) == TOKEN_PERIOD          || \
		                               (t) == TOKEN_BINARY_OPERATOR)

		if (! IS_STMT_SEPARATOR(LastTokenType) &&
		    ! IS_STMT_SEPARATOR(token->type) &&
		    ! IS_BINARY_OPERATOR(LastTokenType) &&
		    ! IS_BINARY_OPERATOR(token->type) &&
		    /* these cannot be followed by a semicolon */
		    ! (LastTokenType == TOKEN_OPEN_PAREN ||
		       LastTokenType == TOKEN_OPEN_SQUARE))
		{
			/* hold the token... */
			Assert (NextToken == NULL);
			NextToken = newToken ();
			copyToken (NextToken, token, FALSE);

			/* ...and emit a semicolon instead */
			token->type		= TOKEN_SEMICOLON;
			token->keyword	= KEYWORD_NONE;
			vStringClear (token->string);
			if (repr)
				vStringPut (token->string, '\n');
		}

		#undef IS_STMT_SEPARATOR
		#undef IS_BINARY_OPERATOR
	}

	LastTokenType = token->type;
}
Beispiel #9
0
static void readToken (tokenInfo *const token)
{
    int c;

    token->type         = TOKEN_UNDEFINED;
    token->keyword      = KEYWORD_NONE;
    vStringClear (token->string);

getNextChar:
    do
	c = fileGetc ();
    while (c == '\t'  ||  c == ' '  ||  c == '\n');

    switch (c)
    {
	case EOF: longjmp (Exception, (int)ExceptionEOF);	break;
	case '(': token->type = TOKEN_OPEN_PAREN;		break;
	case ')': token->type = TOKEN_CLOSE_PAREN;		break;
	case ';': token->type = TOKEN_SEMICOLON;		break;
	case ',': token->type = TOKEN_COMMA;			break;

	case '\'':
	case '"':
	    token->type = TOKEN_STRING;
	    parseString (token->string, c);
	    break;

	case '-':
	    c = fileGetc ();
	    if (c == '-')		/* is this the start of a comment? */
	    {
		skipToCharacter ('\n');
		goto getNextChar;
	    }
	    else
	    {
		if (!isspace (c))
		    fileUngetc (c);
 		token->type = TOKEN_OPERATOR;
	    }
	    break;

	case '/':
	{
	    int d = fileGetc ();
	    if (d != '*')		/* is this the start of a comment? */
		fileUngetc (d);
	    else
	    {
		do
		{
		    skipToCharacter ('*');
		    c = fileGetc ();
		    if (c == '/')
			break;
		    else
			fileUngetc (c);
		} while (c != '\0');
		goto getNextChar;
	    }
	    break;
	}

	default:
	    if (! isIdentChar1 (c))
		token->type = TOKEN_UNDEFINED;
	    else
	    {
		parseIdentifier (token->string, c);
		token->keyword = analyzeToken (token->string);
		if (isKeyword (token, KEYWORD_rem))
		{
		    vStringClear (token->string);
		    skipToCharacter ('\n');
		    goto getNextChar;
		}
		else if (isKeyword (token, KEYWORD_NONE))
		    token->type = TOKEN_IDENTIFIER;
		else
		    token->type = TOKEN_KEYWORD;
	    }
	    break;
    }
}