示例#1
0
文件: eiffel.c 项目: Figoer/i_figoer
static vString *parseNumeric (int c)
{
    static vString *string = NULL;

    if (string == NULL)
        string = vStringNew ();
    vStringCopy (string, parseInteger (c));

    c = fileGetc ();
    if (c == '.')
    {
        vStringPut (string, c);
        vStringCat (string, parseInteger ('\0'));
        c = fileGetc ();
    }
    if (tolower (c) == 'e')
    {
        vStringPut (string, c);
        vStringCat (string, parseInteger ('\0'));
    }
    else if (!isspace (c))
        fileUngetc (c);

    vStringTerminate (string);

    return string;
}
示例#2
0
文件: haskell.c 项目: max630/ctags
static int get_next_char()
{
    int c, nxt;
    c = fileGetc();
    if (c == EOF)
	return c;
    nxt = fileGetc();
    if (nxt == EOF)
	return c;
    fileUngetc(nxt);
    
    if (c == '-' && nxt == '-') {
	skip_rest_of_line();
	return get_next_char();
    }
    if (c == '{' && nxt == '-') {
	int last = '\0';
	do {
	    last = c;
	    c = get_next_char();
	} while (! (c == EOF || (last == '-' && c == '}')));
	return get_next_char();
    }
    return c;
}
示例#3
0
文件: js.c 项目: MihailZenkov/geany
static void parseRegExp (void)
{
	int c;
	boolean in_range = FALSE;

	do
	{
		c = fileGetc ();
		if (! in_range && c == '/')
		{
			do /* skip flags */
			{
				c = fileGetc ();
			} while (isalpha (c));
			fileUngetc (c);
			break;
		}
		else if (c == '\\')
			c = fileGetc (); /* skip next character */
		else if (c == '[')
			in_range = TRUE;
		else if (c == ']')
			in_range = FALSE;
	} while (c != EOF);
}
示例#4
0
文件: eiffel.c 项目: Figoer/i_figoer
/*  If a numeric is passed in 'c', this is used as the first digit of the
 *  numeric being parsed.
 */
static vString *parseInteger (int c)
{
    static vString *string = NULL;

    if (string == NULL)
        string = vStringNew ();
    vStringClear (string);

    if (c == '\0')
        c = fileGetc ();
    if (c == '-')
    {
        vStringPut (string, c);
        c = fileGetc ();
    }
    else if (! isdigit (c))
        c = fileGetc ();
    while (c != EOF  &&  (isdigit (c)  ||  c == '_'))
    {
        vStringPut (string, c);
        c = fileGetc ();
    }
    vStringTerminate (string);
    fileUngetc (c);

    return string;
}
示例#5
0
文件: make.c 项目: Monits/ctags
static void skipLine (void)
{
	int c;
	do
		c = nextChar ();
	while (c != EOF  &&  c != '\n');
	if (c == '\n')
		fileUngetc (c);
}
示例#6
0
文件: go.c 项目: 15ramky/geany
static void parseIdentifier (vString *const string, const int firstChar)
{
	int c = firstChar;
	do
	{
		vStringPut (string, c);
		c = fileGetc ();
	} while (isIdentChar (c));
	vStringTerminate (string);
	fileUngetc (c);		/* always unget, LF might add a semicolon */
}
示例#7
0
文件: css.c 项目: relaxdiego/ctags
static void parseSelector (vString *const string, const int firstChar)
{
	int c = firstChar;
	do
	{
		vStringPut (string, (char) c);
		c = fileGetc ();
	} while (isSelectorChar (c));
	fileUngetc (c);
	vStringTerminate (string);
}
示例#8
0
文件: js.c 项目: BenWiederhake/geany
static void parseString (vString *const string, const int delimiter)
{
	boolean end = FALSE;
	while (! end)
	{
		int c = fileGetc ();
		if (c == EOF)
			end = TRUE;
		else if (c == '\\')
		{
			/* Eat the escape sequence (\", \', etc).  We properly handle
			 * <LineContinuation> by eating a whole \<CR><LF> not to see <LF>
			 * as an unescaped character, which is invalid and handled below.
			 * Also, handle the fact that <LineContinuation> produces an empty
			 * sequence.
			 * See ECMA-262 7.8.4 */
			c = fileGetc();
			if (c != '\r' && c != '\n')
				vStringPut(string, c);
			else if (c == '\r')
			{
				c = fileGetc();
				if (c != '\n')
					fileUngetc (c);
			}
		}
		else if (c == delimiter)
			end = TRUE;
		else if (c == '\r' || c == '\n')
		{
			/* those are invalid when not escaped */
			end = TRUE;
			/* we don't want to eat the newline itself to let the automatic
			 * semicolon insertion code kick in */
			fileUngetc (c);
		}
		else
			vStringPut (string, c);
	}
	vStringTerminate (string);
}
示例#9
0
文件: js.c 项目: BenWiederhake/geany
/*	Read a C identifier beginning with "firstChar" and places it into
 *	"name".
 */
static void parseIdentifier (vString *const string, const int firstChar)
{
	int c = firstChar;
	Assert (isIdentChar (c));
	do
	{
		vStringPut (string, c);
		c = fileGetc ();
	} while (isIdentChar (c));
	vStringTerminate (string);
	fileUngetc (c);		/* unget non-identifier character */
}
示例#10
0
文件: make.c 项目: ajitvin/v
static void readIdentifier (const int first, vString *const id)
{
	int c = first;
	vStringClear (id);
	while (isIdentifier (c))
	{
		vStringPut (id, c);
		c = nextChar ();
	}
	fileUngetc (c);
	vStringTerminate (id);
}
示例#11
0
文件: eiffel.c 项目: Figoer/i_figoer
static void parseFreeOperator (vString *const string, const int firstChar)
{
    int c = firstChar;

    do
    {
        vStringPut (string, c);
        c = fileGetc ();
    } while (c > ' ');

    vStringTerminate (string);
    if (!isspace (c))
        fileUngetc (c);  /* unget non-identifier character */
}
示例#12
0
文件: make.c 项目: Monits/ctags
static void readIdentifier (const int first, vString *const id)
{
	int depth = 0;
	int c = first;
	vStringClear (id);
	while (isIdentifier (c) || (depth > 0 && c != EOF && c != '\n'))
	{
		if (c == '(' || c == '}')
			depth++;
		else if (depth > 0 && (c == ')' || c == '}'))
			depth--;
		vStringPut (id, c);
		c = nextChar ();
	}
	fileUngetc (c);
	vStringTerminate (id);
}
示例#13
0
文件: haskell.c 项目: max630/ctags
static int get_token(char *token, int n)
{
    int c = fileGetc();
    int i = n;
    while (c != EOF && isident(c) && i < 1000) {
	token[i] = c;
	i++;
	c = fileGetc();
    }
    if (c == EOF)
	return 0;
    if (i != n) {
	token[i] = '\0';
	fileUngetc(c);
	return 1;
    } else {
	return 0;
    }
}
示例#14
0
文件: jscript.c 项目: ruchee/ctags
static void parseTemplateString (vString *const string)
{
	int c;
	do
	{
		c = fileGetc ();
		if (c == '`')
			break;
		vStringPut (string, c);
		if (c == '\\')
		{
			c = fileGetc();
			vStringPut(string, c);
		}
		else if (c == '$')
		{
			c = fileGetc ();
			if (c != '{')
				fileUngetc (c);
			else
			{
				int depth = 1;
				/* we need to use the real token machinery to handle strings,
				 * comments, regexes and whatnot */
				tokenInfo *token = newToken ();
				LastTokenType = TOKEN_UNDEFINED;
				vStringPut(string, c);
				do
				{
					readTokenFull (token, FALSE, string);
					if (isType (token, TOKEN_OPEN_CURLY))
						depth++;
					else if (isType (token, TOKEN_CLOSE_CURLY))
						depth--;
				}
				while (! isType (token, TOKEN_EOF) && depth > 0);
				deleteToken (token);
			}
		}
	}
	while (c != EOF);
	vStringTerminate (string);
}
示例#15
0
文件: verilog.c 项目: Cognoscan/ctags
static int vGetc (void)
{
	int c;
	if (Ungetc == '\0')
		c = fileGetc ();
	else
	{
		c = Ungetc;
		Ungetc = '\0';
	}
	if (c == '/')
	{
		int c2 = fileGetc ();
		if (c2 == EOF)
			longjmp (Exception, (int) ExceptionEOF);
		else if (c2 == '/')  /* strip comment until end-of-line */
		{
			do
				c = fileGetc ();
			while (c != '\n'  &&  c != EOF);
		}
		else if (c2 == '*')  /* strip block comment */
		{
			c = skipOverCComment();
		}
		else
		{
			fileUngetc (c2);
		}
	}
	else if (c == '"')  /* strip string contents */
	{
		int c2;
		do
			c2 = fileGetc ();
		while (c2 != '"'  &&  c2 != EOF);
		c = '@';
	}
	if (c == EOF)
		longjmp (Exception, (int) ExceptionEOF);
	return c;
}
示例#16
0
文件: js.c 项目: MihailZenkov/geany
static void readToken (tokenInfo *const token)
{
	int c;

	token->type			= TOKEN_UNDEFINED;
	token->keyword		= KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:
	do
	{
		c = fileGetc ();
	}
	while (c == '\t'  ||  c == ' ' ||  c == '\n');

	token->lineNumber   = getSourceLineNumber ();
	token->filePosition = getInputFilePosition ();

	switch (c)
	{
		case EOF: longjmp (Exception, (int)ExceptionEOF);	break;
		case '(': token->type = TOKEN_OPEN_PAREN;			break;
		case ')': token->type = TOKEN_CLOSE_PAREN;			break;
		case ';': token->type = TOKEN_SEMICOLON;			break;
		case ',': token->type = TOKEN_COMMA;				break;
		case '.': token->type = TOKEN_PERIOD;				break;
		case ':': token->type = TOKEN_COLON;				break;
		case '{': token->type = TOKEN_OPEN_CURLY;			break;
		case '}': token->type = TOKEN_CLOSE_CURLY;			break;
		case '=': token->type = TOKEN_EQUAL_SIGN;			break;
		case '[': token->type = TOKEN_OPEN_SQUARE;			break;
		case ']': token->type = TOKEN_CLOSE_SQUARE;			break;

		case '\'':
		case '"':
				  token->type = TOKEN_STRING;
				  parseString (token->string, c);
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  break;

		case '\\':
				  c = fileGetc ();
				  if (c != '\\'  && c != '"'  &&  !isspace (c))
					  fileUngetc (c);
				  token->type = TOKEN_CHARACTER;
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  break;

		case '/':
				  {
					  int d = fileGetc ();
					  if ( (d != '*') &&		/* is this the start of a comment? */
							  (d != '/') )		/* is a one line comment? */
					  {
						  fileUngetc (d);
						  switch (LastTokenType)
						  {
							  case TOKEN_CHARACTER:
							  case TOKEN_KEYWORD:
							  case TOKEN_IDENTIFIER:
							  case TOKEN_STRING:
							  case TOKEN_CLOSE_CURLY:
							  case TOKEN_CLOSE_PAREN:
							  case TOKEN_CLOSE_SQUARE:
								  token->type = TOKEN_FORWARD_SLASH;
								  break;

							  default:
								  token->type = TOKEN_REGEXP;
								  parseRegExp ();
								  token->lineNumber = getSourceLineNumber ();
								  token->filePosition = getInputFilePosition ();
								  break;
						  }
					  }
					  else
					  {
						  if (d == '*')
						  {
							  do
							  {
								  skipToCharacter ('*');
								  c = fileGetc ();
								  if (c == '/')
									  break;
								  else
									  fileUngetc (c);
							  } while (c != EOF && c != '\0');
							  goto getNextChar;
						  }
						  else if (d == '/')	/* is this the start of a comment?  */
						  {
							  skipToCharacter ('\n');
							  goto getNextChar;
						  }
					  }
					  break;
				  }

		default:
				  if (! isIdentChar (c))
					  token->type = TOKEN_UNDEFINED;
				  else
				  {
					  parseIdentifier (token->string, c);
					  token->lineNumber = getSourceLineNumber ();
					  token->filePosition = getInputFilePosition ();
					  token->keyword = analyzeToken (token->string);
					  if (isKeyword (token, KEYWORD_NONE))
						  token->type = TOKEN_IDENTIFIER;
					  else
						  token->type = TOKEN_KEYWORD;
				  }
				  break;
	}

	LastTokenType = token->type;
}
示例#17
0
文件: json.c 项目: simlrh/ctags
static void readTokenFull (tokenInfo *const token,
                           boolean includeStringRepr)
{
    int c;

    token->type = TOKEN_UNDEFINED;
    vStringClear (token->string);

    do
        c = fileGetc ();
    while (c == '\t' || c == ' ' || c == '\r' || c == '\n');

    token->lineNumber   = getSourceLineNumber ();
    token->filePosition = getInputFilePosition ();

    switch (c)
    {
    case EOF:
        token->type = TOKEN_EOF;
        break;
    case '[':
        token->type = TOKEN_OPEN_SQUARE;
        break;
    case ']':
        token->type = TOKEN_CLOSE_SQUARE;
        break;
    case '{':
        token->type = TOKEN_OPEN_CURLY;
        break;
    case '}':
        token->type = TOKEN_CLOSE_CURLY;
        break;
    case ':':
        token->type = TOKEN_COLON;
        break;
    case ',':
        token->type = TOKEN_COMMA;
        break;

    case '"':
    {
        boolean escaped = FALSE;
        token->type = TOKEN_STRING;
        while (TRUE)
        {
            c = fileGetc ();
            /* we don't handle unicode escapes but they are safe */
            if (escaped)
                escaped = FALSE;
            else if (c == '\\')
                escaped = TRUE;
            else if (c >= 0x00 && c <= 0x1F)
                break; /* break on invalid, unescaped, control characters */
            else if (c == '"' || c == EOF)
                break;
            if (includeStringRepr)
                vStringPut (token->string, c);
        }
        vStringTerminate (token->string);
        break;
    }

    default:
        if (! isIdentChar (c))
            token->type = TOKEN_UNDEFINED;
        else
        {
            do
            {
                vStringPut (token->string, c);
                c = fileGetc ();
            }
            while (c != EOF && isIdentChar (c));
            vStringTerminate (token->string);
            fileUngetc (c);
            switch (lookupKeyword (vStringValue (token->string), Lang_json))
            {
            case KEYWORD_true:
                token->type = TOKEN_TRUE;
                break;
            case KEYWORD_false:
                token->type = TOKEN_FALSE;
                break;
            case KEYWORD_null:
                token->type = TOKEN_NULL;
                break;
            default:
                token->type = TOKEN_NUMBER;
                break;
            }
        }
        break;
    }
}
示例#18
0
文件: js.c 项目: BenWiederhake/geany
static void readTokenFull (tokenInfo *const token, boolean include_newlines, vString *const repr)
{
	int c;
	int i;

	token->type			= TOKEN_UNDEFINED;
	token->keyword		= KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:
	i = 0;
	do
	{
		c = fileGetc ();
		i++;
	}
	while (c == '\t'  ||  c == ' ' ||
		   ((c == '\r' || c == '\n') && ! include_newlines));

	token->lineNumber   = getSourceLineNumber ();
	token->filePosition = getInputFilePosition ();

	if (repr)
	{
		if (i > 1)
			vStringPut (repr, ' ');
		vStringPut (repr, c);
	}

	switch (c)
	{
		case EOF: token->type = TOKEN_EOF;					break;
		case '(': token->type = TOKEN_OPEN_PAREN;			break;
		case ')': token->type = TOKEN_CLOSE_PAREN;			break;
		case ';': token->type = TOKEN_SEMICOLON;			break;
		case ',': token->type = TOKEN_COMMA;				break;
		case '.': token->type = TOKEN_PERIOD;				break;
		case ':': token->type = TOKEN_COLON;				break;
		case '{': token->type = TOKEN_OPEN_CURLY;			break;
		case '}': token->type = TOKEN_CLOSE_CURLY;			break;
		case '=': token->type = TOKEN_EQUAL_SIGN;			break;
		case '[': token->type = TOKEN_OPEN_SQUARE;			break;
		case ']': token->type = TOKEN_CLOSE_SQUARE;			break;

		case '+':
		case '-':
			{
				int d = fileGetc ();
				if (d == c) /* ++ or -- */
					token->type = TOKEN_POSTFIX_OPERATOR;
				else
				{
					fileUngetc (d);
					token->type = TOKEN_BINARY_OPERATOR;
				}
				break;
			}

		case '*':
		case '%':
		case '?':
		case '>':
		case '<':
		case '^':
		case '|':
		case '&':
			token->type = TOKEN_BINARY_OPERATOR;
			break;

		case '\r':
		case '\n':
			/* This isn't strictly correct per the standard, but following the
			 * real rules means understanding all statements, and that's not
			 * what the parser currently does.  What we do here is a guess, by
			 * avoiding inserting semicolons that would make the statement on
			 * the left invalid.  Hopefully this should not have false negatives
			 * (e.g. should not miss insertion of a semicolon) but might have
			 * false positives (e.g. it will wrongfully emit a semicolon for the
			 * newline in "foo\n+bar").
			 * This should however be mostly harmless as we only deal with
			 * newlines in specific situations where we know a false positive
			 * wouldn't hurt too bad. */
			switch (LastTokenType)
			{
				/* these cannot be the end of a statement, so hold the newline */
				case TOKEN_EQUAL_SIGN:
				case TOKEN_COLON:
				case TOKEN_PERIOD:
				case TOKEN_FORWARD_SLASH:
				case TOKEN_BINARY_OPERATOR:
				/* and these already end one, no need to duplicate it */
				case TOKEN_SEMICOLON:
				case TOKEN_COMMA:
				case TOKEN_CLOSE_CURLY:
				case TOKEN_OPEN_CURLY:
					include_newlines = FALSE; /* no need to recheck */
					goto getNextChar;
					break;
				default:
					token->type = TOKEN_SEMICOLON;
			}
			break;

		case '\'':
		case '"':
				  token->type = TOKEN_STRING;
				  parseString (token->string, c);
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  if (repr)
				  {
					  vStringCat (repr, token->string);
					  vStringPut (repr, c);
				  }
				  break;

		case '\\':
				  c = fileGetc ();
				  if (c != '\\'  && c != '"'  &&  !isspace (c))
					  fileUngetc (c);
				  token->type = TOKEN_CHARACTER;
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  break;

		case '/':
				  {
					  int d = fileGetc ();
					  if ( (d != '*') &&		/* is this the start of a comment? */
							  (d != '/') )		/* is a one line comment? */
					  {
						  fileUngetc (d);
						  switch (LastTokenType)
						  {
							  case TOKEN_CHARACTER:
							  case TOKEN_IDENTIFIER:
							  case TOKEN_STRING:
							  case TOKEN_CLOSE_CURLY:
							  case TOKEN_CLOSE_PAREN:
							  case TOKEN_CLOSE_SQUARE:
								  token->type = TOKEN_FORWARD_SLASH;
								  break;

							  default:
								  token->type = TOKEN_REGEXP;
								  parseRegExp ();
								  token->lineNumber = getSourceLineNumber ();
								  token->filePosition = getInputFilePosition ();
								  break;
						  }
					  }
					  else
					  {
						  if (repr) /* remove the / we added */
							  repr->buffer[--repr->length] = 0;
						  if (d == '*')
						  {
							  do
							  {
								  skipToCharacter ('*');
								  c = fileGetc ();
								  if (c == '/')
									  break;
								  else
									  fileUngetc (c);
							  } while (c != EOF && c != '\0');
							  goto getNextChar;
						  }
						  else if (d == '/')	/* is this the start of a comment?  */
						  {
							  skipToCharacter ('\n');
							  /* if we care about newlines, put it back so it is seen */
							  if (include_newlines)
								  fileUngetc ('\n');
							  goto getNextChar;
						  }
					  }
					  break;
				  }

		case '#':
				  /* skip shebang in case of e.g. Node.js scripts */
				  if (token->lineNumber > 1)
					  token->type = TOKEN_UNDEFINED;
				  else if ((c = fileGetc ()) != '!')
				  {
					  fileUngetc (c);
					  token->type = TOKEN_UNDEFINED;
				  }
				  else
				  {
					  skipToCharacter ('\n');
					  goto getNextChar;
				  }
				  break;

		default:
				  if (! isIdentChar (c))
					  token->type = TOKEN_UNDEFINED;
				  else
				  {
					  parseIdentifier (token->string, c);
					  token->lineNumber = getSourceLineNumber ();
					  token->filePosition = getInputFilePosition ();
					  token->keyword = analyzeToken (token->string);
					  if (isKeyword (token, KEYWORD_NONE))
						  token->type = TOKEN_IDENTIFIER;
					  else
						  token->type = TOKEN_KEYWORD;
					  if (repr && vStringLength (token->string) > 1)
						  vStringCatS (repr, vStringValue (token->string) + 1);
				  }
				  break;
	}

	LastTokenType = token->type;
}
示例#19
0
文件: go.c 项目: Novator/geany
static void readToken (tokenInfo *const token)
{
	int c;
	static tokenType lastTokenType = TOKEN_NONE;

	token->type = TOKEN_NONE;
	token->keyword = KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:
	do
	{
		c = fileGetc ();
		token->lineNumber = getSourceLineNumber ();
		token->filePosition = getInputFilePosition ();
		if (c == '\n' && (lastTokenType == TOKEN_IDENTIFIER ||
						  lastTokenType == TOKEN_STRING ||
						  lastTokenType == TOKEN_OTHER ||
						  lastTokenType == TOKEN_CLOSE_PAREN ||
						  lastTokenType == TOKEN_CLOSE_CURLY ||
						  lastTokenType == TOKEN_CLOSE_SQUARE))
		{
			token->type = TOKEN_SEMICOLON;
			goto done;
		}
	}
	while (c == '\t'  ||  c == ' ' ||  c == '\r' || c == '\n');

	switch (c)
	{
		case EOF:
			token->type = TOKEN_EOF;
			break;

		case ';':
			token->type = TOKEN_SEMICOLON;
			break;

		case '/':
			{
				boolean hasNewline = FALSE;
				int d = fileGetc ();
				switch (d)
				{
					case '/':
						fileSkipToCharacter ('\n');
						/* Line comments start with the
						 * character sequence // and
						 * continue through the next
						 * newline. A line comment acts
						 * like a newline.  */
						fileUngetc ('\n');
						goto getNextChar;
					case '*':
						do
						{
							do
							{
								d = fileGetc ();
								if (d == '\n')
								{
									hasNewline = TRUE;
								}
							} while (d != EOF && d != '*');

							c = fileGetc ();
							if (c == '/')
								break;
							else
								fileUngetc (c);
						} while (c != EOF && c != '\0');

						fileUngetc (hasNewline ? '\n' : ' ');
						goto getNextChar;
					default:
						token->type = TOKEN_OTHER;
						fileUngetc (d);
						break;
				}
			}
			break;

		case '"':
		case '\'':
		case '`':
			token->type = TOKEN_STRING;
			parseString (token->string, c);
			token->lineNumber = getSourceLineNumber ();
			token->filePosition = getInputFilePosition ();
			break;

		case '<':
			{
				int d = fileGetc ();
				if (d == '-')
					token->type = TOKEN_LEFT_ARROW;
				else
				{
					fileUngetc (d);
					token->type = TOKEN_OTHER;
				}
			}
			break;

		case '(':
			token->type = TOKEN_OPEN_PAREN;
			break;

		case ')':
			token->type = TOKEN_CLOSE_PAREN;
			break;

		case '{':
			token->type = TOKEN_OPEN_CURLY;
			break;

		case '}':
			token->type = TOKEN_CLOSE_CURLY;
			break;

		case '[':
			token->type = TOKEN_OPEN_SQUARE;
			break;

		case ']':
			token->type = TOKEN_CLOSE_SQUARE;
			break;

		case '*':
			token->type = TOKEN_STAR;
			break;

		case '.':
			token->type = TOKEN_DOT;
			break;

		case ',':
			token->type = TOKEN_COMMA;
			break;

		default:
			if (isStartIdentChar (c))
			{
				parseIdentifier (token->string, c);
				token->lineNumber = getSourceLineNumber ();
				token->filePosition = getInputFilePosition ();
				token->keyword = lookupKeyword (vStringValue (token->string), Lang_go);
				if (isKeyword (token, KEYWORD_NONE))
					token->type = TOKEN_IDENTIFIER;
				else
					token->type = TOKEN_KEYWORD;
			}
			else
				token->type = TOKEN_OTHER;
			break;
	}

done:
	lastTokenType = token->type;
}
示例#20
0
文件: sql.c 项目: neil-yi/ffsource
static void readToken (tokenInfo *const token)
{
    int c;

    token->type         = TOKEN_UNDEFINED;
    token->keyword      = KEYWORD_NONE;
    vStringClear (token->string);

getNextChar:
    do
	c = fileGetc ();
    while (c == '\t'  ||  c == ' '  ||  c == '\n');

    switch (c)
    {
	case EOF: longjmp (Exception, (int)ExceptionEOF);	break;
	case '(': token->type = TOKEN_OPEN_PAREN;		break;
	case ')': token->type = TOKEN_CLOSE_PAREN;		break;
	case ';': token->type = TOKEN_SEMICOLON;		break;
	case ',': token->type = TOKEN_COMMA;			break;

	case '\'':
	case '"':
	    token->type = TOKEN_STRING;
	    parseString (token->string, c);
	    break;

	case '-':
	    c = fileGetc ();
	    if (c == '-')		/* is this the start of a comment? */
	    {
		skipToCharacter ('\n');
		goto getNextChar;
	    }
	    else
	    {
		if (!isspace (c))
		    fileUngetc (c);
 		token->type = TOKEN_OPERATOR;
	    }
	    break;

	case '/':
	{
	    int d = fileGetc ();
	    if (d != '*')		/* is this the start of a comment? */
		fileUngetc (d);
	    else
	    {
		do
		{
		    skipToCharacter ('*');
		    c = fileGetc ();
		    if (c == '/')
			break;
		    else
			fileUngetc (c);
		} while (c != '\0');
		goto getNextChar;
	    }
	    break;
	}

	default:
	    if (! isIdentChar1 (c))
		token->type = TOKEN_UNDEFINED;
	    else
	    {
		parseIdentifier (token->string, c);
		token->keyword = analyzeToken (token->string);
		if (isKeyword (token, KEYWORD_rem))
		{
		    vStringClear (token->string);
		    skipToCharacter ('\n');
		    goto getNextChar;
		}
		else if (isKeyword (token, KEYWORD_NONE))
		    token->type = TOKEN_IDENTIFIER;
		else
		    token->type = TOKEN_KEYWORD;
	    }
	    break;
    }
}
示例#21
0
文件: make.c 项目: ajitvin/v
static void findMakeTags (void)
{
	vString *name = vStringNew ();
	boolean newline = TRUE;
	boolean in_define = FALSE;
	boolean in_rule = FALSE;
	boolean variable_possible = TRUE;
	int c;

	while ((c = nextChar ()) != EOF)
	{
		if (newline)
		{
			if (in_rule)
			{
				if (c == '\t')
				{
					skipLine ();  /* skip rule */
					continue;
				}
				else
					in_rule = FALSE;
			}
			variable_possible = (boolean)(!in_rule);
			newline = FALSE;
		}
		if (c == '\n')
			newline = TRUE;
		else if (isspace (c))
			continue;
		else if (c == '#')
			skipLine ();
		else if (c == '(')
			skipToMatch ("()");
		else if (c == '{')
			skipToMatch ("{}");
		else if (c == ':')
		{
			variable_possible = TRUE;
			in_rule = TRUE;
		}
		else if (variable_possible && isIdentifier (c))
		{
			readIdentifier (c, name);
			if (strcmp (vStringValue (name), "endef") == 0)
				in_define = FALSE;
			else if (in_define)
				skipLine ();
			else if (strcmp (vStringValue (name), "define") == 0  &&
				isIdentifier (c))
			{
				in_define = TRUE;
				c = skipToNonWhite ();
				readIdentifier (c, name);
				makeSimpleTag (name, MakeKinds, K_MACRO);
				skipLine ();
			}
			else {
				c = skipToNonWhite ();
				if (strchr (":?+", c) != NULL)
				{
					boolean append = (boolean)(c == '+');
					if (c == ':')
						in_rule = TRUE;
					c = nextChar ();
					if (c != '=')
						fileUngetc (c);
					else if (append)
					{
						skipLine ();
						continue;
					}
				}
				if (c == '=')
				{
					makeSimpleTag (name, MakeKinds, K_MACRO);
					in_rule = FALSE;
					skipLine ();
				}
			}
		}
		else
			variable_possible = FALSE;
	}
	vStringDelete (name);
}
示例#22
0
文件: make.c 项目: Monits/ctags
static void findMakeTags (void)
{
	stringList *identifiers = stringListNew ();
	boolean newline = TRUE;
	boolean in_define = FALSE;
	boolean in_rule = FALSE;
	boolean variable_possible = TRUE;
	int c;

	while ((c = nextChar ()) != EOF)
	{
		if (newline)
		{
			if (in_rule)
			{
				if (c == '\t' || (c = skipToNonWhite (c)) == '#')
				{
					skipLine ();  /* skip rule or comment */
					c = nextChar ();
				}
				else if (c != '\n')
					in_rule = FALSE;
			}
			stringListClear (identifiers);
			variable_possible = (boolean)(!in_rule);
			newline = FALSE;
		}
		if (c == '\n')
			newline = TRUE;
		else if (isspace (c))
			continue;
		else if (c == '#')
			skipLine ();
		else if (variable_possible && c == '?')
		{
			c = nextChar ();
			fileUngetc (c);
			variable_possible = (c == '=');
		}
		else if (variable_possible && c == ':' &&
				 stringListCount (identifiers) > 0)
		{
			c = nextChar ();
			fileUngetc (c);
			if (c != '=')
			{
				unsigned int i;
				for (i = 0; i < stringListCount (identifiers); i++)
					newTarget (stringListItem (identifiers, i));
				stringListClear (identifiers);
				in_rule = TRUE;
			}
		}
		else if (variable_possible && c == '=' &&
				 stringListCount (identifiers) == 1)
		{
			newMacro (stringListItem (identifiers, 0));
			skipLine ();
			in_rule = FALSE;
		}
		else if (variable_possible && isIdentifier (c))
		{
			vString *name = vStringNew ();
			readIdentifier (c, name);
			stringListAdd (identifiers, name);

			if (stringListCount (identifiers) == 1)
			{
				if (in_define && ! strcmp (vStringValue (name), "endef"))
					in_define = FALSE;
				else if (in_define)
					skipLine ();
				else if (! strcmp (vStringValue (name), "define"))
				{
					in_define = TRUE;
					c = skipToNonWhite (nextChar ());
					vStringClear (name);
					/* all remaining characters on the line are the name -- even spaces */
					while (c != EOF && c != '\n')
					{
						vStringPut (name, c);
						c = nextChar ();
					}
					if (c == '\n')
						fileUngetc (c);
					vStringTerminate (name);
					vStringStripTrailing (name);
					newMacro (name);
				}
				else if (! strcmp (vStringValue (name), "export"))
					stringListClear (identifiers);
			}
		}
		else
			variable_possible = FALSE;
	}
	stringListDelete (identifiers);
}
示例#23
0
文件: eiffel.c 项目: Figoer/i_figoer
static void readToken (tokenInfo *const token)
{
    int c;

    token->type    = TOKEN_UNDEFINED;
    token->keyword = KEYWORD_NONE;
    vStringClear (token->string);

getNextChar:

    do
        c = fileGetc ();
    while (c == '\t'  ||  c == ' '  ||  c == '\n');

    switch (c)
    {
    case EOF:
        longjmp (Exception, (int)ExceptionEOF);
        break;
    case '!':
        token->type = TOKEN_BANG;
        break;
    case '$':
        token->type = TOKEN_DOLLAR;
        break;
    case '(':
        token->type = TOKEN_OPEN_PAREN;
        break;
    case ')':
        token->type = TOKEN_CLOSE_PAREN;
        break;
    case ',':
        token->type = TOKEN_COMMA;
        break;
    case '.':
        token->type = TOKEN_DOT;
        break;
    case ';':
        goto getNextChar;
    case '[':
        token->type = TOKEN_OPEN_BRACKET;
        break;
    case ']':
        token->type = TOKEN_CLOSE_BRACKET;
        break;
    case '{':
        token->type = TOKEN_OPEN_BRACE;
        break;
    case '}':
        token->type = TOKEN_CLOSE_BRACE;
        break;
    case '~':
        token->type = TOKEN_TILDE;
        break;


    case '+':
    case '*':
    case '^':
    case '=':
        token->type = TOKEN_OPERATOR;
        break;

    case '-':
        c = fileGetc ();
        if (c == '>')
            token->type = TOKEN_CONSTRAINT;
        else if (c == '-')  /* is this the start of a comment? */
        {
            skipToCharacter ('\n');
            goto getNextChar;
        }
        else
        {
            if (!isspace (c))
                fileUngetc (c);
            token->type = TOKEN_OPERATOR;
        }
        break;

    case '?':
    case ':':
        c = fileGetc ();
        if (c == '=')
            token->type = TOKEN_OPERATOR;
        else
        {
            token->type = TOKEN_COLON;
            if (!isspace (c))
                fileUngetc (c);
        }
        break;

    case '<':
        c = fileGetc ();
        if (c != '='  &&  c != '>'  &&  !isspace (c))
            fileUngetc (c);
        token->type = TOKEN_OPERATOR;
        break;

    case '>':
        c = fileGetc ();
        if (c != '='  &&  c != '>'  &&  !isspace (c))
            fileUngetc (c);
        token->type = TOKEN_OPERATOR;
        break;

    case '/':
        c = fileGetc ();
        if (c != '/'  &&  c != '='  &&  !isspace (c))
            fileUngetc (c);
        token->type = TOKEN_OPERATOR;
        break;

    case '\\':
        c = fileGetc ();
        if (c != '\\'  &&  !isspace (c))
            fileUngetc (c);
        token->type = TOKEN_OPERATOR;
        break;

    case '"':
        token->type = TOKEN_STRING;
        parseString (token->string);
        break;

    case '\'':
        token->type = TOKEN_CHARACTER;
        parseCharacter ();
        break;

    default:
        if (isalpha (c))
        {
            parseIdentifier (token->string, c);
            token->keyword = analyzeToken (token->string);
            if (isKeyword (token, KEYWORD_NONE))
                token->type = TOKEN_IDENTIFIER;
            else
                token->type = TOKEN_KEYWORD;
        }
        else if (isdigit (c))
        {
            vStringCat (token->string, parseNumeric (c));
            token->type = TOKEN_NUMERIC;
        }
        else if (isFreeOperatorChar (c))
        {
            parseFreeOperator (token->string, c);
            token->type = TOKEN_OPERATOR;
        }
        else
        {
            token->type = TOKEN_UNDEFINED;
            Assert (! isType (token, TOKEN_UNDEFINED));
        }
        break;
    }
}
示例#24
0
文件: jscript.c 项目: ruchee/ctags
static void readTokenFull (tokenInfo *const token, boolean include_newlines, vString *const repr)
{
	int c;
	int i;
	boolean newline_encountered = FALSE;

	/* if we've got a token held back, emit it */
	if (NextToken)
	{
		copyToken (token, NextToken, FALSE);
		deleteToken (NextToken);
		NextToken = NULL;
		return;
	}

	token->type			= TOKEN_UNDEFINED;
	token->keyword		= KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:
	i = 0;
	do
	{
		c = fileGetc ();
		if (include_newlines && (c == '\r' || c == '\n'))
			newline_encountered = TRUE;
		i++;
	}
	while (c == '\t' || c == ' ' || c == '\r' || c == '\n');

	token->lineNumber   = getSourceLineNumber ();
	token->filePosition = getInputFilePosition ();

	if (repr)
	{
		if (i > 1)
			vStringPut (repr, ' ');
		vStringPut (repr, c);
	}

	switch (c)
	{
		case EOF: token->type = TOKEN_EOF;					break;
		case '(': token->type = TOKEN_OPEN_PAREN;			break;
		case ')': token->type = TOKEN_CLOSE_PAREN;			break;
		case ';': token->type = TOKEN_SEMICOLON;			break;
		case ',': token->type = TOKEN_COMMA;				break;
		case '.': token->type = TOKEN_PERIOD;				break;
		case ':': token->type = TOKEN_COLON;				break;
		case '{': token->type = TOKEN_OPEN_CURLY;			break;
		case '}': token->type = TOKEN_CLOSE_CURLY;			break;
		case '=': token->type = TOKEN_EQUAL_SIGN;			break;
		case '[': token->type = TOKEN_OPEN_SQUARE;			break;
		case ']': token->type = TOKEN_CLOSE_SQUARE;			break;

		case '+':
		case '-':
			{
				int d = fileGetc ();
				if (d == c) /* ++ or -- */
					token->type = TOKEN_POSTFIX_OPERATOR;
				else
				{
					fileUngetc (d);
					token->type = TOKEN_BINARY_OPERATOR;
				}
				break;
			}

		case '*':
		case '%':
		case '?':
		case '>':
		case '<':
		case '^':
		case '|':
		case '&':
			token->type = TOKEN_BINARY_OPERATOR;
			break;

		case '\'':
		case '"':
				  token->type = TOKEN_STRING;
				  parseString (token->string, c);
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  if (repr)
				  {
					  vStringCat (repr, token->string);
					  vStringPut (repr, c);
				  }
				  break;

		case '`':
				  token->type = TOKEN_TEMPLATE_STRING;
				  parseTemplateString (token->string);
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  if (repr)
				  {
					  vStringCat (repr, token->string);
					  vStringPut (repr, c);
				  }
				  break;

		case '\\':
				  c = fileGetc ();
				  if (c != '\\'  && c != '"'  &&  !isspace (c))
					  fileUngetc (c);
				  token->type = TOKEN_CHARACTER;
				  token->lineNumber = getSourceLineNumber ();
				  token->filePosition = getInputFilePosition ();
				  break;

		case '/':
				  {
					  int d = fileGetc ();
					  if ( (d != '*') &&		/* is this the start of a comment? */
							  (d != '/') )		/* is a one line comment? */
					  {
						  fileUngetc (d);
						  switch (LastTokenType)
						  {
							  case TOKEN_CHARACTER:
							  case TOKEN_IDENTIFIER:
							  case TOKEN_STRING:
							  case TOKEN_TEMPLATE_STRING:
							  case TOKEN_CLOSE_CURLY:
							  case TOKEN_CLOSE_PAREN:
							  case TOKEN_CLOSE_SQUARE:
								  token->type = TOKEN_BINARY_OPERATOR;
								  break;

							  default:
								  token->type = TOKEN_REGEXP;
								  parseRegExp ();
								  token->lineNumber = getSourceLineNumber ();
								  token->filePosition = getInputFilePosition ();
								  break;
						  }
					  }
					  else
					  {
						  if (repr) /* remove the / we added */
							  repr->buffer[--repr->length] = 0;
						  if (d == '*')
						  {
							  do
							  {
								  fileSkipToCharacter ('*');
								  c = fileGetc ();
								  if (c == '/')
									  break;
								  else
									  fileUngetc (c);
							  } while (c != EOF && c != '\0');
							  goto getNextChar;
						  }
						  else if (d == '/')	/* is this the start of a comment?  */
						  {
							  fileSkipToCharacter ('\n');
							  /* if we care about newlines, put it back so it is seen */
							  if (include_newlines)
								  fileUngetc ('\n');
							  goto getNextChar;
						  }
					  }
					  break;
				  }

		case '#':
				  /* skip shebang in case of e.g. Node.js scripts */
				  if (token->lineNumber > 1)
					  token->type = TOKEN_UNDEFINED;
				  else if ((c = fileGetc ()) != '!')
				  {
					  fileUngetc (c);
					  token->type = TOKEN_UNDEFINED;
				  }
				  else
				  {
					  fileSkipToCharacter ('\n');
					  goto getNextChar;
				  }
				  break;

		default:
				  if (! isIdentChar (c))
					  token->type = TOKEN_UNDEFINED;
				  else
				  {
					  parseIdentifier (token->string, c);
					  token->lineNumber = getSourceLineNumber ();
					  token->filePosition = getInputFilePosition ();
					  token->keyword = analyzeToken (token->string, Lang_js);
					  if (isKeyword (token, KEYWORD_NONE))
						  token->type = TOKEN_IDENTIFIER;
					  else
						  token->type = TOKEN_KEYWORD;
					  if (repr && vStringLength (token->string) > 1)
						  vStringCatS (repr, vStringValue (token->string) + 1);
				  }
				  break;
	}

	if (include_newlines && newline_encountered)
	{
		/* This isn't strictly correct per the standard, but following the
		 * real rules means understanding all statements, and that's not
		 * what the parser currently does.  What we do here is a guess, by
		 * avoiding inserting semicolons that would make the statement on
		 * the left or right obviously invalid.  Hopefully this should not
		 * have false negatives (e.g. should not miss insertion of a semicolon)
		 * but might have false positives (e.g. it will wrongfully emit a
		 * semicolon sometimes, i.e. for the newline in "foo\n(bar)").
		 * This should however be mostly harmless as we only deal with
		 * newlines in specific situations where we know a false positive
		 * wouldn't hurt too bad. */

		/* these already end a statement, so no need to duplicate it */
		#define IS_STMT_SEPARATOR(t) ((t) == TOKEN_SEMICOLON    || \
		                              (t) == TOKEN_EOF          || \
		                              (t) == TOKEN_COMMA        || \
		                              (t) == TOKEN_CLOSE_CURLY  || \
		                              (t) == TOKEN_OPEN_CURLY)
		/* these cannot be the start or end of a statement */
		#define IS_BINARY_OPERATOR(t) ((t) == TOKEN_EQUAL_SIGN      || \
		                               (t) == TOKEN_COLON           || \
		                               (t) == TOKEN_PERIOD          || \
		                               (t) == TOKEN_BINARY_OPERATOR)

		if (! IS_STMT_SEPARATOR(LastTokenType) &&
		    ! IS_STMT_SEPARATOR(token->type) &&
		    ! IS_BINARY_OPERATOR(LastTokenType) &&
		    ! IS_BINARY_OPERATOR(token->type) &&
		    /* these cannot be followed by a semicolon */
		    ! (LastTokenType == TOKEN_OPEN_PAREN ||
		       LastTokenType == TOKEN_OPEN_SQUARE))
		{
			/* hold the token... */
			Assert (NextToken == NULL);
			NextToken = newToken ();
			copyToken (NextToken, token, FALSE);

			/* ...and emit a semicolon instead */
			token->type		= TOKEN_SEMICOLON;
			token->keyword	= KEYWORD_NONE;
			vStringClear (token->string);
			if (repr)
				vStringPut (token->string, '\n');
		}

		#undef IS_STMT_SEPARATOR
		#undef IS_BINARY_OPERATOR
	}

	LastTokenType = token->type;
}
示例#25
0
static void readToken (tokenInfo *const token)
{
    int c;

    token->type         = TOKEN_UNDEFINED;
    token->keyword      = KEYWORD_NONE;
    vStringClear (token->string);

getNextChar:
    do
    {
        c = fileGetc ();
        token->lineNumber   = getSourceLineNumber ();
        token->filePosition = getInputFilePosition ();
    }
    while (c == '\t'  ||  c == ' ' ||  c == '\n');

    switch (c)
    {
        case EOF: longjmp (Exception, (int)ExceptionEOF);   break;
        case '(': token->type = TOKEN_OPEN_PAREN;           break;
        case ')': token->type = TOKEN_CLOSE_PAREN;          break;
        case ';': token->type = TOKEN_SEMICOLON;            break;
        case ',': token->type = TOKEN_COMMA;                break;
        case '.': token->type = TOKEN_PERIOD;               break;
        case ':': token->type = TOKEN_COLON;                break;
        case '{': token->type = TOKEN_OPEN_CURLY;           break;
        case '}': token->type = TOKEN_CLOSE_CURLY;          break;
        case '=': token->type = TOKEN_EQUAL_SIGN;           break;
        case '[': token->type = TOKEN_OPEN_SQUARE;          break;
        case ']': token->type = TOKEN_CLOSE_SQUARE;         break;
        case '?': token->type = TOKEN_QUESTION_MARK;        break;
        case '*': token->type = TOKEN_STAR;                 break;

        case '\'':
        case '"':
                  token->type = TOKEN_STRING;
                  parseString (token->string, c);
                  token->lineNumber = getSourceLineNumber ();
                  token->filePosition = getInputFilePosition ();
                  break;

        case '\\':
                  /*
                   * All Tex tags start with a backslash.
                   * Check if the next character is an alpha character
                   * else it is not a potential tex tag.
                   */
                  c = fileGetc ();
                  if (! isalpha (c))
                      fileUngetc (c);
                  else
                  {
                      parseIdentifier (token->string, c);
                      token->lineNumber = getSourceLineNumber ();
                      token->filePosition = getInputFilePosition ();
                      token->keyword = analyzeToken (token->string, Lang_js);
                      if (isKeyword (token, KEYWORD_NONE))
                          token->type = TOKEN_IDENTIFIER;
                      else
                          token->type = TOKEN_KEYWORD;
                  }
                  break;

        case '%':
                  fileSkipToCharacter ('\n'); /* % are single line comments */
                  goto getNextChar;
                  break;

        default:
                  if (! isIdentChar (c))
                      token->type = TOKEN_UNDEFINED;
                  else
                  {
                      parseIdentifier (token->string, c);
                      token->lineNumber = getSourceLineNumber ();
                      token->filePosition = getInputFilePosition ();
                      token->type = TOKEN_IDENTIFIER;
                  }
                  break;
    }
}
示例#26
0
文件: css.c 项目: relaxdiego/ctags
static void readToken (tokenInfo *const token)
{
	int c;

	vStringClear (token->string);

getNextChar:

	c = fileGetc ();
	while (isspace (c))
		c = fileGetc ();

	token->type = c;
	switch (c)
	{
		case EOF: token->type = TOKEN_EOF; break;

		case '\'':
		case '"':
		{
			const int delimiter = c;
			do
			{
				vStringPut (token->string, c);
				c = fileGetc ();
				if (c == '\\')
					c = fileGetc ();
			}
			while (c != EOF && c != delimiter);
			if (c != EOF)
				vStringPut (token->string, c);
			token->type = TOKEN_STRING;
			break;
		}

		case '/': /* maybe comment start */
		{
			int d = fileGetc ();
			if (d != '*')
			{
				fileUngetc (d);
				vStringPut (token->string, c);
				token->type = c;
			}
			else
			{
				d = fileGetc ();
				do
				{
					c = d;
					d = fileGetc ();
				}
				while (d != EOF && ! (c == '*' && d == '/'));
				goto getNextChar;
			}
			break;
		}

		default:
			if (! isSelectorChar (c))
			{
				vStringPut (token->string, c);
				token->type = c;
			}
			else
			{
				parseSelector (token->string, c);
				token->type = TOKEN_SELECTOR;
			}
			break;
	}
}
示例#27
0
文件: vhdl.c 项目: relaxdiego/ctags
static void readToken (tokenInfo * const token)
{
	int c;

	token->type = TOKEN_NONE;
	token->keyword = KEYWORD_NONE;
	vStringClear (token->string);

  getNextChar:
	do
	{
		c = fileGetc ();
		token->lineNumber = getSourceLineNumber ();
		token->filePosition = getInputFilePosition ();
	}
	while (c == '\t' || c == ' ' || c == '\n');

	switch (c)
	{
	case EOF:
		token->type = TOKEN_EOF;
		break;
	case '(':
		token->type = TOKEN_OPEN_PAREN;
		break;
	case ')':
		token->type = TOKEN_CLOSE_PAREN;
		break;
	case ';':
		token->type = TOKEN_SEMICOLON;
		break;
	case '.':
		token->type = TOKEN_PERIOD;
		break;
	case ',':
		token->type = TOKEN_COMMA;
		break;
	case '\'':	/* only single char are inside simple quotes */
		break;	/* or it is for attributes so we don't care */
	case '"':
		token->type = TOKEN_STRING;
		parseString (token->string, c);
		token->lineNumber = getSourceLineNumber ();
		token->filePosition = getInputFilePosition ();
		break;
	case '-':
		c = fileGetc ();
		if (c == '-')	/* start of a comment */
		{
			fileSkipToCharacter ('\n');
			goto getNextChar;
		}
		else
		{
			if (!isspace (c))
				fileUngetc (c);
			token->type = TOKEN_OPERATOR;
		}
		break;
	default:
		if (!isIdentChar1 (c))
			token->type = TOKEN_NONE;
		else
		{
			parseIdentifier (token->string, c);
			token->lineNumber = getSourceLineNumber ();
			token->filePosition = getInputFilePosition ();
			token->keyword = analyzeToken (token->string, Lang_vhdl);
			if (isKeyword (token, KEYWORD_NONE))
				token->type = TOKEN_IDENTIFIER;
			else
				token->type = TOKEN_KEYWORD;
		}
		break;
	}
}
示例#28
0
文件: go.c 项目: 15ramky/geany
static void readToken (tokenInfo *const token)
{
	int c;
	static tokenType lastTokenType = TOKEN_NONE;
	boolean firstWhitespace = TRUE;
	boolean whitespace;

	token->type = TOKEN_NONE;
	token->keyword = KEYWORD_NONE;
	vStringClear (token->string);

getNextChar:
	do
	{
		c = fileGetc ();
		token->lineNumber = getSourceLineNumber ();
		token->filePosition = getInputFilePosition ();
		if (c == '\n' && (lastTokenType == TOKEN_IDENTIFIER ||
						  lastTokenType == TOKEN_STRING ||
						  lastTokenType == TOKEN_OTHER ||
						  lastTokenType == TOKEN_CLOSE_PAREN ||
						  lastTokenType == TOKEN_CLOSE_CURLY ||
						  lastTokenType == TOKEN_CLOSE_SQUARE))
		{
			c = ';';  // semicolon injection
		}
		whitespace = c == '\t'  ||  c == ' ' ||  c == '\r' || c == '\n';
		if (signature && whitespace && firstWhitespace && vStringLength (signature) < MAX_SIGNATURE_LENGTH)
		{
			firstWhitespace = FALSE;
			vStringPut(signature, ' ');
		}
	}
	while (whitespace);

	switch (c)
	{
		case EOF:
			token->type = TOKEN_EOF;
			break;

		case ';':
			token->type = TOKEN_SEMICOLON;
			break;

		case '/':
			{
				boolean hasNewline = FALSE;
				int d = fileGetc ();
				switch (d)
				{
					case '/':
						fileSkipToCharacter ('\n');
						/* Line comments start with the
						 * character sequence // and
						 * continue through the next
						 * newline. A line comment acts
						 * like a newline.  */
						fileUngetc ('\n');
						goto getNextChar;
					case '*':
						do
						{
							do
							{
								d = fileGetc ();
								if (d == '\n')
								{
									hasNewline = TRUE;
								}
							} while (d != EOF && d != '*');

							c = fileGetc ();
							if (c == '/')
								break;
							else
								fileUngetc (c);
						} while (c != EOF && c != '\0');

						fileUngetc (hasNewline ? '\n' : ' ');
						goto getNextChar;
					default:
						token->type = TOKEN_OTHER;
						fileUngetc (d);
						break;
				}
			}
			break;

		case '"':
		case '\'':
		case '`':
			token->type = TOKEN_STRING;
			parseString (token->string, c);
			token->lineNumber = getSourceLineNumber ();
			token->filePosition = getInputFilePosition ();
			break;

		case '<':
			{
				int d = fileGetc ();
				if (d == '-')
					token->type = TOKEN_LEFT_ARROW;
				else
				{
					fileUngetc (d);
					token->type = TOKEN_OTHER;
				}
			}
			break;

		case '(':
			token->type = TOKEN_OPEN_PAREN;
			break;

		case ')':
			token->type = TOKEN_CLOSE_PAREN;
			break;

		case '{':
			token->type = TOKEN_OPEN_CURLY;
			break;

		case '}':
			token->type = TOKEN_CLOSE_CURLY;
			break;

		case '[':
			token->type = TOKEN_OPEN_SQUARE;
			break;

		case ']':
			token->type = TOKEN_CLOSE_SQUARE;
			break;

		case '*':
			token->type = TOKEN_STAR;
			break;

		case '.':
			token->type = TOKEN_DOT;
			break;

		case ',':
			token->type = TOKEN_COMMA;
			break;

		default:
			if (isStartIdentChar (c))
			{
				parseIdentifier (token->string, c);
				token->lineNumber = getSourceLineNumber ();
				token->filePosition = getInputFilePosition ();
				token->keyword = lookupKeyword (vStringValue (token->string), Lang_go);
				if (isKeyword (token, KEYWORD_NONE))
					token->type = TOKEN_IDENTIFIER;
				else
					token->type = TOKEN_KEYWORD;
			}
			else
				token->type = TOKEN_OTHER;
			break;
	}

	if (signature && vStringLength (signature) < MAX_SIGNATURE_LENGTH)
	{
		if (token->type == TOKEN_LEFT_ARROW)
			vStringCatS(signature, "<-");
		else if (token->type == TOKEN_STRING)
		{
			// only struct member annotations can appear in function prototypes
			// so only `` type strings are possible
			vStringPut(signature, '`');
			vStringCat(signature, token->string);
			vStringPut(signature, '`');
		}
		else if (token->type == TOKEN_IDENTIFIER || token->type == TOKEN_KEYWORD)
			vStringCat(signature, token->string);
		else if (c != EOF)
			vStringPut(signature, c);
	}

	lastTokenType = token->type;
}