/** pushes a token with content */ void pushInitializedToken(Lexer *self, int type, char *content) { Token *tok = createToken(self->lineNumber, self->charNumber, self->fileName); tok->type = type; tok->content = content; verboseModeMessage("Lexed token: %-15s, type %s", tok->content, getTokenTypeName(tok->type)); pushBackItem(self->tokenStream, tok); }
/** pushes a token with no content */ void pushToken(Lexer *self, int type) { Token *tok = createToken(self->lineNumber, self->charNumber, self->fileName); tok->type = type; tok->content = extractToken(self, self->startPos, self->pos - self->startPos); verboseModeMessage("Lexed token: %-15s, type %s", tok->content, getTokenTypeName(tok->type)); pushBackItem(self->tokenStream, tok); }
/** pushes a token with content */ void pushInitializedToken(Lexer *self, int type, char *content) { int tokenLength = self->pos - self->startPos; // The length (in characters) of this token Token *tok = createToken(self->lineNumber, self->startPos, self->charNumber - tokenLength, self->charNumber, self->fileName); tok->type = type; tok->content = content; verboseModeMessage("Lexed token: %-15s, type %s", tok->content, getTokenTypeName(tok->type)); pushBackItem(self->tokenStream, tok); }