Token* readNumber(void) { int count = 0; Token* token = makeToken(TK_NUMBER, lineNo, colNo); while (charCodes[currentChar] == CHAR_DIGIT) { if (count > 9) { error(ERR_NUMBERTOOLONG, token->lineNo, token->colNo); } // Add current character to the number token->string[count] = currentChar; // Increase string index count++; // Read next character readChar(); } // End string token->string[count] = '\0'; // Convert current number to string token->value = atoi(token->string); return token; }
Token* readConstChar(void) { // TODO Token *token = makeToken(TK_CHAR, lineNo, colNo); readChar(); if (currentChar == EOF) { token->tokenType = TK_NONE; error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo); return token; } token->string[0] = currentChar; token->string[1] = '\0'; token->value = currentChar; readChar(); if (currentChar == EOF) { token->tokenType = TK_NONE; error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo); return token; } if (charCodes[currentChar] == CHAR_SINGLEQUOTE) { readChar(); return token; } else { token->tokenType = TK_NONE; error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo); return token; } }
Token* readIdentKeyword(void) { int count = 0; Token* token = makeToken(TK_IDENT, lineNo, colNo); while (charCodes[currentChar] == CHAR_LETTER || charCodes[currentChar] == CHAR_DIGIT) { // Add current character to identifier token->string[count] = currentChar; // Increase identifier length count++; // Get next character readChar(); } // End string token->string[count] = '\0'; // Limit identifier length if (count > MAX_IDENT_LEN) { // Announce error error(ERR_IDENTTOOLONG, lineNo, colNo - count); } else { // If this identifier is a keyword TokenType type = checkKeyword(token->string); // Otherwise if (type != TK_NONE) { token->tokenType = type; } } return token; }
Token* readIdentKeyword(void) { // TODO Token *token = makeToken(TK_NONE, lineNo, colNo); int count = 1; token->string[0] = toupper((char)currentChar); readChar(); while ((currentChar != EOF) && ((charCodes[currentChar] == CHAR_LETTER) || (charCodes[currentChar] == CHAR_DIGIT))){ if (count <= MAX_IDENT_LEN) token->string[count++] = toupper((char)currentChar); readChar(); } if (count > MAX_IDENT_LEN) { error(ERR_IDENTTOOLONG, token->lineNo, token->colNo); } token->string[count] = '\0'; token->tokenType = checkKeyword(token->string); if (token->tokenType == TK_NONE) { token->tokenType = TK_IDENT; } return token; }
void Scanner::handleEOFState() { loc_ = getTokenLocation(); makeToken(TokenType::END_OF_FILE, TokenValue::UNRESERVED, loc_, std::string("END_OF_FILE"), -1); // close the file input_.close(); }
TokenT *_octal(TokenizerT *tk) { nextChar(tk); if(isOctal(tk->inputIter[0])) { return _octal(tk); } else { return makeToken(tk, "octal integer"); } }
TokenT *_double_quote(TokenizerT *tk) { int atEndOfFile = nextChar(tk); while(tk->inputIter[0] != '"') { if(tk->inputIter[0] == '\\') { if(atEndOfFile) { return makeToken(tk, "unended string literal"); } atEndOfFile = nextChar(tk); } if(atEndOfFile) { return makeToken(tk, "unended string literal"); } atEndOfFile = nextChar(tk); } nextChar(tk); return makeToken(tk, "string literal"); }
static int makeQuotedToken(EcCompiler *cp, EcToken *tp, int c) { EcStream *stream; int quoteType; stream = cp->stream; quoteType = c; for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) { if (c == 0) { return makeToken(tp, 0, T_ERR, 0); } if (c == '\\') { c = getNextChar(stream); switch (c) { // TBD -- others case '\\': break; case '\'': case '\"': break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'u': c = decodeNumber(cp, 16, 4); break; case 'x': c = decodeNumber(cp, 16, 2); break; case 'v': c = '\v'; break; case '0': c = decodeNumber(cp, 8, 3); break; default: break; } } addCharToToken(tp, c); } assert(tp->text); setTokenID(tp, T_STRING, -1, 0); return finalizeToken(tp); }
Token* readConstChar(void) { Token*tmp; tmp=makeToken(TK_CHAR,lineNo,colNo); tmp->string[0]=(char)currentChar; tmp->string[1]='\0'; readChar(); return tmp; // TODO }
/* * C, C++ and doc style comments. Return token or zero for no token. */ static int getComment(EcInput *input, EcToken *tp, int c) { EcStream *stream; int form, startLine; startLine = tp->stream->lineNumber; stream = input->stream; form = c; // TODO - would be great to warn about comment nested in a comment. for (form = c; c > 0;) { c = getNextChar(stream); if (c <= 0) { /* * Unterminated Comment */ addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine); makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0); return 1; } if (form == '/') { if (c == '\n') { break; } } else { if (c == '*') { c = getNextChar(stream); if (c == '/') { break; } addCharToToken(tp, '*'); putBackChar(stream, c); } else if (c == '/') { c = getNextChar(stream); if (c == '*') { /* * Nested comment */ if (input->compiler->warnLevel > 0) { ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0, stream->column, "Possible nested comment"); } } addCharToToken(tp, '/'); } } addCharToToken(tp, c); } return 0; }
Token* readConstChar(void) { Token* token = makeToken(TK_CHAR, lineNo, colNo); // Read next character readChar(); if (currentChar == -1) { // End of File error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo); } else { switch(charCodes[currentChar]) { // Escape character for Single Quote: case CHAR_SINGLEQUOTE: // Read next character readChar(); if (charCodes[currentChar] == CHAR_SINGLEQUOTE) { token->string[0] = currentChar; readChar(); if (charCodes[currentChar] == CHAR_SINGLEQUOTE) { token->string[1] = '\0'; readChar(); return token; } else { error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo); } } else { error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo); } break; default: // Add the character to token string token->string[0] = currentChar; // Read next character readChar(); switch(charCodes[currentChar]) { case CHAR_SINGLEQUOTE: // End token token->string[1] = '\0'; readChar(); return token; default: error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo); break; } break; } } return token; }
void Scanner::handleStringState() { loc_ = getTokenLocation(); // eat ' and NOT update currentChar_ // because we don't want ' (single quote). getNextChar(); while (true) { if (currentChar_ == '\'') { // '''' condition // see pascal standard section 6.1.7 if (peekChar() == '\'') { getNextChar(); } // otherwise, we have handle string literal completely. else { break; } } addToBuffer(currentChar_); getNextChar(); } // eat end ' and update currentChar_ . getNextChar(); // just one char if (buffer_.length() == 1) { makeToken(TokenType::CHAR, TokenValue::UNRESERVED, loc_, static_cast<long>(buffer_.at(0)), buffer_); } else { makeToken(TokenType::STRING_LITERAL, TokenValue::UNRESERVED, loc_, buffer_, -1); } }
Token* readConstChar(void) { Token * token; int beginColNo = colNo; int beginLineNo = lineNo; char string[MAX_IDENT_LEN + 1]; readChar(); string[0] = currentChar; string[1] = '\0'; if (charCodes[currentChar] == CHAR_SINGLEQUOTE) { readChar(); if (charCodes[currentChar] != CHAR_SINGLEQUOTE) { token = makeToken(TK_NONE, lineNo, colNo); error(ERR_INVALIDCHARCONSTANT, beginColNo, beginLineNo); return token; } } readChar(); switch (charCodes[currentChar]) { case CHAR_SINGLEQUOTE: token = makeToken(TK_CHAR, beginLineNo, beginColNo); strcpy(token->string, string); readChar(); return token; default: if (CATCH_CONSTCHAR_ERR) { warning(ERR_INVALIDCHARCONSTANT, beginColNo, beginLineNo); token = makeToken(TK_CHAR, beginLineNo, beginColNo); strcpy(token->string, string); return token; } token = makeToken(TK_NONE, lineNo, colNo); error(ERR_INVALIDCHARCONSTANT, beginColNo, beginLineNo); return token; } }
TokenT *_hex(TokenizerT *tk, int isFirst) { nextChar(tk); if(isxdigit(tk->inputIter[0])) { return _hex(tk, 0); } else { if(isFirst) { return _invalid(tk); } else { return makeToken(tk, "hexadecimal integer"); } } }
TokenT *_decimal(TokenizerT *tk) { nextChar(tk); if(isdigit(tk->inputIter[0])) { return _decimal(tk); } else if(tk->inputIter[0] == '.') { return _float(tk, 1); } else if(tk->inputIter[0] == 'e' || tk->inputIter[0] == 'E') { return _expofloat(tk, 1, 0); } else { return makeToken(tk, "decimal integer"); } }
/* * Handle being given a zero as the first char in a new token. */ TokenT *_zero(TokenizerT *tk) { nextChar(tk); if(isOctal(tk->inputIter[0])) { return _octal(tk); } else if(tk->inputIter[0] == 'x' || (tk->inputIter[0]) == 'X') { return _hex(tk, 1); } else if(tk->inputIter[0] == '.') { return _float(tk, 1); } else { return makeToken(tk, "zero integer"); } }
PUBLIC int ecGetRegExpToken(EcCompiler *cp, wchar *prefix) { EcToken *token, *tp; EcStream *stream; wchar *pp; int c; stream = cp->stream; tp = token = cp->token; assert(tp != 0); initializeToken(tp, stream); for (pp = prefix; pp && *pp; pp++) { addCharToToken(tp, *pp); } while (1) { c = getNextChar(stream); switch (c) { case -1: return makeToken(tp, 0, T_ERR, 0); case 0: if (stream->flags & EC_STREAM_EOL) { return makeToken(tp, 0, T_NOP, 0); } return makeToken(tp, 0, T_EOF, 0); case '/': addCharToToken(tp, '/'); while (1) { c = getNextChar(stream); if (c != 'g' && c != 'i' && c != 'm' && c != 'y' && c != 'x' && c != 'X' && c != 'U' && c != 's') { putBackChar(stream, c); break; } addCharToToken(tp, c); } return makeToken(tp, 0, T_REGEXP, 0); case '\\': c = getNextChar(stream); if (c == '\r' || c == '\n' || c == 0) { ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression"); return makeToken(tp, 0, T_ERR, 0); } addCharToToken(tp, '\\'); addCharToToken(tp, c); break; case '\r': case '\n': ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression"); return makeToken(tp, 0, T_ERR, 0); default: addCharToToken(tp, c); } } }
void KnowledgeBaseUtils::addBreezeClause(KnowledgeBase &kb, size_t x, size_t y, size_t width, size_t height) { Clause clause; vector<Token> hazardTokens; // List all the cells that must have the hazard in them to imply the location of the wumpus if (x > 0) { hazardTokens.push_back(makeToken("breeze", x - 1, y)); } if (x < width - 1) { hazardTokens.push_back(makeToken("breeze", x + 1, y)); } if (y > 0) { hazardTokens.push_back(makeToken("breeze", x, y - 1)); } if (y < height - 1) { hazardTokens.push_back(makeToken("breeze", x, y + 1)); } if (hazardTokens.size() > 0) { clause << hazardTokens[0]; // Add the rest of the tokens with the AND token in reverse polish order for (size_t i = 1; i < hazardTokens.size(); i++) { clause << hazardTokens[i] << Token(AND); } // Add the implication part clause << makeToken("pit", x, y); clause << Token(IMPL); kb << clause; } }
TokenT *_expofloat(TokenizerT *tk, int isFirst, int lastWasSign) { nextChar(tk); if(isdigit(tk->inputIter[0])) { return _expofloat(tk, 0, 0); } else if(tk->inputIter[0] == '+' || tk->inputIter[0] == '-') { if(isFirst) { return _expofloat(tk, 0, 1); } else if(lastWasSign) { return _invalid(tk); } else { return makeToken(tk, "float with exponent"); } } else { if(isFirst) { return _invalid(tk); } else if(lastWasSign) { return _invalid(tk); } else { return makeToken(tk, "float with exponent"); } } }
void FMTLexer::mNONL(bool _createToken) { int _ttype; antlr::RefToken _token; std::string::size_type _begin = text.length(); _ttype = NONL; std::string::size_type _saveIndex; match('$' /* charlit */ ); if ( _createToken && _token==antlr::nullToken && _ttype!=antlr::Token::SKIP ) { _token = makeToken(_ttype); _token->setText(text.substr(_begin, text.length()-_begin)); } _returnToken = _token; _saveIndex=0; }
Token* readNumber(void) { Token *token = makeToken(TK_NUMBER, lineNo, colNo); int count = 0; while ((currentChar != EOF) && (charCodes[currentChar] == CHAR_DIGIT)) { token->string[count++] = (char)currentChar; readChar(); } token->string[count] = '\0'; token->value = atoi(token->string); return token; }
// TODO - handle triple quoting static int getQuotedToken(EcInput *input, EcToken *tp, int c) { EcStream *stream; int quoteType; stream = input->stream; quoteType = c; for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) { if (c == 0) { return makeToken(tp, 0, T_ERR, 0); } if (c == '\\') { c = getNextChar(stream); switch (c) { // TBD -- others case '\\': break; case '\'': case '\"': break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'u': case 'x': c = decodeNumber(input, 16, 4); break; case '0': c = decodeNumber(input, 8, 3); break; default: break; } } addCharToToken(tp, c); } return finishToken(tp, T_STRING, -1, 0); }
Tokens Bytecode::lexer(const std::string& source) const { Strings lines; split(source, '\n', lines); Strings tokens_str; for (int i = 0; i < lines.size(); i++) { split(lines[i], ' ', tokens_str); tokens_str.push_back("\n"); } Tokens result; for (int i = 0; i < tokens_str.size(); i++) { result.push_back(makeToken(tokens_str[i])); } return result; }
Token* readConstString(){ Token *token; token = makeToken(TK_STRING, lineNo, colNo); int i = 0; while(charCodes[readChar()] != CHAR_DOUBLEQUOTE){ token->string[i++] = currentChar; if(i == 16 || currentChar == '\n'){ error(ERR_INVALID_STRING, token->lineNo, token->colNo); } } token->string[i] = '\0'; readChar(); return token; }
temp<Token> Lexer::readNumber() { // TODO(bob): Handle EOF. while (isDigit(peek())) advance(); // Read the fractional part, if any. if (peek() == '.') { advance(); while (isDigit(peek())) advance(); } return makeToken(TOKEN_NUMBER); }
TokenT *_float(TokenizerT *tk, int isFirst) { nextChar(tk); if(isdigit(tk->inputIter[0])) { return _float(tk, 0); } else if(tk->inputIter[0] == 'e' || tk->inputIter[0] == 'E') { return _expofloat(tk, 1, 0); } else { if(isFirst) { return _invalid(tk); } else { return makeToken(tk, "float"); } } }
/* C, C++ and doc style comments. Return token or zero for no token. */ static int getComment(EcCompiler *cp, EcToken *tp, int c) { EcStream *stream; int form, startLine; startLine = cp->stream->loc.lineNumber; stream = cp->stream; form = c; for (form = c; c > 0;) { c = getNextChar(stream); if (c <= 0) { /* Unterminated Comment */ addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine); makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0); return 1; } if (form == '/') { if (c == '\n' || c == '\r') { break; } } else { if (c == '*') { c = getNextChar(stream); if (c == '/') { break; } addCharToToken(tp, '*'); putBackChar(stream, c); } else if (c == '/') { c = getNextChar(stream); if (c == '*') { /* Nested comment */ if (cp->warnLevel > 0) { ecError(cp, "Warning", &stream->loc, "Possible nested comment"); } } addCharToToken(tp, '/'); } } addCharToToken(tp, c); } return 0; }
Token* readNumber(void) { int i = 0; Token *token = makeToken(TK_NUMBER, lineNo, colNo); char *str = token->string; while (i < MAX_IDENT_LEN && charCodes[currentChar] == CHAR_DIGIT) { str[i++] = currentChar; readChar(); } str[i] = '\0'; while (charCodes[currentChar] == CHAR_DIGIT) readChar(); return token; // TODO }
void FMTLexer::mCHAR(bool _createToken) { int _ttype; antlr::RefToken _token; std::string::size_type _begin = text.length(); _ttype = CHAR; std::string::size_type _saveIndex; { matchRange('\3',static_cast<unsigned char>('\377')); } if ( _createToken && _token==antlr::nullToken && _ttype!=antlr::Token::SKIP ) { _token = makeToken(_ttype); _token->setText(text.substr(_begin, text.length()-_begin)); } _returnToken = _token; _saveIndex=0; }
Token *readFloat (void){ Token *token = makeToken(TK_NUMBER, lineNo, colNo); int count = 0; while ((currentChar != EOF) && ((charCodes[currentChar] == CHAR_DIGIT) || (charCodes[currentChar] == CHAR_PERIOD))) { if((charCodes[currentChar] == CHAR_PERIOD)&&(count==0)) token->string[count++]='0'; token->string[count++] = (char)currentChar; if((charCodes[currentChar]== CHAR_PERIOD)&& (count>0)) token->string[count++]='0'; readChar(); } token->string[count] = '\0'; token->value = atoi(token->string); return token; }