Token Tokenizer::next_token() { std::list<std::string>::iterator it; size_t starting_pos = pos; for (; pos < program.size(); ++pos) { check_for_token(functions,FUNCTION); check_for_token(keywords,KEYWORD); check_for_token(operators,OPERATOR); check_for_token(variables,VARIABLE); } return Token(EOF,"EOF"); }
void tokenize(std::istream &in, std::vector<token> &out) { std::string line; while(!in.fail() && !in.eof()) { std::getline(in,line,'\n'); std::string word; for (char c : line) { if (isDelim(c)) { if (word == "attributes") break; if (word == "align") {word.clear();break;} check_for_token(word,out); check_for_delim(c,out); } else { if (c != '\n') word.push_back(c); } } check_for_token(word,out); out.push_back({ENDLINE,"\n"}); } }