Token Tokenizer::next_token() {
std::list<std::string>::iterator it;
size_t starting_pos = pos;
for (; pos < program.size(); ++pos) {
check_for_token(functions,FUNCTION);
check_for_token(keywords,KEYWORD);
check_for_token(operators,OPERATOR);
check_for_token(variables,VARIABLE);
}
return Token(EOF,"EOF");
}
void tokenize(std::istream &in, std::vector<token> &out) {
std::string line;
while(!in.fail() && !in.eof()) {
std::getline(in,line,'\n');
std::string word;
for (char c : line) {
if (isDelim(c)) {
if (word == "attributes") break;
if (word == "align") {word.clear();break;}
check_for_token(word,out);
check_for_delim(c,out);
} else {
if (c != '\n') word.push_back(c);
}
}
check_for_token(word,out);
out.push_back({ENDLINE,"\n"});
}
}
