//Returns the next token. Hint: consider the substr function //Updates the tokenizer state //Updates offset, resets tokenLength, updates processingABC member variables //Calls Tokenizer::prepareNextToken() as the last statement before returning. string Tokenizer::getNextToken() { string temp = str->substr(offset, tokenLength); //substr takes a string and outputs a string of a specified length starting at a point offset = offset + tokenLength; //move to the new position to seek forward tokenLength = 0; prepareNextToken(); return temp; }
//Sets the current string to be tokenized //Resets all Tokenizer state variables //Calls Tokenizer::prepareNextToken() as the last statement before returning. void Tokenizer::setString(string *str) { if (str->length() == 0) { //this is executed if a blank string is passed in complete = true; } //if the string has characters in it then you initialize some variables and prepare the next token for the get next token funtion else { complete = false; comment = false; offset = 0; tokenLength = 0; this->str = str; prepareNextToken(); } }
////////////////////////////////////////////////////////////////////////////////////// //Returns the next token. Hint: consider the substr function //Updates the tokenizer state //Updates offset, resets tokenLength, updates processingABC member variables //Calls Tokenizer::prepareNextToken() as the last statement before returning. ////////////////////////////////////////////////////////////////////////////////////// string Tokenizer::getNextToken() { string temp = str->substr(offset, tokenLength); //Use sub String function to returns a new string of length at point if (offset == str->length()) //At end of lsit { complete = true; //return complete return temp; // must return string **** } //Skips these character since they are case sensivtive if (temp == "'Z'" || temp == "'U'" || temp == "'X'" || temp == "'W'" || temp == "'L'" || temp == "'H'") { //NOTHING BOIIII!!! } else { for (size_t i = 0; i < tokenLength; i++) { temp[i] = char(tolower(temp[i])); } } offset = offset + tokenLength;//move to the new positio tokenLength = 0; //Reset length token //Check for comment '--' //A comment on a line every character after is a commet if (comment) { comment = false; } if (temp == "--") { comment = true; } prepareNextToken(); return temp; }
//Sets the current string to be tokenized //Resets all Tokenizer state variables //Calls Tokenizer::prepareNextToken() as the last statement before returning. void Tokenizer::setString(string *input_str) { if (input_str->length() == 0) //Empty string is detected { complete = true; //complete } else { //Reinitilze characters for next token complete = false;//Not complete offset = 0; tokenLength = 0; this -> str = input_str; // this points to private member str of tokenizer class prepareNextToken(); //calls as the last statement before returning for tokenize return; } return; }