//Returns the next token. Hint: consider the substr function
//Updates the tokenizer state
//Updates offset, resets tokenLength, updates processingABC member variables
//Calls Tokenizer::prepareNextToken() as the last statement before returning.
string Tokenizer::getNextToken() {
string temp = str->substr(offset, tokenLength);
//substr takes a string and outputs a string of a specified length starting at a point
offset = offset + tokenLength;
//move to the new position to seek forward
tokenLength = 0;
prepareNextToken();
return temp;
}
//Sets the current string to be tokenized
//Resets all Tokenizer state variables
//Calls Tokenizer::prepareNextToken() as the last statement before returning.
void Tokenizer::setString(string *str) {
if (str->length() == 0) {
//this is executed if a blank string is passed in
complete = true;
}
//if the string has characters in it then you initialize some variables and prepare the next token for the get next token funtion
else {
complete = false;
comment = false;
offset = 0;
tokenLength = 0;
this->str = str;
prepareNextToken();
}
}
//////////////////////////////////////////////////////////////////////////////////////
//Returns the next token. Hint: consider the substr function
//Updates the tokenizer state
//Updates offset, resets tokenLength, updates processingABC member variables
//Calls Tokenizer::prepareNextToken() as the last statement before returning.
//////////////////////////////////////////////////////////////////////////////////////
string Tokenizer::getNextToken()
{
string temp = str->substr(offset, tokenLength);
//Use sub String function to returns a new string of length at point
if (offset == str->length()) //At end of lsit
{
complete = true; //return complete
return temp; // must return string ****
}
//Skips these character since they are case sensivtive
if (temp == "'Z'" || temp == "'U'" || temp == "'X'" ||
temp == "'W'" || temp == "'L'" || temp == "'H'")
{
//NOTHING BOIIII!!!
}
else
{
for (size_t i = 0; i < tokenLength; i++)
{
temp[i] = char(tolower(temp[i]));
}
}
offset = offset + tokenLength;//move to the new positio
tokenLength = 0; //Reset length token
//Check for comment '--'
//A comment on a line every character after is a commet
if (comment)
{
comment = false;
}
if (temp == "--")
{
comment = true;
}
prepareNextToken();
return temp;
}
//Sets the current string to be tokenized
//Resets all Tokenizer state variables
//Calls Tokenizer::prepareNextToken() as the last statement before returning.
void Tokenizer::setString(string *input_str)
{
if (input_str->length() == 0) //Empty string is detected
{
complete = true; //complete
}
else
{
//Reinitilze characters for next token
complete = false;//Not complete
offset = 0;
tokenLength = 0;
this -> str = input_str; // this points to private member str of tokenizer class
prepareNextToken(); //calls as the last statement before returning for tokenize
return;
}
return;
}