Ejemplo n.º 1
0
//Returns the next token. Hint: consider the substr function
//Updates the tokenizer state
//Updates offset, resets tokenLength, updates processingABC member variables
//Calls Tokenizer::prepareNextToken() as the last statement before returning.
string Tokenizer::getNextToken() {
    string temp = str->substr(offset, tokenLength);
    //substr takes a string and outputs a string of a specified length starting at a point
    offset = offset + tokenLength;
    //move to the new position to seek forward
    tokenLength = 0;
    prepareNextToken();
    return temp;
}
Ejemplo n.º 2
0
//Sets the current string to be tokenized
//Resets all Tokenizer state variables
//Calls Tokenizer::prepareNextToken() as the last statement before returning.
void Tokenizer::setString(string *str) {
    if (str->length() == 0) {
        //this is executed if a blank string is passed in
        complete = true;
    }
    //if the string has characters in it then you initialize some variables and prepare the next token for the get next token funtion
    else {
        complete = false;
        comment = false;
        offset = 0;
        tokenLength = 0;
        this->str = str;
        prepareNextToken();
    }
}
Ejemplo n.º 3
0
//////////////////////////////////////////////////////////////////////////////////////
//Returns the next token. Hint: consider the substr function
//Updates the tokenizer state
//Updates offset, resets tokenLength, updates processingABC member variables
//Calls Tokenizer::prepareNextToken() as the last statement before returning.
//////////////////////////////////////////////////////////////////////////////////////
string Tokenizer::getNextToken()
{
	string temp = str->substr(offset, tokenLength);
	//Use sub String function to returns a new string of length at point

	if (offset == str->length()) //At end of lsit
	{
		complete = true; //return complete
		return temp; // must return string ****
	}
    //Skips these character since they are case sensivtive
	if (temp == "'Z'" || temp == "'U'" || temp == "'X'" ||
		temp == "'W'" || temp == "'L'" || temp == "'H'")
	{
		//NOTHING BOIIII!!!
	}
	else
	{
		for (size_t i = 0; i < tokenLength; i++)
		{
			temp[i] = char(tolower(temp[i]));
		}
	}

	offset = offset + tokenLength;//move to the new positio
	tokenLength = 0; //Reset length token

	//Check for comment '--'
	//A comment on a line every character after is a commet
	if (comment)
	{
		comment = false;
	}
	if (temp == "--")
	{
		comment = true;
	}
	prepareNextToken();
	return temp;
}
Ejemplo n.º 4
0
//Sets the current string to be tokenized
//Resets all Tokenizer state variables
//Calls Tokenizer::prepareNextToken() as the last statement before returning.
void Tokenizer::setString(string *input_str)
{
	if (input_str->length() == 0) //Empty string is detected
	{
		complete = true; //complete
	}
	else
	{
		//Reinitilze characters for next token
		complete = false;//Not complete

		offset = 0;
		tokenLength = 0;
		this -> str = input_str; // this points to private member str of tokenizer class

		prepareNextToken(); //calls as the last statement before returning for tokenize
		return;
	}


	return;
}