//Remove all block comments from the tokenList including /* and */ markers //Returns the number of block comments removed (where each pair of /* and */ count as one comment) int removeBlockComments(TokenList &tokenList) { //this funtion deletes the block comments in the tokenlist int count = 0; bool deleted = false; Token* temp, *destroy; temp = tokenList.getFirst(); while (temp) { //loop check if list is empty deleted = false; if (temp->getStringRep() == "/*") { //upon finding a block entry comment you keep deleating tokens till you find the exit token count++; while (!deleted) { destroy = temp; temp = temp->getNext(); tokenList.deleteToken(destroy); if (temp->getStringRep() == "*/") { //once the exit block token is found stop delete looping and continue searching through the list for block entry symbols destroy = temp; temp = temp->getNext(); deleted = true; tokenList.deleteToken(destroy); } } } else { temp = temp->getNext(); } } return count; }
unsigned int Bayes::GetTokenCount(const string& token, BayesTokenType type) { TokenList list = type == Spam ? spam : type == Ham ? ham : TokenList(); if(list.count(token) == 0) return 0; return list[token]; }
Expression ExpressionBuilder::Build(std::string expression) { TokenList tokens = m_tokenizer(expression); TokenList postfix = m_converter(tokens); std::stack<ExpressionNode*> nodeStk; auto it = postfix.begin(); nodeStk.push(m_nodeBuilder.NewExpressionNode(*it)); while (++it != postfix.end()) { ExpressionNode* node = m_nodeBuilder.NewExpressionNode(*it); switch (node->NumChildren()) { case 0: nodeStk.push(node); break; case 1: static_cast<OperatorNode*>(node)->SetChild(nodeStk.top()); nodeStk.pop(); nodeStk.push(node); break; case 2: static_cast<OperatorNode*>(node)->SetChild(nodeStk.top(), 1); nodeStk.pop(); static_cast<OperatorNode*>(node)->SetChild(nodeStk.top(), 0); nodeStk.pop(); nodeStk.push(node); break; } } return{ nodeStk.top(), m_nodeBuilder.GetVariableMap() }; }
Declaration* CssParser::parseDeclaration () { Declaration* declaration = NULL; TokenList property; if (!parseProperty(&property)) return NULL; skipWhitespace(); declaration = new Declaration(property.toString()); if (tokenizer->getTokenType() != Token::COLON) { throw new ParseException(tokenizer->getToken()->str, "colon following property(':')"); } tokenizer->readNextToken(); skipWhitespace(); TokenList* value = parseValue(); if (value == NULL) { throw new ParseException(tokenizer->getToken()->str, "value for property"); } declaration->setValue(value); return declaration; }
//Removes all comments from the tokenList including the -- marker //Returns the number of comments removed int removeComments(TokenList &tokenList) { int count = 0; Token* current = tokenList.getFirst();//initialized current to the first token while (current) //this loop run till end of the list { if (current->getStringRep() == "--") { count++; //count comment to remove Token* remove_first = current;//make a pointer to remove '--' if (current->getNext() != NULL) { Token* remove_second = current->getNext(); //check comment on the last line current = remove_second->getNext(); tokenList.deleteToken(remove_first); tokenList.deleteToken(remove_second); } else { delete current; } } else { current = current->getNext(); } } return count; //Returns the number of comments removed }
//remove token of given type int removeTokensOfType(TokenList &tokenList, tokenType type) { int count = 0; Token* current = tokenList.getFirst();//initialized current to the first token while (current) //this loop run till end of the list { if (current->getTokenType() == type) { count++; Token* remove_first = current; if (current->getNext() != NULL) { tokenList.deleteToken(remove_first); } else { delete current; } } else { current = current->getNext(); } } return count; }
////////////////////////////////////////////////////////////////////////// // // Class VMemory // void ETCompiler::VMemory::LoadCode( const std::string& code ) { Reset(); std::string line; std::string::const_iterator iter = code.begin(); while ( iter != code.end() ) { line += *iter; if ( *iter == '\n' ) { //TranslateLine(line); TokenList tokList = str2Tok( line ); Instruction ins; for ( auto iter = tokList.begin(); iter != tokList.end(); iter++ ) { ins.m_elements.push_back( *iter ); } m_instructions.push_back( ins ); line = ""; } iter++; } }
//Creates a new TokenList, and returns a pointer to this list //Searches for all conditional expressions in tokenList and appends them to the new list //Format is as follows: //Each token that is part of a condtional expression is appended sequentially //At the end of a conditional expression a newline character is appened //Example: if (a = true) then //Your list should include "(", "a", "=", "true", ")" and "\n" //tokenList is NOT modified TokenList* findAllConditionalExpressions(const TokenList &tokenList) { TokenList* conditionalExpressionTokenList = new TokenList(); Token* t = tokenList.getFirst(); while (t!= nullptr && t->getNext()!= nullptr) { if (t!= nullptr && (t->getStringRep() == "if" || t->getStringRep() == "elsif") && t->getPrev()->getStringRep()!="end" && !t->isComment()&& t->getNext()!= nullptr) { t = t->getNext(); while (t!= nullptr && !(t->getStringRep() == "then" && !t->isComment()) && t->getNext()!= nullptr){ conditionalExpressionTokenList->append(t->getStringRep()); t = t->getNext();} conditionalExpressionTokenList ->append("\n"); } if (t!= nullptr && t->getStringRep() == "when" && !t->isComment() && t->getNext()!= nullptr) { t = t->getNext(); while (t!=nullptr && !(t->getStringRep() == "else" && !t->isComment())&& t->getNext()!= nullptr) { conditionalExpressionTokenList->append(t->getStringRep()); t = t->getNext();} conditionalExpressionTokenList ->append("\n"); } t = t->getNext();} return conditionalExpressionTokenList; }
//check for mismatch type int error_type(const TokenList &tokenList, const TokenList &tokenList2) { int mis_type = 0; Token *current = new Token; Token *numbers = new Token; Token *temp = new Token; current = tokenList.getFirst(); numbers = tokenList2.getFirst(); while (current && numbers) { // find a operator and look at surrounding tokens if (current->getTokenType() == T_Identifier && current->getTokenDetails() != NULL && current->getNext()->getTokenType() == T_Operator && current->getNext()->getNext()->getTokenDetails() != NULL && current->getNext()->getNext()->getTokenType() != T_Literal) { string temp_typeA = current->getTokenDetails()->type; string temp_typeB = current->getNext()->getNext()->getTokenDetails()->type; //if details of type are not the same if (temp_typeA != temp_typeB) { cout << "ERROR mismatch type on line: " << numbers->getStringRep() << endl; mis_type++; } } numbers = numbers->getNext(); current = current->getNext(); } return mis_type; }
string GetSection(const string& File, const string& Sec) { istringstream is(File); ASSERT(is); TokenList TL; bool Found = false; do { is >> TL; if (!TL.empty()) { TokenList TL2(TL[0],"=\\"); Found = TL2[0] == Sec; } } while(!Found && is); if (Found) { // Get the rest of the lines in the section while (*(TL.rbegin()->rbegin()) == '\\') { TokenList TL2; is >> TL2; TL += TL2; } } return TL.getString(); }
void ValueProcessor::interpolate(std::string &str, const ValueScope &scope) const { size_t start, end = 0; string key, value; const TokenList *var; TokenList variable; while ((start = str.find("@{", end)) != string::npos && (end = str.find("}", start)) != string::npos) { key = "@"; key.append(str.substr(start + 2, end - (start + 2))); var = scope.getVariable(key); if (var != NULL) { variable = *var; processValue(variable, scope); // Remove quotes off strings. if (variable.size() == 1 && variable.front().type == Token::STRING) { variable.front().removeQuotes(); } value = variable.toString(); str.replace(start, (end + 1) - start, value); end = start + value.length(); } } }
bool UnprocessedStatement::processDeclaration (Declaration* declaration) { TokenList property; Token keyword; #ifdef WITH_LIBGLOG VLOG(3) << "Declaration"; #endif getValue(declaration->getValue()); if (declaration->getValue().empty() || declaration->getValue().front().type != Token::COLON) { return NULL; } declaration->getValue().pop_front(); getProperty(property); keyword = property.front(); keyword.assign(property.toString()); declaration->setProperty(keyword); return true; }
const TokenList *ValueProcessor::processDeepVariable( TokenList::const_iterator &i, TokenList::const_iterator &end, const ValueScope &scope) const { const TokenList *var; TokenList variable; std::string key = "@"; if (i == end || (*i).type != Token::OTHER || (*i) != "@") return NULL; i++; if (i == end || (*i).type != Token::ATKEYWORD || (var = scope.getVariable((*i))) == NULL) { i--; return NULL; } variable = *var; processValue(variable, scope); if (variable.size() != 1 || variable.front().type != Token::STRING) { i--; return NULL; } i++; // generate key with '@' + var without quotes variable.front().removeQuotes(); key.append(variable.front()); return scope.getVariable(key); }
Variable_array * VariableScript::parse(const c8 *text, size_t length, const Token::File *file) { TokenList tokens; if (m_lexer.scan(text, length, file, tokens) < 0) { _makeErrorMsg("lexical error at %s", m_lexer.error().c_str()); return 0; } tokens.push_back(vnnew Token(file)); for (TokenList::iterator it = tokens.begin(); it != tokens.end(); ++it) { LR_Parser::Result ret = m_parser.input(it->ptr()); if (ret == LR_Parser::kAccept) { break; } if (ret == LR_Parser::kFailed) { _makeErrorMsg("syntax error at %s(%u): %s", (file ? file->name.c_str() : ""), it->ptr()->row + 1, it->ptr()->text.c_str()); break; } } LR_Node *root = m_parser.result(); if (root) { root->grab(); m_parser.reset(); } else { m_parser.reset(); return 0; } m_errorMsg.clear(); Variable_array *ret = vnnew Variable_array(); _build_value_list(root->child(0), ret); root->drop(); return ret; }
bool ValueProcessor::validateCondition(const TokenList &value, const ValueScope &scope, bool defaultVal) const { TokenList::const_iterator i = value.begin(); TokenList::const_iterator end = value.end(); bool negate = false; bool ret; skipWhitespace(i, end); if (*i == "not") { negate = true; i++; } ret = validateValue(i, end, scope, defaultVal); skipWhitespace(i, end); while (ret == true && i != value.end() && *i == "and") { i++; skipWhitespace(i, end); ret = validateValue(i, end, scope, defaultVal); skipWhitespace(i, end); } return negate ? !ret : ret; }
int removeComments(TokenList &tokenList) { /*Fill in implementation */ Token *temp, *temp1; int counter=0; if(tokenList.getFirst() != nullptr) { temp = tokenList.getFirst(); while(temp) { temp1 = temp->getNext(); if(temp->getStringRep() == "--")//if the comment symbol is found { counter++;//counts how many comments are removed tokenList.deleteToken(temp);//delets the comment symbol temp = temp1; temp1 = temp->getNext(); tokenList.deleteToken(temp);//deletes the comment body temp= temp1; } else { temp = temp->getNext(); } } } return counter;// returns the number of comments removed }
// getValue // Gets the value of the number/expression at the current position // If the current value is an integer, return it // If it's a parenthetical expression, evaluate, and return // evalMultiplication is a flag used to ensure multiplication happens before addition // If true, it will handle any multiplication that is applied to the current value // That way, what is returned to the addition function is what we actually want to add // Parameters: // iter - the iterator for the list of tokens // postExpr - the postfix expression we are converting to // evalMultiplication (input boolean) whether to evaluate multiplication, see above // Pre-condition: expr[pos] is an int or parenthesis // Post-condition: expr[pos] is the end of the value // If it was an int, it's on the int // If it was a parenthesis, it's on the end parenthesis // If handling multiplication, it's on the last multiplied int // postExpr has the values handled here pushed to it void getValue(ListIterator& iter, TokenList& postExpr, bool evalMultiplication) { bool negative = false; if (!iter.currentIsInteger() && iter.tokenChar() == '-') { negative = true; iter.advance(); } if (iter.currentIsInteger()) { postExpr.push_back(iter.integerValue()); if (negative) { postExpr.push_back(Token('~')); } while (evalMultiplication && isNextOperatorMultiplication(iter)) { iter.advance(); handleMultiplyLevelOperation(iter, postExpr); } } else { handleParenthesis(iter, postExpr); if (negative) { postExpr.push_back(Token('~')); } } }
Token* getAssignmentStatements(TokenList &tokenList) { Token* ptr = tokenList.getFirst(); //create a pointer pointing to the head of the full list of tokens from our cpp file while(ptr) //until the pointer is at the end { //if statement for checking any assignment / compound assignment operators if (ptr -> getStringRep() == "=" || ptr -> getStringRep() == "+=" || ptr -> getStringRep() == "-=" || ptr -> getStringRep() == "*=" || ptr -> getStringRep() == "/=" || ptr -> getStringRep() == "%=" || ptr -> getStringRep() == "^=" || ptr -> getStringRep() == "<<=" || ptr -> getStringRep() == ">>=" || ptr -> getStringRep() == "&=" || ptr -> getStringRep() == "|=") { numAssignmentStatements++; //since form is (Variable = Expression), only one token before the assignment operator would be taken into account ptr = ptr -> getPrev(); while (ptr -> getStringRep() != ";") //until a semicolon is found { //append the token string (part of the assignment statement) to the assigstats TokenList assigstats.append(ptr->getStringRep()); ptr = ptr -> getNext(); //move pointer to next token } assigstats.append(ptr->getStringRep()); //append semicolon to assigstats } ptr = ptr -> getNext(); //move pointer to next token } return assigstats.getFirst(); //return the head pointer so we can begin traversing through the list }
// Here we have to find the character c, and expand the token T. // only top-level characters are considered. Active chars are allowed. // MX is decreased. Job aborted if it becomes negative. bool token_ns::expand_nct(TokenList&L,Token T, int n, uchar c, int& MX) { TokenList res; bool result = false; TokenList Table[10]; // arguments of the commands while(!L.empty()) { Token t = L.front(); L.pop_front(); if(t.is_a_left_brace()) { // get a whole block L.push_front(t); // re-enter the brace in the list token_ns::fast_get_block(L,res); continue; } if(t.cmd_val() == 10) continue; // ignore spaces. if(!(t.is_a_char() && t.char_val()==c)) { // just copy res.push_back(t); continue; } result = true; // We found something MX--; if(MX<0) return true; for(int k=0;k<n;k++) Table[k+1] = get_a_param(L,false); TokenList W = the_parser.special_expand(Table); L.splice(L.begin(),W); } L.splice(L.end(),res); return result; }
//Removes all inline comments from the tokenList including the // marker //Returns the number of comments removed //tokenList must be existing object int removeInlineComments(TokenList &tokenList) { int count = 0; //counts the amount of inline comments removed Token* temp; //temp is going to be moved around the list looking for inline comments temp = tokenList.getFirst(); //temp gets initialized to the first token in the list while (temp) { //this loop will run till you hit the end of the list (i.e NULL) if (temp->getStringRep() == "//") { count++; //count as one inline comment to delete Token* destroy1; destroy1 = temp; //make a token to use in destroying the // if (temp->getNext() != NULL) { //check to make sure you do not have a blank inline comment on the last line //deletes the comment included in the inline comment Token* destroy2; destroy2 = temp->getNext(); temp = destroy2->getNext(); tokenList.deleteToken(destroy1); tokenList.deleteToken(destroy2); } else { delete temp; } } else { temp = temp->getNext(); } } return count; }
//Removes all inline comments from the tokenList including the // marker //Returns the number of comments removed int removeInlineComments(TokenList &tokenList) { int count = 0; Token *t = tokenList.getFirst(); while (t) { Token *c1, *c2; if (t->getStringRep() == "//") { c1 = t; c2 = t->getNext(); if (c2->getStringRep() != "\n") { t = c2->getNext(); tokenList.deleteToken(c2); tokenList.deleteToken(c1); count++; } else { t = c2; tokenList.deleteToken(c1); count++; } } else { t = t->getNext(); } } return count; }
//Remove all block comments from the tokenList including /* and */ markers //Returns the number of block comments removed (where each pair of /* and */ count as one comment) int removeBlockComments(TokenList &tokenList) { int count = 0; Token *t = tokenList.getFirst(); Token *prev; bool deleteNext = false; while (t) { if (t->getStringRep() == "/*") { deleteNext = true; } prev = t; t = t->getNext(); if (prev->getStringRep() == "*/") { tokenList.deleteToken(prev); deleteNext = false; count++; } if (deleteNext) { tokenList.deleteToken(prev); } } return count; }
void TestLex::test_tokenizeString_if_keywordToken() { Lex lex(m_SymbolTable); TokenList *tokens = lex.tokenizeString("if"); assert(tokens->length() == 1); assert((*tokens)[0]->value() == "if"); assert((*tokens)[0]->type() == "keyword"); }
void ValueProcessor::interpolate(TokenList &tokens, const ValueScope &scope) const { TokenList::iterator i; for (i = tokens.begin(); i != tokens.end(); i++) { interpolate((*i), scope); } }
// checks for missing then after if or elsif int error_then(const TokenList &tokenList, const TokenList &tokenList2) { int miss_then = 0; Token *current = new Token; Token *numbers = new Token; Token *temp = new Token; current = tokenList.getFirst(); numbers = tokenList2.getFirst(); while (current && numbers) { //finds if if (current->getStringRep() == "if" && current->getPrev() != NULL && current->getPrev()->getStringRep() != "end") { for (temp = current; temp != NULL; temp = temp->getNext()) { //not found then before elsif or else or end if (temp->getNext()->getStringRep() == "elsif" || temp->getStringRep() == "else" || temp->getStringRep() == "end") { cout << "ERROR missing \"then\" on line: " << numbers->getStringRep() << endl; miss_then++; break; } //found then else if (temp->getStringRep() == "then") { break; } } } //checks for elsif and then after if (current->getStringRep() == "elsif") { for (temp = current->getNext(); temp != NULL; temp = temp->getNext()) { //found then if (temp->getStringRep() == "then") { break; } //found elsif or else or end before then else if (temp->getStringRep() == "elsif" || temp->getStringRep() == "else" || temp->getStringRep() == "end") { cout << "ERROR missing \"then\" on line: " << numbers->getStringRep() << endl; miss_then++; break; } } } numbers = numbers->getNext(); current = current->getNext(); } return miss_then; }
cli::framework::TokenList cli::framework::Framework::tokenize(const int tokenCount, const char *tokens[], CommandSpecList allCommandSpecs) { Trace trace(__FILE__, __FUNCTION__, __LINE__); // build up all verbs, options, targets, and properties: StringList verbs; StringList targets; StringList properties; StringList options; for (size_t c = 0; c < allCommandSpecs.size(); c++) { verbs.push_back(allCommandSpecs[c].verb); for (size_t t = 0; t < allCommandSpecs[c].targets.size(); t++) { targets.push_back(allCommandSpecs[c].targets[t].name); } for (size_t p = 0; p < allCommandSpecs[c].properties.size(); p++) { properties.push_back(allCommandSpecs[c].properties[p].name); } for (size_t o = 0; o < allCommandSpecs[c].options.size(); o++) { options.push_back(allCommandSpecs[c].options[o].name); // only options have abbreviations if (!allCommandSpecs[c].options[o].abr.empty()) { options.push_back(allCommandSpecs[c].options[o].abr); } } } // Options come from built in framework option map and command specs cli::framework::StringMap optionsMap; getOptionsMap(optionsMap); Lexer lexer(verbs, targets, options, properties); TokenList result = lexer.tokenize(tokenCount, tokens); // convert potential option abbreviations to name for (size_t i = 0; i < result.size(); i++) { if (result[i].tokenType == TOKENTYPE_OPTION) { if (optionsMap.find(result[i].lexeme) != optionsMap.end()) { result[i].lexeme = optionsMap[result[i].lexeme]; } } } return result; }
void CssWriter::writeMediaQueryStart(const TokenList &selector) { TokenList::const_iterator i; for (i = selector.begin(); i != selector.end(); i++) { const Token next = *i; out->write(next.c_str(), next.size()); } out->write("{", 1); }
void Instruction::print_tokens(const TokenList& t) const { for (TokenList::const_iterator it = t.begin(); it != t.end(); ++it) { it->debug(std::cout); std::cout << " "; } std::cout << std::endl; }
void Imm2asm::TranslateLine( std::string& line ) { TokenList tokenList = str2Tok(line); if ( tokenList.size() > 0 ) { _translateLine(tokenList); } }
TokenList* TokenList::clone() { TokenList* newtokens = new TokenList(); list<Token*>::iterator it; for (it = tokens.begin(); it != tokens.end(); it++) { newtokens->push((*it)->clone()); } return newtokens; }