//Remove all block comments from the tokenList including /* and */ markers
//Returns the number of block comments removed (where each pair of /* and */ count as one comment)
int removeBlockComments(TokenList &tokenList) {
//this funtion deletes the block comments in the tokenlist
int count = 0;
bool deleted = false;
Token* temp, *destroy;
temp = tokenList.getFirst();
while (temp) {
//loop check if list is empty
deleted = false;
if (temp->getStringRep() == "/*") {
//upon finding a block entry comment you keep deleating tokens till you find the exit token
count++;
while (!deleted) {
destroy = temp;
temp = temp->getNext();
tokenList.deleteToken(destroy);
if (temp->getStringRep() == "*/") {
//once the exit block token is found stop delete looping and continue searching through the list for block entry symbols
destroy = temp;
temp = temp->getNext();
deleted = true;
tokenList.deleteToken(destroy);
}
}
}
else {
temp = temp->getNext();
}
}
return count;
}
unsigned int Bayes::GetTokenCount(const string& token, BayesTokenType type)
{
TokenList list = type == Spam ? spam : type == Ham ? ham : TokenList();
if(list.count(token) == 0)
return 0;
return list[token];
}
Expression ExpressionBuilder::Build(std::string expression) {
TokenList tokens = m_tokenizer(expression);
TokenList postfix = m_converter(tokens);
std::stack<ExpressionNode*> nodeStk;
auto it = postfix.begin();
nodeStk.push(m_nodeBuilder.NewExpressionNode(*it));
while (++it != postfix.end()) {
ExpressionNode* node = m_nodeBuilder.NewExpressionNode(*it);
switch (node->NumChildren()) {
case 0:
nodeStk.push(node);
break;
case 1:
static_cast<OperatorNode*>(node)->SetChild(nodeStk.top());
nodeStk.pop();
nodeStk.push(node);
break;
case 2:
static_cast<OperatorNode*>(node)->SetChild(nodeStk.top(), 1);
nodeStk.pop();
static_cast<OperatorNode*>(node)->SetChild(nodeStk.top(), 0);
nodeStk.pop();
nodeStk.push(node);
break;
}
}
return{ nodeStk.top(), m_nodeBuilder.GetVariableMap() };
}
Declaration* CssParser::parseDeclaration () {
Declaration* declaration = NULL;
TokenList property;
if (!parseProperty(&property))
return NULL;
skipWhitespace();
declaration = new Declaration(property.toString());
if (tokenizer->getTokenType() != Token::COLON) {
throw new ParseException(tokenizer->getToken()->str,
"colon following property(':')");
}
tokenizer->readNextToken();
skipWhitespace();
TokenList* value = parseValue();
if (value == NULL) {
throw new ParseException(tokenizer->getToken()->str,
"value for property");
}
declaration->setValue(value);
return declaration;
}
//Removes all comments from the tokenList including the -- marker
//Returns the number of comments removed
int removeComments(TokenList &tokenList)
{
int count = 0;
Token* current = tokenList.getFirst();//initialized current to the first token
while (current) //this loop run till end of the list
{
if (current->getStringRep() == "--")
{
count++; //count comment to remove
Token* remove_first = current;//make a pointer to remove '--'
if (current->getNext() != NULL)
{
Token* remove_second = current->getNext();
//check comment on the last line
current = remove_second->getNext();
tokenList.deleteToken(remove_first);
tokenList.deleteToken(remove_second);
}
else
{
delete current;
}
}
else
{
current = current->getNext();
}
}
return count; //Returns the number of comments removed
}
//remove token of given type
int removeTokensOfType(TokenList &tokenList, tokenType type)
{
int count = 0;
Token* current = tokenList.getFirst();//initialized current to the first token
while (current) //this loop run till end of the list
{
if (current->getTokenType() == type)
{
count++;
Token* remove_first = current;
if (current->getNext() != NULL)
{
tokenList.deleteToken(remove_first);
}
else
{
delete current;
}
}
else
{
current = current->getNext();
}
}
return count;
}
//////////////////////////////////////////////////////////////////////////
//
// Class VMemory
//
void ETCompiler::VMemory::LoadCode( const std::string& code )
{
Reset();
std::string line;
std::string::const_iterator iter = code.begin();
while ( iter != code.end() )
{
line += *iter;
if ( *iter == '\n' )
{
//TranslateLine(line);
TokenList tokList = str2Tok( line );
Instruction ins;
for ( auto iter = tokList.begin(); iter != tokList.end(); iter++ )
{
ins.m_elements.push_back( *iter );
}
m_instructions.push_back( ins );
line = "";
}
iter++;
}
}
//Creates a new TokenList, and returns a pointer to this list
//Searches for all conditional expressions in tokenList and appends them to the new list
//Format is as follows:
//Each token that is part of a condtional expression is appended sequentially
//At the end of a conditional expression a newline character is appened
//Example: if (a = true) then
//Your list should include "(", "a", "=", "true", ")" and "\n"
//tokenList is NOT modified
TokenList* findAllConditionalExpressions(const TokenList &tokenList)
{
TokenList* conditionalExpressionTokenList = new TokenList();
Token* t = tokenList.getFirst();
while (t!= nullptr && t->getNext()!= nullptr)
{
if (t!= nullptr && (t->getStringRep() == "if" || t->getStringRep() == "elsif") && t->getPrev()->getStringRep()!="end" && !t->isComment()&& t->getNext()!= nullptr)
{
t = t->getNext();
while (t!= nullptr && !(t->getStringRep() == "then" && !t->isComment()) && t->getNext()!= nullptr){
conditionalExpressionTokenList->append(t->getStringRep());
t = t->getNext();}
conditionalExpressionTokenList ->append("\n");
}
if (t!= nullptr && t->getStringRep() == "when" && !t->isComment() && t->getNext()!= nullptr)
{
t = t->getNext();
while (t!=nullptr && !(t->getStringRep() == "else" && !t->isComment())&& t->getNext()!= nullptr)
{
conditionalExpressionTokenList->append(t->getStringRep());
t = t->getNext();}
conditionalExpressionTokenList ->append("\n");
}
t = t->getNext();}
return conditionalExpressionTokenList;
}
//check for mismatch type
int error_type(const TokenList &tokenList, const TokenList &tokenList2)
{
int mis_type = 0;
Token *current = new Token;
Token *numbers = new Token;
Token *temp = new Token;
current = tokenList.getFirst();
numbers = tokenList2.getFirst();
while (current && numbers)
{
// find a operator and look at surrounding tokens
if (current->getTokenType() == T_Identifier && current->getTokenDetails() != NULL &&
current->getNext()->getTokenType() == T_Operator && current->getNext()->getNext()->getTokenDetails() != NULL &&
current->getNext()->getNext()->getTokenType() != T_Literal)
{
string temp_typeA = current->getTokenDetails()->type;
string temp_typeB = current->getNext()->getNext()->getTokenDetails()->type;
//if details of type are not the same
if (temp_typeA != temp_typeB)
{
cout << "ERROR mismatch type on line: " << numbers->getStringRep() << endl;
mis_type++;
}
}
numbers = numbers->getNext();
current = current->getNext();
}
return mis_type;
}
string GetSection(const string& File, const string& Sec)
{
istringstream is(File);
ASSERT(is);
TokenList TL;
bool Found = false;
do {
is >> TL;
if (!TL.empty()) {
TokenList TL2(TL[0],"=\\");
Found = TL2[0] == Sec;
}
} while(!Found && is);
if (Found) { // Get the rest of the lines in the section
while (*(TL.rbegin()->rbegin()) == '\\') {
TokenList TL2;
is >> TL2;
TL += TL2;
}
}
return TL.getString();
}
void ValueProcessor::interpolate(std::string &str,
const ValueScope &scope) const {
size_t start, end = 0;
string key, value;
const TokenList *var;
TokenList variable;
while ((start = str.find("@{", end)) != string::npos &&
(end = str.find("}", start)) != string::npos) {
key = "@";
key.append(str.substr(start + 2, end - (start + 2)));
var = scope.getVariable(key);
if (var != NULL) {
variable = *var;
processValue(variable, scope);
// Remove quotes off strings.
if (variable.size() == 1 && variable.front().type == Token::STRING) {
variable.front().removeQuotes();
}
value = variable.toString();
str.replace(start, (end + 1) - start, value);
end = start + value.length();
}
}
}
bool UnprocessedStatement::processDeclaration (Declaration* declaration) {
TokenList property;
Token keyword;
#ifdef WITH_LIBGLOG
VLOG(3) << "Declaration";
#endif
getValue(declaration->getValue());
if (declaration->getValue().empty() ||
declaration->getValue().front().type != Token::COLON) {
return NULL;
}
declaration->getValue().pop_front();
getProperty(property);
keyword = property.front();
keyword.assign(property.toString());
declaration->setProperty(keyword);
return true;
}
const TokenList *ValueProcessor::processDeepVariable(
TokenList::const_iterator &i,
TokenList::const_iterator &end,
const ValueScope &scope) const {
const TokenList *var;
TokenList variable;
std::string key = "@";
if (i == end || (*i).type != Token::OTHER || (*i) != "@")
return NULL;
i++;
if (i == end || (*i).type != Token::ATKEYWORD ||
(var = scope.getVariable((*i))) == NULL) {
i--;
return NULL;
}
variable = *var;
processValue(variable, scope);
if (variable.size() != 1 || variable.front().type != Token::STRING) {
i--;
return NULL;
}
i++;
// generate key with '@' + var without quotes
variable.front().removeQuotes();
key.append(variable.front());
return scope.getVariable(key);
}
Variable_array * VariableScript::parse(const c8 *text, size_t length, const Token::File *file) {
TokenList tokens;
if (m_lexer.scan(text, length, file, tokens) < 0) {
_makeErrorMsg("lexical error at %s", m_lexer.error().c_str());
return 0;
}
tokens.push_back(vnnew Token(file));
for (TokenList::iterator it = tokens.begin(); it != tokens.end(); ++it) {
LR_Parser::Result ret = m_parser.input(it->ptr());
if (ret == LR_Parser::kAccept) {
break;
}
if (ret == LR_Parser::kFailed) {
_makeErrorMsg("syntax error at %s(%u): %s", (file ? file->name.c_str() : ""), it->ptr()->row + 1, it->ptr()->text.c_str());
break;
}
}
LR_Node *root = m_parser.result();
if (root) {
root->grab();
m_parser.reset();
} else {
m_parser.reset();
return 0;
}
m_errorMsg.clear();
Variable_array *ret = vnnew Variable_array();
_build_value_list(root->child(0), ret);
root->drop();
return ret;
}
bool ValueProcessor::validateCondition(const TokenList &value,
const ValueScope &scope,
bool defaultVal) const {
TokenList::const_iterator i = value.begin();
TokenList::const_iterator end = value.end();
bool negate = false;
bool ret;
skipWhitespace(i, end);
if (*i == "not") {
negate = true;
i++;
}
ret = validateValue(i, end, scope, defaultVal);
skipWhitespace(i, end);
while (ret == true && i != value.end() && *i == "and") {
i++;
skipWhitespace(i, end);
ret = validateValue(i, end, scope, defaultVal);
skipWhitespace(i, end);
}
return negate ? !ret : ret;
}
int removeComments(TokenList &tokenList)
{
/*Fill in implementation */
Token *temp, *temp1;
int counter=0;
if(tokenList.getFirst() != nullptr)
{
temp = tokenList.getFirst();
while(temp)
{
temp1 = temp->getNext();
if(temp->getStringRep() == "--")//if the comment symbol is found
{
counter++;//counts how many comments are removed
tokenList.deleteToken(temp);//delets the comment symbol
temp = temp1;
temp1 = temp->getNext();
tokenList.deleteToken(temp);//deletes the comment body
temp= temp1;
}
else
{
temp = temp->getNext();
}
}
}
return counter;// returns the number of comments removed
}
// getValue
// Gets the value of the number/expression at the current position
// If the current value is an integer, return it
// If it's a parenthetical expression, evaluate, and return
// evalMultiplication is a flag used to ensure multiplication happens before addition
// If true, it will handle any multiplication that is applied to the current value
// That way, what is returned to the addition function is what we actually want to add
// Parameters:
// iter - the iterator for the list of tokens
// postExpr - the postfix expression we are converting to
// evalMultiplication (input boolean) whether to evaluate multiplication, see above
// Pre-condition: expr[pos] is an int or parenthesis
// Post-condition: expr[pos] is the end of the value
// If it was an int, it's on the int
// If it was a parenthesis, it's on the end parenthesis
// If handling multiplication, it's on the last multiplied int
// postExpr has the values handled here pushed to it
void getValue(ListIterator& iter, TokenList& postExpr, bool evalMultiplication)
{
bool negative = false;
if (!iter.currentIsInteger() && iter.tokenChar() == '-')
{
negative = true;
iter.advance();
}
if (iter.currentIsInteger())
{
postExpr.push_back(iter.integerValue());
if (negative)
{
postExpr.push_back(Token('~'));
}
while (evalMultiplication && isNextOperatorMultiplication(iter))
{
iter.advance();
handleMultiplyLevelOperation(iter, postExpr);
}
}
else
{
handleParenthesis(iter, postExpr);
if (negative)
{
postExpr.push_back(Token('~'));
}
}
}
Token* getAssignmentStatements(TokenList &tokenList)
{
Token* ptr = tokenList.getFirst(); //create a pointer pointing to the head of the full list of tokens from our cpp file
while(ptr) //until the pointer is at the end
{
//if statement for checking any assignment / compound assignment operators
if (ptr -> getStringRep() == "=" || ptr -> getStringRep() == "+=" ||
ptr -> getStringRep() == "-=" || ptr -> getStringRep() == "*=" ||
ptr -> getStringRep() == "/=" || ptr -> getStringRep() == "%=" ||
ptr -> getStringRep() == "^=" || ptr -> getStringRep() == "<<=" ||
ptr -> getStringRep() == ">>=" || ptr -> getStringRep() == "&=" || ptr -> getStringRep() == "|=")
{
numAssignmentStatements++;
//since form is (Variable = Expression), only one token before the assignment operator would be taken into account
ptr = ptr -> getPrev();
while (ptr -> getStringRep() != ";") //until a semicolon is found
{
//append the token string (part of the assignment statement) to the assigstats TokenList
assigstats.append(ptr->getStringRep());
ptr = ptr -> getNext(); //move pointer to next token
}
assigstats.append(ptr->getStringRep()); //append semicolon to assigstats
}
ptr = ptr -> getNext(); //move pointer to next token
}
return assigstats.getFirst(); //return the head pointer so we can begin traversing through the list
}
// Here we have to find the character c, and expand the token T.
// only top-level characters are considered. Active chars are allowed.
// MX is decreased. Job aborted if it becomes negative.
bool token_ns::expand_nct(TokenList&L,Token T, int n, uchar c, int& MX)
{
TokenList res;
bool result = false;
TokenList Table[10]; // arguments of the commands
while(!L.empty()) {
Token t = L.front();
L.pop_front();
if(t.is_a_left_brace()) { // get a whole block
L.push_front(t); // re-enter the brace in the list
token_ns::fast_get_block(L,res);
continue;
}
if(t.cmd_val() == 10)
continue; // ignore spaces.
if(!(t.is_a_char() && t.char_val()==c)) { // just copy
res.push_back(t);
continue;
}
result = true; // We found something
MX--;
if(MX<0) return true;
for(int k=0;k<n;k++)
Table[k+1] = get_a_param(L,false);
TokenList W = the_parser.special_expand(Table);
L.splice(L.begin(),W);
}
L.splice(L.end(),res);
return result;
}
//Removes all inline comments from the tokenList including the // marker
//Returns the number of comments removed
//tokenList must be existing object
int removeInlineComments(TokenList &tokenList) {
int count = 0; //counts the amount of inline comments removed
Token* temp;
//temp is going to be moved around the list looking for inline comments
temp = tokenList.getFirst();
//temp gets initialized to the first token in the list
while (temp) {
//this loop will run till you hit the end of the list (i.e NULL)
if (temp->getStringRep() == "//") {
count++;
//count as one inline comment to delete
Token* destroy1;
destroy1 = temp;
//make a token to use in destroying the //
if (temp->getNext() != NULL) {
//check to make sure you do not have a blank inline comment on the last line
//deletes the comment included in the inline comment
Token* destroy2;
destroy2 = temp->getNext();
temp = destroy2->getNext();
tokenList.deleteToken(destroy1);
tokenList.deleteToken(destroy2);
}
else {
delete temp;
}
}
else {
temp = temp->getNext();
}
}
return count;
}
//Removes all inline comments from the tokenList including the // marker
//Returns the number of comments removed
int removeInlineComments(TokenList &tokenList) {
int count = 0;
Token *t = tokenList.getFirst();
while (t) {
Token *c1, *c2;
if (t->getStringRep() == "//") {
c1 = t;
c2 = t->getNext();
if (c2->getStringRep() != "\n") {
t = c2->getNext();
tokenList.deleteToken(c2);
tokenList.deleteToken(c1);
count++;
}
else {
t = c2;
tokenList.deleteToken(c1);
count++;
}
}
else {
t = t->getNext();
}
}
return count;
}
//Remove all block comments from the tokenList including /* and */ markers
//Returns the number of block comments removed (where each pair of /* and */ count as one comment)
int removeBlockComments(TokenList &tokenList) {
int count = 0;
Token *t = tokenList.getFirst();
Token *prev;
bool deleteNext = false;
while (t) {
if (t->getStringRep() == "/*") {
deleteNext = true;
}
prev = t;
t = t->getNext();
if (prev->getStringRep() == "*/") {
tokenList.deleteToken(prev);
deleteNext = false;
count++;
}
if (deleteNext) {
tokenList.deleteToken(prev);
}
}
return count;
}
void TestLex::test_tokenizeString_if_keywordToken()
{
Lex lex(m_SymbolTable);
TokenList *tokens = lex.tokenizeString("if");
assert(tokens->length() == 1);
assert((*tokens)[0]->value() == "if");
assert((*tokens)[0]->type() == "keyword");
}
void ValueProcessor::interpolate(TokenList &tokens,
const ValueScope &scope) const {
TokenList::iterator i;
for (i = tokens.begin(); i != tokens.end(); i++) {
interpolate((*i), scope);
}
}
// checks for missing then after if or elsif
int error_then(const TokenList &tokenList, const TokenList &tokenList2)
{
int miss_then = 0;
Token *current = new Token;
Token *numbers = new Token;
Token *temp = new Token;
current = tokenList.getFirst();
numbers = tokenList2.getFirst();
while (current && numbers)
{
//finds if
if (current->getStringRep() == "if" && current->getPrev() != NULL &&
current->getPrev()->getStringRep() != "end")
{
for (temp = current; temp != NULL; temp = temp->getNext())
{
//not found then before elsif or else or end
if (temp->getNext()->getStringRep() == "elsif" || temp->getStringRep() == "else" || temp->getStringRep() == "end")
{
cout << "ERROR missing \"then\" on line: " << numbers->getStringRep() << endl;
miss_then++;
break;
}
//found then
else if (temp->getStringRep() == "then")
{
break;
}
}
}
//checks for elsif and then after
if (current->getStringRep() == "elsif")
{
for (temp = current->getNext(); temp != NULL; temp = temp->getNext())
{
//found then
if (temp->getStringRep() == "then")
{
break;
}
//found elsif or else or end before then
else if (temp->getStringRep() == "elsif" || temp->getStringRep() == "else" || temp->getStringRep() == "end")
{
cout << "ERROR missing \"then\" on line: " << numbers->getStringRep() << endl;
miss_then++;
break;
}
}
}
numbers = numbers->getNext();
current = current->getNext();
}
return miss_then;
}
cli::framework::TokenList cli::framework::Framework::tokenize(const int tokenCount,
const char *tokens[],
CommandSpecList allCommandSpecs)
{
Trace trace(__FILE__, __FUNCTION__, __LINE__);
// build up all verbs, options, targets, and properties:
StringList verbs;
StringList targets;
StringList properties;
StringList options;
for (size_t c = 0; c < allCommandSpecs.size(); c++)
{
verbs.push_back(allCommandSpecs[c].verb);
for (size_t t = 0; t < allCommandSpecs[c].targets.size(); t++)
{
targets.push_back(allCommandSpecs[c].targets[t].name);
}
for (size_t p = 0; p < allCommandSpecs[c].properties.size(); p++)
{
properties.push_back(allCommandSpecs[c].properties[p].name);
}
for (size_t o = 0; o < allCommandSpecs[c].options.size(); o++)
{
options.push_back(allCommandSpecs[c].options[o].name);
// only options have abbreviations
if (!allCommandSpecs[c].options[o].abr.empty())
{
options.push_back(allCommandSpecs[c].options[o].abr);
}
}
}
// Options come from built in framework option map and command specs
cli::framework::StringMap optionsMap;
getOptionsMap(optionsMap);
Lexer lexer(verbs, targets, options, properties);
TokenList result = lexer.tokenize(tokenCount, tokens);
// convert potential option abbreviations to name
for (size_t i = 0; i < result.size(); i++)
{
if (result[i].tokenType == TOKENTYPE_OPTION)
{
if (optionsMap.find(result[i].lexeme) != optionsMap.end())
{
result[i].lexeme = optionsMap[result[i].lexeme];
}
}
}
return result;
}
void CssWriter::writeMediaQueryStart(const TokenList &selector) {
TokenList::const_iterator i;
for (i = selector.begin(); i != selector.end(); i++) {
const Token next = *i;
out->write(next.c_str(), next.size());
}
out->write("{", 1);
}
void Instruction::print_tokens(const TokenList& t) const
{
for (TokenList::const_iterator it = t.begin(); it != t.end(); ++it)
{
it->debug(std::cout);
std::cout << " ";
}
std::cout << std::endl;
}
void Imm2asm::TranslateLine( std::string& line )
{
TokenList tokenList = str2Tok(line);
if ( tokenList.size() > 0 )
{
_translateLine(tokenList);
}
}
TokenList* TokenList::clone() {
TokenList* newtokens = new TokenList();
list<Token*>::iterator it;
for (it = tokens.begin(); it != tokens.end(); it++) {
newtokens->push((*it)->clone());
}
return newtokens;
}
