bool Toker::isTokEnd() { if (isspace(nextChar)) return true; if (isSingleCharTok(nextChar) || isSingleCharTok(currChar)) return true; if (isIdentifierChar(currChar) && !isIdentifierChar(nextChar)) return true; if (!isIdentifierChar(currChar) && isIdentifierChar(nextChar)) return true; if (isFileEnd()) return true; return false; }
std::string Toker::getTok() { std::string tok = ""; stripWhiteSpace(); if(isSingleCharTok(nextChar)) { getChar(); tok.append(1,currChar); return tok; } do { if(isFileEnd()) return tok; getChar(); if(isBeginComment()) { if(tok.length() > 0) { this->putback(currChar); return tok; } tok = eatComment(); if(doReturnComments) return tok; else { tok = ""; continue; } } if(isBeginQuote()) { if(tok.length() > 0) { this->putback(currChar); return tok; } tok = eatQuote(); return tok; } std::locale loc; if(!isspace(currChar, loc)) tok.append(1,currChar); } while(!isTokEnd() || tok.length() == 0); return tok; }