C++ (Cpp) isSingleCharTok Examples

Example #1

File: Tokenizer.cpp Project: karantech9/XML-Document-Object-Model

bool Toker::isTokEnd()
{
	if (isspace(nextChar))
		return true;
	if (isSingleCharTok(nextChar) || isSingleCharTok(currChar))
		return true;
	if (isIdentifierChar(currChar) && !isIdentifierChar(nextChar))
		return true;
	if (!isIdentifierChar(currChar) && isIdentifierChar(nextChar))
		return true;
	if (isFileEnd())
		return true;
	return false;
}

Example #2

File: Tokenizer.cpp Project: Grayer123/Remote-Performance-Assessment

std::string Toker::getTok()
{
  std::string tok = "";
  stripWhiteSpace();
  if(isSingleCharTok(nextChar))
  {
    getChar();
    tok.append(1,currChar);
    return tok;
  }
  do
  {
    if(isFileEnd())
      return tok;

    getChar();
    if(isBeginComment())
    {
      if(tok.length() > 0)
      {
        this->putback(currChar);
        return tok;
      }
      tok = eatComment();
      if(doReturnComments)
        return tok;
      else
      {
        tok = "";
        continue;
      }
    }
    if(isBeginQuote())
    {
      if(tok.length() > 0)
      {
        this->putback(currChar);
        return tok;
      }
      tok = eatQuote();
      return tok;
    }
    std::locale loc;
    if(!isspace(currChar, loc))
      tok.append(1,currChar);
  }  while(!isTokEnd() || tok.length() == 0);
  return tok;
}