Example #1
0
void GString::TrimRightWS()
{
	while ((_len) && (isWS(_str[_len - 1])))
		_len--;

	_str[_len] = 0;
}
Example #2
0
void GString::TrimLeftWS()
{
	int i = 0;
	while ((i < _len) && (isWS(_str[i])))
		i++;

	if (i != 0)
	{
		_len -= i;
		memmove(_str, &_str[i], _len + 1);
	}
}
Example #3
0
bool CueTokenizer<CharT>::nextline()
{
    m_fields.clear();
    int_type c;
    std::basic_string<CharT> field;
    while (traits_type::not_eof(c = m_sb->sbumpc())) {
	if (c == '"') {
	    // eat until closing quote
	    while (traits_type::not_eof(c = m_sb->sbumpc())) {
		if (c == '\n')
		    throw std::runtime_error(format(
				"Runaway string at line %d", m_lineno + 1));
		else if (c != '"')
		    field.push_back(c);
		else if (m_sb->sgetc() != '"') // closing quote
		    break;
		else { // escaped quote
		    m_sb->snextc();
		    field.push_back(c);
		}
	    }
	}
	else if (c == '\n') {
	    ++m_lineno;
	    break;
	}
	else if (isWS(c)) {
	    if (field.size()) {
		m_fields.push_back(field);
		field.clear();
	    }
	    while (isWS(m_sb->sgetc()))
		m_sb->snextc();
	}
	else
	    field.push_back(c);
    }
    if (field.size()) m_fields.push_back(field);
    return field.size() > 0 || c == '\n';
}
Example #4
0
//! Checks if there is whitespace in the range specified in the string
inline std::string findComment(const ExprNode& node) {
    const Expression& expr = *node.expr();
    typedef std::vector<std::pair<int, int> > Comments;
    const Comments& comments = expr.getComments();
    const std::string& s = expr.getExpr();

    // TODO: user lower_bound to make this O(lg n) instead of O(n)
    for (Comments::const_iterator i = comments.begin(); i != comments.end(); ++i) {
        if (i->first >= node.endPos() && isWS(s.c_str(), node.endPos(), i->first))
            return s.substr(i->first, i->second - i->first + 1);
    }
    return "";
}
Example #5
0
bool Lexer::GetToken(Token&obj)
{
	string strBuf;
	int id;
	DataValue dv;
	while (!isEOF())
	{
		skipWS();
		/////////////
		//  Check for chr/strlit
		//
		if (nextChar() == '-')
			bool b=true;
		if (nextChar() == '\'' || nextChar() == '\"')
		{
			char ch = getChar();
			strBuf =  "";
			while (!isEOF())
			{
				if  (nextChar() == '\r' || nextChar() == '\n')
				{
					dv.SetStrData("Newline in constant.");
					throw Token(Token::LEX_ERROR,dv);
				}
				if (nextChar() == ch)
				{
					getChar();
					if (ch == '\'')
					{
						if (strBuf.length() != 1)
						{
							dv.SetStrData("Character literals must have exactly one character.");
							throw Token(Token::LEX_ERROR,dv);
						}
						dv.SetCharData(strBuf[0]);
						obj=Token(Token::LEX_CHRLIT,dv);
						return true;
					}
					else
					{
						dv.SetStrData(strBuf);
						obj=Token(Token::LEX_STRLIT,dv);
						return true;
					}
				}
				strBuf += getChar();
			}
			dv.SetStrData("EOF in constant.");
			throw Token(Token::LEX_ERROR,dv);
		}
		/////////////
		//  Check for Operators, then keywords
		//
		StorePosition();
		
		id = dfaOperators.GetString(strBuf);
		if (id == 0)
		{
			ResetPosition();
			id = dfaKeywords.GetString(strBuf);
			if (id == 0)
			{

				/////////////
				//  Check for numbers/idents
				//
				while (!isEOF() && !isWS(nextChar()) && !dfaOperators.ValidFirst(nextChar()))
				{
					strBuf += toLower(getChar());
				}
				//if it's all numbers
				if (strBuf.find_first_not_of("0123456789",0,10) == string::npos)
				{
					//read in anything, including dots  (will cover floats and invalid idents)
					while (!isEOF() && !isWS(nextChar()) && (nextChar() == '.' || !dfaOperators.ValidFirst(nextChar())))
					{
						strBuf += toLower(getChar());
					}
				}
				bool found = false;
				if (strBuf.find_first_not_of("0123456789.",0,11) == string::npos)
				{
					string::size_type off;
					off = strBuf.find('.',0);
					if (off == string::npos)
					{
						dv.SetIntData(atoi(strBuf.c_str()));
						obj=Token(Token::LEX_INTLIT,dv);
						return true;
					}
					else
					{
						if (strBuf.find('.',off+1) == string::npos)
						{
							dv.SetFloatData(atof(strBuf.c_str()));
							obj=Token(Token::LEX_FLOLIT,dv);
							return true;
						}
					}
				}
				//validate identifier
				try
				{
					if (strBuf.length() > 20)
						throw 2;
					for (string::iterator it = strBuf.begin();it != strBuf.end();++it)
					{
						if (it == strBuf.begin())
						{
							if (!isAlpha(*it))
							{
								throw 0;
							}
						}
						if (!isAlpha(*it) && *it != '_' && !isNum(*it))
						{
							throw 1;
						}
					}
				} catch(int iError)
				{
					switch (iError)
					{
						case 0:
							dv.SetStrData("Unrecognized lexeme ("+strBuf+"): identifiers must begin with a letter.");
							throw Token(Token::LEX_ERROR,dv);
						case 1:
							dv.SetStrData("Unrecognized lexeme ("+strBuf+"): identifiers can only contain underscores and alphanumeric characters.");
							throw Token(Token::LEX_ERROR,dv);
						case 2:
							dv.SetStrData("Unrecognized lexeme ("+strBuf+"): identifiers can be at most 20 characters long.");
							throw Token(Token::LEX_ERROR,dv);
					}
				}
				dv.SetStrData(strBuf);
				obj=Token(Token::LEX_IDENT,dv);
				return true;
			}
		}
		if (id)
		{
			dv.Clear();
			obj=Token((Token::TokenType)id,dv);
			return true;
		}
		dv.SetStrData("Undefined error.");
		throw Token(Token::LEX_ERROR,dv);
	}
	//int 0 means eof
	dv.SetStrData("EOF");
	obj=Token(Token::LEX_ERROR,dv);
	return false;
}