Ejemplos de token_id_t en C++ (Cpp)

Ejemplo n.º 1

Archivo: pparser.c Proyecto: richard42/mame

ptokenizer::token_t ptokenizer::get_token_internal()
{
	/* skip ws */
	char c = getc();
	while (m_whitespace.find(c)>=0)
	{
		c = getc();
		if (eof())
		{
			return token_t(ENDOFFILE);
		}
	}
	if (m_number_chars_start.find(c)>=0)
	{
		/* read number while we receive number or identifier chars
		 * treat it as an identifier when there are identifier chars in it
		 *
		 */
		token_type ret = NUMBER;
		pstring tokstr = "";
		while (true) {
			if (m_identifier_chars.find(c)>=0 && m_number_chars.find(c)<0)
				ret = IDENTIFIER;
			else if (m_number_chars.find(c)<0)
				break;
			tokstr += c;
			c = getc();
		}
		ungetc();
		return token_t(ret, tokstr);
	}
	else if (m_identifier_chars.find(c)>=0)
	{
		/* read identifier till non identifier char */
		pstring tokstr = "";
		while (m_identifier_chars.find(c)>=0) {
			tokstr += c;
			c = getc();
		}
		ungetc();
		token_id_t id = token_id_t(m_tokens.indexof(tokstr));
		if (id.id() >= 0)
			return token_t(id, tokstr);
		else
		{
			return token_t(IDENTIFIER, tokstr);
		}
	}
	else if (c == m_string)
	{
		pstring tokstr = "";
		c = getc();
		while (c != m_string)
		{
			tokstr += c;
			c = getc();
		}
		return token_t(STRING, tokstr);
	}
	else
	{
		/* read identifier till first identifier char or ws */
		pstring tokstr = "";
		while ((m_identifier_chars.find(c)) < 0 && (m_whitespace.find(c) < 0)) {
			tokstr += c;
			/* expensive, check for single char tokens */
			if (tokstr.len() == 1)
			{
				token_id_t id = token_id_t(m_tokens.indexof(tokstr));
				if (id.id() >= 0)
					return token_t(id, tokstr);
			}
			c = getc();
		}
		ungetc();
		token_id_t id = token_id_t(m_tokens.indexof(tokstr));
		if (id.id() >= 0)
			return token_t(id, tokstr);
		else
		{
			return token_t(UNKNOWN, tokstr);
		}
	}

}

Ejemplo n.º 2

Archivo: nl_parser.c Proyecto: hstampfl/mame

ptokenizer::token_t ptokenizer::get_token_internal()
{
    /* skip ws */
    char c = getc();
    while (m_whitespace.find(c)>=0)
    {
        c = getc();
        if (eof())
        {
            return token_t(ENDOFFILE);
        }
    }
    if (m_identifier_chars.find(c)>=0)
    {
        /* read identifier till non identifier char */
        pstring tokstr = "";
        while (m_identifier_chars.find(c)>=0) {
            tokstr += c;
            c = getc();
        }
        ungetc();
        token_id_t id = token_id_t(m_tokens.indexof(tokstr));
        if (id.id() >= 0)
            return token_t(id, tokstr);
        else
        {
            return token_t(IDENTIFIER, tokstr);
        }
    }
    else if (c == m_string)
    {
        pstring tokstr = "";
        c = getc();
        while (c != m_string)
        {
            tokstr += c;
            c = getc();
        }
        return token_t(STRING, tokstr);
    }
    else
    {
        /* read identifier till first identifier char or ws */
        pstring tokstr = "";
        while ((m_identifier_chars.find(c)) < 0 && (m_whitespace.find(c) < 0)) {
            tokstr += c;
            /* expensive, check for single char tokens */
            if (tokstr.len() == 1)
            {
                token_id_t id = token_id_t(m_tokens.indexof(tokstr));
                if (id.id() >= 0)
                    return token_t(id, tokstr);
            }
            c = getc();
        }
        ungetc();
        token_id_t id = token_id_t(m_tokens.indexof(tokstr));
        if (id.id() >= 0)
            return token_t(id, tokstr);
        else
        {
            return token_t(UNKNOWN, tokstr);
        }
    }

}