Example #1
0
bool CssTokenizer::readNumSuffix() {
  if (lastRead == '%') {
    currentToken.type = Token::PERCENTAGE;
    currentToken.add(lastRead);
    readChar();
    return true;
  } else if (readIdent())  {
    currentToken.type = Token::DIMENSION;
    return true;
  }
  // TODO: Support identifiers starting with '-'
  return false;
}
Example #2
0
// -----------------------------------
void OggVorbisSubStream::procHeaders(Channel *ch)
{
	unsigned int packPtr=0;

	for(int i=0; i<pack.numPackets; i++)
	{
		MemoryStream vin(&pack.body[packPtr],pack.packetSizes[i]);

		packPtr += pack.packetSizes[i];

		char id[8];

		vin.read(id,7);
		id[7]=0;

		switch (id[0])
		{
			case 1:	// ident
				LOG_CHANNEL("OGG Vorbis Header: Ident (%d bytes)",vin.len);
				readIdent(vin,ch->info);
				break;
			case 3: // comment
				{
					LOG_CHANNEL("OGG Vorbis Header: Comment (%d bytes)",vin.len);
					ChanInfo newInfo = ch->info;
					readComment(vin,newInfo);
					ch->updateInfo(newInfo);
				}
				break;
			case 5: // setup
				LOG_CHANNEL("OGG Vorbis Header: Setup (%d bytes)",vin.len);
				//readSetup(vin);
				break;
			default:
				throw StreamException("Unknown Vorbis packet header type");
				break;
		}
	}

}
Example #3
0
Lexeme Lexal::getNext()
{
    skipSpaces();
    if (reader.isEof())
        return Lexeme(Lexeme::Eof, L"", line, pos);

    int startLine = line;
    int startPos = pos;

    wchar_t ch = reader.getNextChar();
    pos++;

    if (isIdentStart(ch))
        return readIdent(startLine, startPos, ch);
    else if (isDigit(ch))
        return readNumber(startLine, startPos, ch);
    else if (isQuote(ch))
        return readString(startLine, startPos, ch);
    else if (isSymbol(ch))
        return Lexeme(Lexeme::Symbol, toString(ch), startLine, startPos);
    
    throw Exception(L"Invalid character at "+ posToStr(startLine, startPos));
}
Example #4
0
Ttoken *Tmlex::getToken()
{
    Ttoken        *token   = NULL;
    TstrBuf        text    = "";
    unsigned long  textIdx = 0;
    int            found = 0;

    if (NULL != (token = popToken()))
    {
        return(token);
    }
    else
    {
        token = new Ttoken;
    }

    token->setType(TOK_ILLEGAL);

    while (!found)
    {
        int ch = Dinput.get();
        if (EOF != ch) text[textIdx++] = ch;

        if (EOF == ch)
        {
            // End of file
            token->setType(TOK_END);
            token->setText("");
            token->setPosn(Dinput.posn());
            found = 1;
        }
        else if ('\n' == ch)
        {
            token->setType(TOK_EOL);
            token->setText("");
            found = 1;
        }
        else if (isspace(ch))
        {
            // Ignore white space
        }
        else if (SEP_ESC == ch)
        {
            // excape
            ch = Dinput.get();

            if ('\n' == ch)
            {
                // ignore the end-of-line
            }
            else if (SEP_ESC == ch)
            {
                // Litteral
                Dinput.putBack(ch);
                token->setPosn(Dinput.posn());
                found = readIdent(*token);
            }
            else
            {
                // Excaped charactor
                Dinput.putBack(ch);
            }
        }
        else if ('#' == ch)
        {
            // Check for comment
            token->setPosn(Dinput.posn());
            ch = Dinput.get();

            found = readComment(*token);
        }
        else if (SEP_TEXT == ch)
        {
            // Text
            token->setPosn(Dinput.posn());
            found = readText(*token);
        }
        else if (SEP_S_VAR == ch)
        {
            // Start an evaluated variable (define)
            readVar();
        }
        else if (SEP_CALL == ch)
        {
            // Call
            token->setPosn(Dinput.posn());
            found = readCall(*token);
        }
        else if (isSymb(ch))
        {
            // Symbol
            token->setPosn(Dinput.posn());
            Dinput.putBack(ch);
            found = readSymb(*token);
        }
        else if (isIdent(ch))
        {
            // Ident
            token->setPosn(Dinput.posn());
            Dinput.putBack(ch);
            found = readIdent(*token);
        }
        else
        {
            while ((EOF != ch) && !isspace(ch)) ch = Dinput.get();
            if (EOF != ch) Dinput.putBack(ch);
        }
    }

    return(token);
}
Example #5
0
Token::Type CssTokenizer::readNextToken(){
  if (in == NULL) {
    currentToken.type = Token::EOS;
    return Token::EOS;
  }

  currentToken.clear();
  switch (lastRead) {
  case '@':
    currentToken.type = Token::ATKEYWORD;
    currentToken.add(lastRead);
    readChar();
    if (!readIdent()) {
      currentToken.type = Token::OTHER;
    }
    break;
    
  case '#':
    currentToken.type = Token::HASH;
    currentToken.add(lastRead);
    readChar();
    if (!readName()) {
      throw new ParseException(&lastRead,
                               "name following '#'");
    }
    break;
    
  case '-':
    currentToken.add(lastRead);
    readChar();
    if (readNum(true)) {
      currentToken.type = Token::NUMBER;
      readNumSuffix();
    } else if (readIdent()) {
      currentToken.type = Token::IDENTIFIER;
    } else
      currentToken.type = Token::OTHER;
    break;
    
  case '~':
    currentToken.add(lastRead);
    readChar();
    if (lastRead == '=') {
      currentToken.add(lastRead);
      readChar();
      currentToken.type = Token::INCLUDES;
    } else
      currentToken.type = Token::OTHER;
    break;
    
  case '|':
    currentToken.add(lastRead);
    readChar();
    if (lastRead == '=') {
      currentToken.add(lastRead);
      readChar();
      currentToken.type = Token::DASHMATCH;
    } else
      currentToken.type = Token::OTHER;
    break;
    
  case '/':
    currentToken.add(lastRead);
    readChar();
    if (readComment()) 
      currentToken.type = Token::COMMENT;
    else
      currentToken.type = Token::OTHER;
    break;
    
  case ';':
    currentToken.type = Token::DELIMITER;
    currentToken.add(lastRead);
    readChar();
    break;
  case ':':
    currentToken.type = Token::COLON;
    currentToken.add(lastRead);
    readChar();
    break;
  case '{':
    currentToken.type = Token::BRACKET_OPEN;
    currentToken.add(lastRead);
    readChar();
    break;
  case '}':
    currentToken.type = Token::BRACKET_CLOSED;
    currentToken.add(lastRead);
    readChar();
    break;
  case '(':
    currentToken.type = Token::PAREN_OPEN;
    currentToken.add(lastRead);
    readChar();
    break;
  case ')':
    currentToken.type = Token::PAREN_CLOSED;
    currentToken.add(lastRead);
    readChar();
    break;
  case '[':
    currentToken.type = Token::BRACE_OPEN;
    currentToken.add(lastRead);
    readChar();
    break;
  case ']':
    currentToken.type = Token::BRACE_CLOSED;
    currentToken.add(lastRead);
    readChar();
    break;
    
  case '.':
    currentToken.add(lastRead);
    readChar();
    if (readNum(false)) {
      currentToken.type = Token::NUMBER;
      readNumSuffix();
    } 
    break;

  default:
    if (readString()) 
      currentToken.type = Token::STRING;
    else if (readNum(true)) {
      currentToken.type = Token::NUMBER;
      readNumSuffix();

    } else if (readIdent()) {
      currentToken.type = Token::IDENTIFIER;

      if (currentToken.str == "url" && readUrl())
        currentToken.type = Token::URL;
      else if (currentToken.str == "u" && lastReadEq('+')) {
        currentToken.add(lastRead);
        readChar();
        currentToken.type = Token::UNICODE_RANGE;
        readUnicodeRange();
      }
    } else if (readWhitespace()) {
      currentToken.type = Token::WHITESPACE;
      while (readWhitespace()) {};
    } else {
      currentToken.add(lastRead);
      readChar();
    }
    break;
  }

  return currentToken.type;
}