bool CssTokenizer::readNumSuffix() {
if (lastRead == '%') {
currentToken.type = Token::PERCENTAGE;
currentToken.add(lastRead);
readChar();
return true;
} else if (readIdent()) {
currentToken.type = Token::DIMENSION;
return true;
}
// TODO: Support identifiers starting with '-'
return false;
}
// -----------------------------------
void OggVorbisSubStream::procHeaders(Channel *ch)
{
unsigned int packPtr=0;
for(int i=0; i<pack.numPackets; i++)
{
MemoryStream vin(&pack.body[packPtr],pack.packetSizes[i]);
packPtr += pack.packetSizes[i];
char id[8];
vin.read(id,7);
id[7]=0;
switch (id[0])
{
case 1: // ident
LOG_CHANNEL("OGG Vorbis Header: Ident (%d bytes)",vin.len);
readIdent(vin,ch->info);
break;
case 3: // comment
{
LOG_CHANNEL("OGG Vorbis Header: Comment (%d bytes)",vin.len);
ChanInfo newInfo = ch->info;
readComment(vin,newInfo);
ch->updateInfo(newInfo);
}
break;
case 5: // setup
LOG_CHANNEL("OGG Vorbis Header: Setup (%d bytes)",vin.len);
//readSetup(vin);
break;
default:
throw StreamException("Unknown Vorbis packet header type");
break;
}
}
}
Lexeme Lexal::getNext()
{
skipSpaces();
if (reader.isEof())
return Lexeme(Lexeme::Eof, L"", line, pos);
int startLine = line;
int startPos = pos;
wchar_t ch = reader.getNextChar();
pos++;
if (isIdentStart(ch))
return readIdent(startLine, startPos, ch);
else if (isDigit(ch))
return readNumber(startLine, startPos, ch);
else if (isQuote(ch))
return readString(startLine, startPos, ch);
else if (isSymbol(ch))
return Lexeme(Lexeme::Symbol, toString(ch), startLine, startPos);
throw Exception(L"Invalid character at "+ posToStr(startLine, startPos));
}
Ttoken *Tmlex::getToken()
{
Ttoken *token = NULL;
TstrBuf text = "";
unsigned long textIdx = 0;
int found = 0;
if (NULL != (token = popToken()))
{
return(token);
}
else
{
token = new Ttoken;
}
token->setType(TOK_ILLEGAL);
while (!found)
{
int ch = Dinput.get();
if (EOF != ch) text[textIdx++] = ch;
if (EOF == ch)
{
// End of file
token->setType(TOK_END);
token->setText("");
token->setPosn(Dinput.posn());
found = 1;
}
else if ('\n' == ch)
{
token->setType(TOK_EOL);
token->setText("");
found = 1;
}
else if (isspace(ch))
{
// Ignore white space
}
else if (SEP_ESC == ch)
{
// excape
ch = Dinput.get();
if ('\n' == ch)
{
// ignore the end-of-line
}
else if (SEP_ESC == ch)
{
// Litteral
Dinput.putBack(ch);
token->setPosn(Dinput.posn());
found = readIdent(*token);
}
else
{
// Excaped charactor
Dinput.putBack(ch);
}
}
else if ('#' == ch)
{
// Check for comment
token->setPosn(Dinput.posn());
ch = Dinput.get();
found = readComment(*token);
}
else if (SEP_TEXT == ch)
{
// Text
token->setPosn(Dinput.posn());
found = readText(*token);
}
else if (SEP_S_VAR == ch)
{
// Start an evaluated variable (define)
readVar();
}
else if (SEP_CALL == ch)
{
// Call
token->setPosn(Dinput.posn());
found = readCall(*token);
}
else if (isSymb(ch))
{
// Symbol
token->setPosn(Dinput.posn());
Dinput.putBack(ch);
found = readSymb(*token);
}
else if (isIdent(ch))
{
// Ident
token->setPosn(Dinput.posn());
Dinput.putBack(ch);
found = readIdent(*token);
}
else
{
while ((EOF != ch) && !isspace(ch)) ch = Dinput.get();
if (EOF != ch) Dinput.putBack(ch);
}
}
return(token);
}
Token::Type CssTokenizer::readNextToken(){
if (in == NULL) {
currentToken.type = Token::EOS;
return Token::EOS;
}
currentToken.clear();
switch (lastRead) {
case '@':
currentToken.type = Token::ATKEYWORD;
currentToken.add(lastRead);
readChar();
if (!readIdent()) {
currentToken.type = Token::OTHER;
}
break;
case '#':
currentToken.type = Token::HASH;
currentToken.add(lastRead);
readChar();
if (!readName()) {
throw new ParseException(&lastRead,
"name following '#'");
}
break;
case '-':
currentToken.add(lastRead);
readChar();
if (readNum(true)) {
currentToken.type = Token::NUMBER;
readNumSuffix();
} else if (readIdent()) {
currentToken.type = Token::IDENTIFIER;
} else
currentToken.type = Token::OTHER;
break;
case '~':
currentToken.add(lastRead);
readChar();
if (lastRead == '=') {
currentToken.add(lastRead);
readChar();
currentToken.type = Token::INCLUDES;
} else
currentToken.type = Token::OTHER;
break;
case '|':
currentToken.add(lastRead);
readChar();
if (lastRead == '=') {
currentToken.add(lastRead);
readChar();
currentToken.type = Token::DASHMATCH;
} else
currentToken.type = Token::OTHER;
break;
case '/':
currentToken.add(lastRead);
readChar();
if (readComment())
currentToken.type = Token::COMMENT;
else
currentToken.type = Token::OTHER;
break;
case ';':
currentToken.type = Token::DELIMITER;
currentToken.add(lastRead);
readChar();
break;
case ':':
currentToken.type = Token::COLON;
currentToken.add(lastRead);
readChar();
break;
case '{':
currentToken.type = Token::BRACKET_OPEN;
currentToken.add(lastRead);
readChar();
break;
case '}':
currentToken.type = Token::BRACKET_CLOSED;
currentToken.add(lastRead);
readChar();
break;
case '(':
currentToken.type = Token::PAREN_OPEN;
currentToken.add(lastRead);
readChar();
break;
case ')':
currentToken.type = Token::PAREN_CLOSED;
currentToken.add(lastRead);
readChar();
break;
case '[':
currentToken.type = Token::BRACE_OPEN;
currentToken.add(lastRead);
readChar();
break;
case ']':
currentToken.type = Token::BRACE_CLOSED;
currentToken.add(lastRead);
readChar();
break;
case '.':
currentToken.add(lastRead);
readChar();
if (readNum(false)) {
currentToken.type = Token::NUMBER;
readNumSuffix();
}
break;
default:
if (readString())
currentToken.type = Token::STRING;
else if (readNum(true)) {
currentToken.type = Token::NUMBER;
readNumSuffix();
} else if (readIdent()) {
currentToken.type = Token::IDENTIFIER;
if (currentToken.str == "url" && readUrl())
currentToken.type = Token::URL;
else if (currentToken.str == "u" && lastReadEq('+')) {
currentToken.add(lastRead);
readChar();
currentToken.type = Token::UNICODE_RANGE;
readUnicodeRange();
}
} else if (readWhitespace()) {
currentToken.type = Token::WHITESPACE;
while (readWhitespace()) {};
} else {
currentToken.add(lastRead);
readChar();
}
break;
}
return currentToken.type;
}
