bool CssTokenizer::readString() {
if (!lastReadEq('"') && !lastReadEq('\''))
return false;
char delim = lastRead;
currentToken.add(lastRead);
readChar();
while (in != NULL) {
if (lastReadEq(delim)) {
currentToken.add(lastRead);
readChar();
return true;
} else if (lastReadEq('\n') ||
lastReadEq('\r') ||
lastReadEq('\f')) {
throw new ParseException("end of line",
"end of string");
} else if (lastReadEq('\\'))
// note that even though readEscape() returns false it still
// eats the '\'.
readEscape() || readNewline();
else {
currentToken.add(lastRead);
readChar();
}
}
throw new ParseException("end of input",
"end of string");
return false;
}
GLECSVDataStatus GLECSVData::readCellString(GLEBYTE quote) {
unsigned int cellSize = 1;
unsigned int cellPos = lastCharPos();
initWritePos();
while (true) {
GLEBYTE ch = readChar();
writeChar(ch);
cellSize++;
if (ch == 0) {
m_error.errorCode = GLECSVErrorUnterminatedString;
m_error.errorLine = m_lines;
m_error.errorColumn = getUTF8Column(cellPos);
createErrorString("unterminated string");
return GLECSVDataStatusEOF;
} else if (isEol(ch)) {
m_error.errorCode = GLECSVErrorUnterminatedString;
m_error.errorLine = m_lines;
m_error.errorColumn = getUTF8Column(cellPos);
createErrorString("unterminated string");
return readNewline(ch);
} else if (ch == quote) {
GLEBYTE ch = readChar();
if (ch != quote) {
writeChar(ch);
createCell(cellSize, cellPos);
return skipSpacesAndFirstDelim(ch);
}
}
}
return GLECSVDataStatusOK;
}
void DefaultLexer::readLineComment() {
skipChar(); // skip '/'
skipChar(); // skip '/'
char c = lookChar();
while (c && !isNewline(c)) {
skipChar();
c = lookChar();
}
if (isNewline(c)) readNewline(c);
}
GLECSVDataStatus GLECSVData::skipTillEol() {
while (true) {
GLEBYTE ch = readChar();
if (ch == 0) {
return GLECSVDataStatusEOF;
}
if (isEol(ch)) {
return readNewline(ch);
}
}
}
GLECSVDataStatus GLECSVData::skipSpacesAndFirstDelim(GLEBYTE ch) {
while (true) {
if (!isSpace(ch)) {
if (ch == 0) {
return GLECSVDataStatusEOF;
} else if (isEol(ch)) {
return readNewline(ch);
} else if (isDelim(ch)) {
m_lastDelimWasSpace = isSpace(ch);
return GLECSVDataStatusOK;
} else {
goBack();
return GLECSVDataStatusOK;
}
}
ch = readChar();
}
return GLECSVDataStatusOK;
}
GLECSVDataStatus GLECSVData::readCell() {
GLEBYTE ch = readSignificantChar();
if (ch == '"' || ch == '\'') {
return readCellString(ch);
}
unsigned int cellCount = 0;
unsigned int cellSize = 0;
unsigned int cellPos = lastCharPos();
while (true) {
if (ch == 0) {
if (isSizeCheckOKEndOfLine(cellSize)) {
createCell(cellSize, cellPos);
}
return GLECSVDataStatusEOF;
} else if (isEol(ch)) {
if (isSizeCheckOKEndOfLine(cellSize)) {
createCell(cellSize, cellPos);
}
return readNewline(ch);
} else if (isDelim(ch)) {
m_lastDelimWasSpace = isSpace(ch);
if (isSizeCheckOKAtDelim(ch, cellSize)) {
createCell(cellSize, cellPos);
}
return skipSpacesAndFirstDelim(ch);
} else if (isComment(ch)) {
if (isSizeCheckOKEndOfLine(cellSize)) {
createCell(cellSize, cellPos);
}
return skipTillEol();
}
cellCount++;
if (!isSpace(ch)) {
cellSize = cellCount;
}
ch = readChar();
}
return GLECSVDataStatusOK;
}
Token DefaultLexer::readToken() {
char c = lookChar();
while (true) {
// skip whitespace
while (isWhiteSpace(c)) {
skipChar();
c = lookChar();
}
// newlines
if (isNewline(c)) {
readNewline(c);
if (interactive_ && getCurrentBraceNesting() == 0)
return Token(TK_Newline);
c = lookChar();
continue;
}
// treat line comments as newlines
if (c == '/' && lookChar(1) == '/') {
readLineComment();
c = lookChar();
continue;
}
break;
}
SourceLocation sloc = getCurrentLocation();
// punctuation
if (c == '(') {
skipChar();
signalOpenBrace(TK_LParen);
return Token(TK_LParen, "(", sloc);
}
if (c == ')') {
skipChar();
signalCloseBrace(TK_LParen);
return Token(TK_RParen, ")", sloc);
}
if (c == '{') {
skipChar();
signalOpenBrace(TK_LCurlyBrace);
return Token(TK_LCurlyBrace, "{", sloc);
}
if (c == '}') {
skipChar();
signalCloseBrace(TK_LCurlyBrace);
return Token(TK_RCurlyBrace, "}", sloc);
}
if (c == '[') {
skipChar();
signalOpenBrace(TK_LSquareBrace);
return Token(TK_LSquareBrace, "[", sloc);
}
if (c == ']') {
skipChar();
signalCloseBrace(TK_LSquareBrace);
return Token(TK_RSquareBrace, "]", sloc);
}
if (c == ',') {
skipChar();
return Token(TK_Comma, ",", sloc);
}
if (c == ';') {
skipChar();
return Token(TK_Semicolon, ";", sloc);
}
if (c == ':' && !isOperatorChar(lookChar(1))) {
skipChar();
return Token(TK_Colon, ":", sloc);
}
if (c == '.') {
skipChar();
return Token(TK_Period, ".", sloc);
}
// identifiers
if (isLetter(c)) {
readIdentifier(c);
StringRef str = copyStr(finishToken());
unsigned keyid = lookupKeyword(str.c_str());
if (keyid) {
return Token(keyid, str, sloc);
}
return Token(TK_Identifier, str, sloc);
}
// generic operators
if (isOperatorChar(c)) {
readOperator(c);
StringRef str = copyStr(finishToken());
unsigned keyid = lookupKeyword(str.c_str());
if (keyid) {
return Token(keyid, str, sloc);
}
return Token(TK_Operator, str, sloc);
}
// numbers
if (isDigit(c)) {
readInteger(c);
StringRef str = copyStr(finishToken());
return Token(TK_LitInteger, str, sloc);
}
// characters
if (c == '\'') {
if (!readCharacter())
return Token(TK_Error);
StringRef str = copyStr(finishToken());
return Token(TK_LitCharacter, str, sloc);
}
// strings
if (c == '\"') {
if (!readString())
return Token(TK_Error);
StringRef str = copyStr(finishToken());
return Token(TK_LitString, str, sloc);
}
// if we're out of buffer, put in an EOF token.
if (c == 0 || stream_eof()) {
return Token(TK_EOF, "", sloc);
}
// Can't get the next token -- signal an error and bail.
signalLexicalError();
return Token(TK_Error, "", sloc);
}
