bool CssTokenizer::readString() { if (!lastReadEq('"') && !lastReadEq('\'')) return false; char delim = lastRead; currentToken.add(lastRead); readChar(); while (in != NULL) { if (lastReadEq(delim)) { currentToken.add(lastRead); readChar(); return true; } else if (lastReadEq('\n') || lastReadEq('\r') || lastReadEq('\f')) { throw new ParseException("end of line", "end of string"); } else if (lastReadEq('\\')) // note that even though readEscape() returns false it still // eats the '\'. readEscape() || readNewline(); else { currentToken.add(lastRead); readChar(); } } throw new ParseException("end of input", "end of string"); return false; }
GLECSVDataStatus GLECSVData::readCellString(GLEBYTE quote) { unsigned int cellSize = 1; unsigned int cellPos = lastCharPos(); initWritePos(); while (true) { GLEBYTE ch = readChar(); writeChar(ch); cellSize++; if (ch == 0) { m_error.errorCode = GLECSVErrorUnterminatedString; m_error.errorLine = m_lines; m_error.errorColumn = getUTF8Column(cellPos); createErrorString("unterminated string"); return GLECSVDataStatusEOF; } else if (isEol(ch)) { m_error.errorCode = GLECSVErrorUnterminatedString; m_error.errorLine = m_lines; m_error.errorColumn = getUTF8Column(cellPos); createErrorString("unterminated string"); return readNewline(ch); } else if (ch == quote) { GLEBYTE ch = readChar(); if (ch != quote) { writeChar(ch); createCell(cellSize, cellPos); return skipSpacesAndFirstDelim(ch); } } } return GLECSVDataStatusOK; }
void DefaultLexer::readLineComment() { skipChar(); // skip '/' skipChar(); // skip '/' char c = lookChar(); while (c && !isNewline(c)) { skipChar(); c = lookChar(); } if (isNewline(c)) readNewline(c); }
GLECSVDataStatus GLECSVData::skipTillEol() { while (true) { GLEBYTE ch = readChar(); if (ch == 0) { return GLECSVDataStatusEOF; } if (isEol(ch)) { return readNewline(ch); } } }
GLECSVDataStatus GLECSVData::skipSpacesAndFirstDelim(GLEBYTE ch) { while (true) { if (!isSpace(ch)) { if (ch == 0) { return GLECSVDataStatusEOF; } else if (isEol(ch)) { return readNewline(ch); } else if (isDelim(ch)) { m_lastDelimWasSpace = isSpace(ch); return GLECSVDataStatusOK; } else { goBack(); return GLECSVDataStatusOK; } } ch = readChar(); } return GLECSVDataStatusOK; }
GLECSVDataStatus GLECSVData::readCell() { GLEBYTE ch = readSignificantChar(); if (ch == '"' || ch == '\'') { return readCellString(ch); } unsigned int cellCount = 0; unsigned int cellSize = 0; unsigned int cellPos = lastCharPos(); while (true) { if (ch == 0) { if (isSizeCheckOKEndOfLine(cellSize)) { createCell(cellSize, cellPos); } return GLECSVDataStatusEOF; } else if (isEol(ch)) { if (isSizeCheckOKEndOfLine(cellSize)) { createCell(cellSize, cellPos); } return readNewline(ch); } else if (isDelim(ch)) { m_lastDelimWasSpace = isSpace(ch); if (isSizeCheckOKAtDelim(ch, cellSize)) { createCell(cellSize, cellPos); } return skipSpacesAndFirstDelim(ch); } else if (isComment(ch)) { if (isSizeCheckOKEndOfLine(cellSize)) { createCell(cellSize, cellPos); } return skipTillEol(); } cellCount++; if (!isSpace(ch)) { cellSize = cellCount; } ch = readChar(); } return GLECSVDataStatusOK; }
Token DefaultLexer::readToken() { char c = lookChar(); while (true) { // skip whitespace while (isWhiteSpace(c)) { skipChar(); c = lookChar(); } // newlines if (isNewline(c)) { readNewline(c); if (interactive_ && getCurrentBraceNesting() == 0) return Token(TK_Newline); c = lookChar(); continue; } // treat line comments as newlines if (c == '/' && lookChar(1) == '/') { readLineComment(); c = lookChar(); continue; } break; } SourceLocation sloc = getCurrentLocation(); // punctuation if (c == '(') { skipChar(); signalOpenBrace(TK_LParen); return Token(TK_LParen, "(", sloc); } if (c == ')') { skipChar(); signalCloseBrace(TK_LParen); return Token(TK_RParen, ")", sloc); } if (c == '{') { skipChar(); signalOpenBrace(TK_LCurlyBrace); return Token(TK_LCurlyBrace, "{", sloc); } if (c == '}') { skipChar(); signalCloseBrace(TK_LCurlyBrace); return Token(TK_RCurlyBrace, "}", sloc); } if (c == '[') { skipChar(); signalOpenBrace(TK_LSquareBrace); return Token(TK_LSquareBrace, "[", sloc); } if (c == ']') { skipChar(); signalCloseBrace(TK_LSquareBrace); return Token(TK_RSquareBrace, "]", sloc); } if (c == ',') { skipChar(); return Token(TK_Comma, ",", sloc); } if (c == ';') { skipChar(); return Token(TK_Semicolon, ";", sloc); } if (c == ':' && !isOperatorChar(lookChar(1))) { skipChar(); return Token(TK_Colon, ":", sloc); } if (c == '.') { skipChar(); return Token(TK_Period, ".", sloc); } // identifiers if (isLetter(c)) { readIdentifier(c); StringRef str = copyStr(finishToken()); unsigned keyid = lookupKeyword(str.c_str()); if (keyid) { return Token(keyid, str, sloc); } return Token(TK_Identifier, str, sloc); } // generic operators if (isOperatorChar(c)) { readOperator(c); StringRef str = copyStr(finishToken()); unsigned keyid = lookupKeyword(str.c_str()); if (keyid) { return Token(keyid, str, sloc); } return Token(TK_Operator, str, sloc); } // numbers if (isDigit(c)) { readInteger(c); StringRef str = copyStr(finishToken()); return Token(TK_LitInteger, str, sloc); } // characters if (c == '\'') { if (!readCharacter()) return Token(TK_Error); StringRef str = copyStr(finishToken()); return Token(TK_LitCharacter, str, sloc); } // strings if (c == '\"') { if (!readString()) return Token(TK_Error); StringRef str = copyStr(finishToken()); return Token(TK_LitString, str, sloc); } // if we're out of buffer, put in an EOF token. if (c == 0 || stream_eof()) { return Token(TK_EOF, "", sloc); } // Can't get the next token -- signal an error and bail. signalLexicalError(); return Token(TK_Error, "", sloc); }