void DefaultLexer::readHexInteger() {
char c = lookChar();
while (isDigit(c) || isHexDigit(c)) {
putChar(c);
skipChar();
c = lookChar();
}
}
bool DefaultLexer::readCharacter() {
skipChar(); // skip leading '''
char c = lookChar();
while (c != '\'') {
if (!readEscapeCharacter(c))
return false;
c = lookChar();
}
skipChar(); // skip trailing '''
return true;
}
void DefaultLexer::readLineComment() {
skipChar(); // skip '/'
skipChar(); // skip '/'
char c = lookChar();
while (c && !isNewline(c)) {
skipChar();
c = lookChar();
}
if (isNewline(c)) readNewline(c);
}
void DefaultLexer::readIdentifier(char startChar) {
char c = startChar;
putChar(c);
skipChar();
c = lookChar();
while (isLetter(c) | isDigit(c)) {
putChar(c);
skipChar();
c = lookChar();
};
}
void DefaultLexer::readNewline(char c) {
if (c == '\n') {
skipChar();
if (lookChar() == '\r') skipChar();
}
if (c == '\r') {
skipChar();
if (lookChar() == '\n') skipChar();
}
signalNewline();
}
void Lexer::signalLexicalError() {
char c = lookChar();
std::cerr << "Lexical error: unknown character ";
if (c) std::cerr << "'" << c << "'";
else std::cerr << 0;
lexical_error = true;
}
void DefaultLexer::readOperator(char startChar) {
char c = startChar;
do {
putChar(c);
skipChar();
c = lookChar();
} while (isOperatorChar(c));
}
void DefaultLexer::readInteger(char startChar) {
char c = startChar;
do {
putChar(c);
skipChar();
c = lookChar();
} while (isDigit(c));
}
bool DefaultLexer::readFloatExp(char startChar) {
putChar(startChar); // put 'e' or 'E'
skipChar();
char c = lookChar();
if (c == '+' || c == '-') {
putChar(c);
skipChar();
} else
return false;
c = lookChar();
if (isDigit(c)) {
readInteger(c);
return true;
}
return false;
}
bool DefaultLexer::readEscapeCharacter(char c) {
// translate string contents on the fly.
if (c == 0) {
signalLexicalError();
return false;
}
if (c == '\n' || c == '\r' || c == '\t') {
signalLexicalError();
return false;
}
if (c == '\\') {
skipChar();
c = lookChar();
switch (c) {
case 0:
signalLexicalError();
return false;
case 'n':
putChar('\n');
break;
case 'r':
putChar('\r');
break;
case 't':
putChar('\t');
break;
default:
putChar(c);
break;
}
}
else {
putChar(c);
}
skipChar();
return true;
}
Token DefaultLexer::readToken() {
char c = lookChar();
while (true) {
// skip whitespace
while (isWhiteSpace(c)) {
skipChar();
c = lookChar();
}
// newlines
if (isNewline(c)) {
readNewline(c);
if (interactive_ && getCurrentBraceNesting() == 0)
return Token(TK_Newline);
c = lookChar();
continue;
}
// treat line comments as newlines
if (c == '/' && lookChar(1) == '/') {
readLineComment();
c = lookChar();
continue;
}
break;
}
SourceLocation sloc = getCurrentLocation();
// punctuation
if (c == '(') {
skipChar();
signalOpenBrace(TK_LParen);
return Token(TK_LParen, "(", sloc);
}
if (c == ')') {
skipChar();
signalCloseBrace(TK_LParen);
return Token(TK_RParen, ")", sloc);
}
if (c == '{') {
skipChar();
signalOpenBrace(TK_LCurlyBrace);
return Token(TK_LCurlyBrace, "{", sloc);
}
if (c == '}') {
skipChar();
signalCloseBrace(TK_LCurlyBrace);
return Token(TK_RCurlyBrace, "}", sloc);
}
if (c == '[') {
skipChar();
signalOpenBrace(TK_LSquareBrace);
return Token(TK_LSquareBrace, "[", sloc);
}
if (c == ']') {
skipChar();
signalCloseBrace(TK_LSquareBrace);
return Token(TK_RSquareBrace, "]", sloc);
}
if (c == ',') {
skipChar();
return Token(TK_Comma, ",", sloc);
}
if (c == ';') {
skipChar();
return Token(TK_Semicolon, ";", sloc);
}
if (c == ':' && !isOperatorChar(lookChar(1))) {
skipChar();
return Token(TK_Colon, ":", sloc);
}
if (c == '.') {
skipChar();
return Token(TK_Period, ".", sloc);
}
// identifiers
if (isLetter(c)) {
readIdentifier(c);
StringRef str = copyStr(finishToken());
unsigned keyid = lookupKeyword(str.c_str());
if (keyid) {
return Token(keyid, str, sloc);
}
return Token(TK_Identifier, str, sloc);
}
// generic operators
if (isOperatorChar(c)) {
readOperator(c);
StringRef str = copyStr(finishToken());
unsigned keyid = lookupKeyword(str.c_str());
if (keyid) {
return Token(keyid, str, sloc);
}
return Token(TK_Operator, str, sloc);
}
// numbers
if (isDigit(c)) {
readInteger(c);
StringRef str = copyStr(finishToken());
return Token(TK_LitInteger, str, sloc);
}
// characters
if (c == '\'') {
if (!readCharacter())
return Token(TK_Error);
StringRef str = copyStr(finishToken());
return Token(TK_LitCharacter, str, sloc);
}
// strings
if (c == '\"') {
if (!readString())
return Token(TK_Error);
StringRef str = copyStr(finishToken());
return Token(TK_LitString, str, sloc);
}
// if we're out of buffer, put in an EOF token.
if (c == 0 || stream_eof()) {
return Token(TK_EOF, "", sloc);
}
// Can't get the next token -- signal an error and bail.
signalLexicalError();
return Token(TK_Error, "", sloc);
}
GBool PSTokenizer::getToken(char *buf, int size, int *length) {
GBool comment, backslash;
int c;
int i;
// skip leading whitespace and comments
comment = gFalse;
while (1) {
if ((c = getChar()) == EOF) {
buf[0] = '\0';
*length = 0;
return gFalse;
}
if (comment) {
if (c == '\x0a' || c == '\x0d') {
comment = gFalse;
}
} else if (c == '%') {
comment = gTrue;
} else if (specialChars[c] != 1) {
break;
}
}
// Reserve room for terminating '\0'
size--;
// read a token
i = 0;
buf[i++] = c;
if (c == '(') {
backslash = gFalse;
while ((c = lookChar()) != EOF) {
consumeChar();
if (i < size) {
buf[i++] = c;
}
if (c == '\\') {
backslash = gTrue;
} else if (!backslash && c == ')') {
break;
} else {
backslash = gFalse;
}
}
} else if (c == '<') {
while ((c = lookChar()) != EOF) {
consumeChar();
if (i < size && specialChars[c] != 1) {
buf[i++] = c;
}
if (c == '>') {
break;
}
}
} else if (c != '[' && c != ']') {
while ((c = lookChar()) != EOF && !specialChars[c]) {
consumeChar();
if (i < size) {
buf[i++] = c;
}
}
}
// Zero terminate token string
buf[i] = '\0';
// Return length of token
*length = i;
return gTrue;
}
int JPXStream::getChar() {
int result = lookChar();
++counter;
return result;
}
