Token* readNumber(void) {
int count = 0;
Token* token = makeToken(TK_NUMBER, lineNo, colNo);
while (charCodes[currentChar] == CHAR_DIGIT) {
if (count > 9) {
error(ERR_NUMBERTOOLONG, token->lineNo, token->colNo);
}
// Add current character to the number
token->string[count] = currentChar;
// Increase string index
count++;
// Read next character
readChar();
}
// End string
token->string[count] = '\0';
// Convert current number to string
token->value = atoi(token->string);
return token;
}
Token* readConstChar(void) {
// TODO
Token *token = makeToken(TK_CHAR, lineNo, colNo);
readChar();
if (currentChar == EOF) {
token->tokenType = TK_NONE;
error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo);
return token;
}
token->string[0] = currentChar;
token->string[1] = '\0';
token->value = currentChar;
readChar();
if (currentChar == EOF) {
token->tokenType = TK_NONE;
error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo);
return token;
}
if (charCodes[currentChar] == CHAR_SINGLEQUOTE) {
readChar();
return token;
} else {
token->tokenType = TK_NONE;
error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo);
return token;
}
}
Token* readIdentKeyword(void) {
int count = 0;
Token* token = makeToken(TK_IDENT, lineNo, colNo);
while (charCodes[currentChar] == CHAR_LETTER || charCodes[currentChar] == CHAR_DIGIT) {
// Add current character to identifier
token->string[count] = currentChar;
// Increase identifier length
count++;
// Get next character
readChar();
}
// End string
token->string[count] = '\0';
// Limit identifier length
if (count > MAX_IDENT_LEN) {
// Announce error
error(ERR_IDENTTOOLONG, lineNo, colNo - count);
} else {
// If this identifier is a keyword
TokenType type = checkKeyword(token->string);
// Otherwise
if (type != TK_NONE) {
token->tokenType = type;
}
}
return token;
}
Token* readIdentKeyword(void) {
// TODO
Token *token = makeToken(TK_NONE, lineNo, colNo);
int count = 1;
token->string[0] = toupper((char)currentChar);
readChar();
while ((currentChar != EOF) &&
((charCodes[currentChar] == CHAR_LETTER) || (charCodes[currentChar] == CHAR_DIGIT))){
if (count <= MAX_IDENT_LEN)
token->string[count++] = toupper((char)currentChar);
readChar();
}
if (count > MAX_IDENT_LEN)
{
error(ERR_IDENTTOOLONG, token->lineNo, token->colNo);
}
token->string[count] = '\0';
token->tokenType = checkKeyword(token->string);
if (token->tokenType == TK_NONE)
{
token->tokenType = TK_IDENT;
}
return token;
}
void Scanner::handleEOFState()
{
loc_ = getTokenLocation();
makeToken(TokenType::END_OF_FILE, TokenValue::UNRESERVED,
loc_, std::string("END_OF_FILE"), -1);
// close the file
input_.close();
}
TokenT *_octal(TokenizerT *tk) {
nextChar(tk);
if(isOctal(tk->inputIter[0])) {
return _octal(tk);
} else {
return makeToken(tk, "octal integer");
}
}
TokenT *_double_quote(TokenizerT *tk) {
int atEndOfFile = nextChar(tk);
while(tk->inputIter[0] != '"') {
if(tk->inputIter[0] == '\\') {
if(atEndOfFile) {
return makeToken(tk, "unended string literal");
}
atEndOfFile = nextChar(tk);
}
if(atEndOfFile) {
return makeToken(tk, "unended string literal");
}
atEndOfFile = nextChar(tk);
}
nextChar(tk);
return makeToken(tk, "string literal");
}
static int makeQuotedToken(EcCompiler *cp, EcToken *tp, int c)
{
EcStream *stream;
int quoteType;
stream = cp->stream;
quoteType = c;
for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) {
if (c == 0) {
return makeToken(tp, 0, T_ERR, 0);
}
if (c == '\\') {
c = getNextChar(stream);
switch (c) {
// TBD -- others
case '\\':
break;
case '\'':
case '\"':
break;
case 'b':
c = '\b';
break;
case 'f':
c = '\f';
break;
case 'n':
c = '\n';
break;
case 'r':
c = '\r';
break;
case 't':
c = '\t';
break;
case 'u':
c = decodeNumber(cp, 16, 4);
break;
case 'x':
c = decodeNumber(cp, 16, 2);
break;
case 'v':
c = '\v';
break;
case '0':
c = decodeNumber(cp, 8, 3);
break;
default:
break;
}
}
addCharToToken(tp, c);
}
assert(tp->text);
setTokenID(tp, T_STRING, -1, 0);
return finalizeToken(tp);
}
Token* readConstChar(void) {
Token*tmp;
tmp=makeToken(TK_CHAR,lineNo,colNo);
tmp->string[0]=(char)currentChar;
tmp->string[1]='\0';
readChar();
return tmp;
// TODO
}
/*
* C, C++ and doc style comments. Return token or zero for no token.
*/
static int getComment(EcInput *input, EcToken *tp, int c)
{
EcStream *stream;
int form, startLine;
startLine = tp->stream->lineNumber;
stream = input->stream;
form = c;
// TODO - would be great to warn about comment nested in a comment.
for (form = c; c > 0;) {
c = getNextChar(stream);
if (c <= 0) {
/*
* Unterminated Comment
*/
addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine);
makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0);
return 1;
}
if (form == '/') {
if (c == '\n') {
break;
}
} else {
if (c == '*') {
c = getNextChar(stream);
if (c == '/') {
break;
}
addCharToToken(tp, '*');
putBackChar(stream, c);
} else if (c == '/') {
c = getNextChar(stream);
if (c == '*') {
/*
* Nested comment
*/
if (input->compiler->warnLevel > 0) {
ecReportError(input->compiler, "warning", stream->name, stream->lineNumber, 0,
stream->column, "Possible nested comment");
}
}
addCharToToken(tp, '/');
}
}
addCharToToken(tp, c);
}
return 0;
}
Token* readConstChar(void) {
Token* token = makeToken(TK_CHAR, lineNo, colNo);
// Read next character
readChar();
if (currentChar == -1) { // End of File
error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo);
} else {
switch(charCodes[currentChar]) {
// Escape character for Single Quote:
case CHAR_SINGLEQUOTE:
// Read next character
readChar();
if (charCodes[currentChar] == CHAR_SINGLEQUOTE) {
token->string[0] = currentChar;
readChar();
if (charCodes[currentChar] == CHAR_SINGLEQUOTE) {
token->string[1] = '\0';
readChar();
return token;
} else {
error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo);
}
} else {
error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo);
}
break;
default:
// Add the character to token string
token->string[0] = currentChar;
// Read next character
readChar();
switch(charCodes[currentChar]) {
case CHAR_SINGLEQUOTE:
// End token
token->string[1] = '\0';
readChar();
return token;
default:
error(ERR_INVALIDCHARCONSTANT, token->lineNo, token->colNo);
break;
}
break;
}
}
return token;
}
void Scanner::handleStringState()
{
loc_ = getTokenLocation();
// eat ' and NOT update currentChar_
// because we don't want ' (single quote).
getNextChar();
while (true)
{
if (currentChar_ == '\'')
{
// '''' condition
// see pascal standard section 6.1.7
if (peekChar() == '\'')
{
getNextChar();
}
// otherwise, we have handle string literal completely.
else
{
break;
}
}
addToBuffer(currentChar_);
getNextChar();
}
// eat end ' and update currentChar_ .
getNextChar();
// just one char
if (buffer_.length() == 1)
{
makeToken(TokenType::CHAR, TokenValue::UNRESERVED, loc_,
static_cast<long>(buffer_.at(0)), buffer_);
}
else
{
makeToken(TokenType::STRING_LITERAL, TokenValue::UNRESERVED,
loc_, buffer_, -1);
}
}
Token* readConstChar(void)
{
Token * token;
int beginColNo = colNo;
int beginLineNo = lineNo;
char string[MAX_IDENT_LEN + 1];
readChar();
string[0] = currentChar;
string[1] = '\0';
if (charCodes[currentChar] == CHAR_SINGLEQUOTE)
{
readChar();
if (charCodes[currentChar] != CHAR_SINGLEQUOTE)
{
token = makeToken(TK_NONE, lineNo, colNo);
error(ERR_INVALIDCHARCONSTANT, beginColNo, beginLineNo);
return token;
}
}
readChar();
switch (charCodes[currentChar])
{
case CHAR_SINGLEQUOTE:
token = makeToken(TK_CHAR, beginLineNo, beginColNo);
strcpy(token->string, string);
readChar();
return token;
default:
if (CATCH_CONSTCHAR_ERR)
{
warning(ERR_INVALIDCHARCONSTANT, beginColNo, beginLineNo);
token = makeToken(TK_CHAR, beginLineNo, beginColNo);
strcpy(token->string, string);
return token;
}
token = makeToken(TK_NONE, lineNo, colNo);
error(ERR_INVALIDCHARCONSTANT, beginColNo, beginLineNo);
return token;
}
}
TokenT *_hex(TokenizerT *tk, int isFirst) {
nextChar(tk);
if(isxdigit(tk->inputIter[0])) {
return _hex(tk, 0);
} else {
if(isFirst) {
return _invalid(tk);
} else {
return makeToken(tk, "hexadecimal integer");
}
}
}
TokenT *_decimal(TokenizerT *tk) {
nextChar(tk);
if(isdigit(tk->inputIter[0])) {
return _decimal(tk);
} else if(tk->inputIter[0] == '.') {
return _float(tk, 1);
} else if(tk->inputIter[0] == 'e' || tk->inputIter[0] == 'E') {
return _expofloat(tk, 1, 0);
} else {
return makeToken(tk, "decimal integer");
}
}
/*
* Handle being given a zero as the first char in a new token.
*/
TokenT *_zero(TokenizerT *tk) {
nextChar(tk);
if(isOctal(tk->inputIter[0])) {
return _octal(tk);
} else if(tk->inputIter[0] == 'x' || (tk->inputIter[0]) == 'X') {
return _hex(tk, 1);
} else if(tk->inputIter[0] == '.') {
return _float(tk, 1);
} else {
return makeToken(tk, "zero integer");
}
}
PUBLIC int ecGetRegExpToken(EcCompiler *cp, wchar *prefix)
{
EcToken *token, *tp;
EcStream *stream;
wchar *pp;
int c;
stream = cp->stream;
tp = token = cp->token;
assert(tp != 0);
initializeToken(tp, stream);
for (pp = prefix; pp && *pp; pp++) {
addCharToToken(tp, *pp);
}
while (1) {
c = getNextChar(stream);
switch (c) {
case -1:
return makeToken(tp, 0, T_ERR, 0);
case 0:
if (stream->flags & EC_STREAM_EOL) {
return makeToken(tp, 0, T_NOP, 0);
}
return makeToken(tp, 0, T_EOF, 0);
case '/':
addCharToToken(tp, '/');
while (1) {
c = getNextChar(stream);
if (c != 'g' && c != 'i' && c != 'm' && c != 'y' && c != 'x' && c != 'X' && c != 'U' && c != 's') {
putBackChar(stream, c);
break;
}
addCharToToken(tp, c);
}
return makeToken(tp, 0, T_REGEXP, 0);
case '\\':
c = getNextChar(stream);
if (c == '\r' || c == '\n' || c == 0) {
ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression");
return makeToken(tp, 0, T_ERR, 0);
}
addCharToToken(tp, '\\');
addCharToToken(tp, c);
break;
case '\r':
case '\n':
ecError(cp, "Warning", &stream->loc, "Illegal newline in regular expression");
return makeToken(tp, 0, T_ERR, 0);
default:
addCharToToken(tp, c);
}
}
}
void KnowledgeBaseUtils::addBreezeClause(KnowledgeBase &kb, size_t x, size_t y, size_t width, size_t height)
{
Clause clause;
vector<Token> hazardTokens;
// List all the cells that must have the hazard in them to imply the location of the wumpus
if (x > 0)
{
hazardTokens.push_back(makeToken("breeze", x - 1, y));
}
if (x < width - 1)
{
hazardTokens.push_back(makeToken("breeze", x + 1, y));
}
if (y > 0)
{
hazardTokens.push_back(makeToken("breeze", x, y - 1));
}
if (y < height - 1)
{
hazardTokens.push_back(makeToken("breeze", x, y + 1));
}
if (hazardTokens.size() > 0)
{
clause << hazardTokens[0];
// Add the rest of the tokens with the AND token in reverse polish order
for (size_t i = 1; i < hazardTokens.size(); i++)
{
clause << hazardTokens[i] << Token(AND);
}
// Add the implication part
clause << makeToken("pit", x, y);
clause << Token(IMPL);
kb << clause;
}
}
TokenT *_expofloat(TokenizerT *tk, int isFirst, int lastWasSign) {
nextChar(tk);
if(isdigit(tk->inputIter[0])) {
return _expofloat(tk, 0, 0);
} else if(tk->inputIter[0] == '+' || tk->inputIter[0] == '-') {
if(isFirst) {
return _expofloat(tk, 0, 1);
} else if(lastWasSign) {
return _invalid(tk);
} else {
return makeToken(tk, "float with exponent");
}
} else {
if(isFirst) {
return _invalid(tk);
} else if(lastWasSign) {
return _invalid(tk);
} else {
return makeToken(tk, "float with exponent");
}
}
}
void FMTLexer::mNONL(bool _createToken) {
int _ttype; antlr::RefToken _token; std::string::size_type _begin = text.length();
_ttype = NONL;
std::string::size_type _saveIndex;
match('$' /* charlit */ );
if ( _createToken && _token==antlr::nullToken && _ttype!=antlr::Token::SKIP ) {
_token = makeToken(_ttype);
_token->setText(text.substr(_begin, text.length()-_begin));
}
_returnToken = _token;
_saveIndex=0;
}
Token* readNumber(void) {
Token *token = makeToken(TK_NUMBER, lineNo, colNo);
int count = 0;
while ((currentChar != EOF) && (charCodes[currentChar] == CHAR_DIGIT)) {
token->string[count++] = (char)currentChar;
readChar();
}
token->string[count] = '\0';
token->value = atoi(token->string);
return token;
}
// TODO - handle triple quoting
static int getQuotedToken(EcInput *input, EcToken *tp, int c)
{
EcStream *stream;
int quoteType;
stream = input->stream;
quoteType = c;
for (c = getNextChar(stream); c && c != quoteType; c = getNextChar(stream)) {
if (c == 0) {
return makeToken(tp, 0, T_ERR, 0);
}
if (c == '\\') {
c = getNextChar(stream);
switch (c) {
// TBD -- others
case '\\':
break;
case '\'':
case '\"':
break;
case 'b':
c = '\b';
break;
case 'f':
c = '\f';
break;
case 'n':
c = '\n';
break;
case 'r':
c = '\r';
break;
case 't':
c = '\t';
break;
case 'u':
case 'x':
c = decodeNumber(input, 16, 4);
break;
case '0':
c = decodeNumber(input, 8, 3);
break;
default:
break;
}
}
addCharToToken(tp, c);
}
return finishToken(tp, T_STRING, -1, 0);
}
Tokens Bytecode::lexer(const std::string& source) const {
Strings lines;
split(source, '\n', lines);
Strings tokens_str;
for (int i = 0; i < lines.size(); i++) {
split(lines[i], ' ', tokens_str);
tokens_str.push_back("\n");
}
Tokens result;
for (int i = 0; i < tokens_str.size(); i++) {
result.push_back(makeToken(tokens_str[i]));
}
return result;
}
Token* readConstString(){
Token *token;
token = makeToken(TK_STRING, lineNo, colNo);
int i = 0;
while(charCodes[readChar()] != CHAR_DOUBLEQUOTE){
token->string[i++] = currentChar;
if(i == 16 || currentChar == '\n'){
error(ERR_INVALID_STRING, token->lineNo, token->colNo);
}
}
token->string[i] = '\0';
readChar();
return token;
}
temp<Token> Lexer::readNumber()
{
// TODO(bob): Handle EOF.
while (isDigit(peek())) advance();
// Read the fractional part, if any.
if (peek() == '.')
{
advance();
while (isDigit(peek())) advance();
}
return makeToken(TOKEN_NUMBER);
}
TokenT *_float(TokenizerT *tk, int isFirst) {
nextChar(tk);
if(isdigit(tk->inputIter[0])) {
return _float(tk, 0);
} else if(tk->inputIter[0] == 'e' || tk->inputIter[0] == 'E') {
return _expofloat(tk, 1, 0);
} else {
if(isFirst) {
return _invalid(tk);
} else {
return makeToken(tk, "float");
}
}
}
/*
C, C++ and doc style comments. Return token or zero for no token.
*/
static int getComment(EcCompiler *cp, EcToken *tp, int c)
{
EcStream *stream;
int form, startLine;
startLine = cp->stream->loc.lineNumber;
stream = cp->stream;
form = c;
for (form = c; c > 0;) {
c = getNextChar(stream);
if (c <= 0) {
/*
Unterminated Comment
*/
addFormattedStringToToken(tp, "Unterminated comment starting on line %d", startLine);
makeToken(tp, 0, form == '/' ? T_EOF: T_ERR, 0);
return 1;
}
if (form == '/') {
if (c == '\n' || c == '\r') {
break;
}
} else {
if (c == '*') {
c = getNextChar(stream);
if (c == '/') {
break;
}
addCharToToken(tp, '*');
putBackChar(stream, c);
} else if (c == '/') {
c = getNextChar(stream);
if (c == '*') {
/*
Nested comment
*/
if (cp->warnLevel > 0) {
ecError(cp, "Warning", &stream->loc, "Possible nested comment");
}
}
addCharToToken(tp, '/');
}
}
addCharToToken(tp, c);
}
return 0;
}
Token* readNumber(void) {
int i = 0;
Token *token = makeToken(TK_NUMBER, lineNo, colNo);
char *str = token->string;
while (i < MAX_IDENT_LEN && charCodes[currentChar] == CHAR_DIGIT) {
str[i++] = currentChar;
readChar();
}
str[i] = '\0';
while (charCodes[currentChar] == CHAR_DIGIT)
readChar();
return token;
// TODO
}
void FMTLexer::mCHAR(bool _createToken) {
int _ttype; antlr::RefToken _token; std::string::size_type _begin = text.length();
_ttype = CHAR;
std::string::size_type _saveIndex;
{
matchRange('\3',static_cast<unsigned char>('\377'));
}
if ( _createToken && _token==antlr::nullToken && _ttype!=antlr::Token::SKIP ) {
_token = makeToken(_ttype);
_token->setText(text.substr(_begin, text.length()-_begin));
}
_returnToken = _token;
_saveIndex=0;
}
Token *readFloat (void){
Token *token = makeToken(TK_NUMBER, lineNo, colNo);
int count = 0;
while ((currentChar != EOF) && ((charCodes[currentChar] == CHAR_DIGIT) || (charCodes[currentChar] == CHAR_PERIOD))) {
if((charCodes[currentChar] == CHAR_PERIOD)&&(count==0)) token->string[count++]='0';
token->string[count++] = (char)currentChar;
if((charCodes[currentChar]== CHAR_PERIOD)&& (count>0)) token->string[count++]='0';
readChar();
}
token->string[count] = '\0';
token->value = atoi(token->string);
return token;
}