void recognizeIdentifierToken(Lexer *self) {
consumeCharacter(self);
while (isLetterOrDigit(self->currentChar)) {
consumeCharacter(self);
}
while (isUnderscore(self->currentChar) && isLetterOrDigit(peekAhead(self, 1))) {
consumeCharacter(self);
while (isLetterOrDigit(self->currentChar)) {
consumeCharacter(self);
}
}
pushToken(self, TOKEN_IDENTIFIER);
}
void drawWrappedText(RenderingContext* rc, SkCanvas* cv, TextDrawInfo* text, float textSize, SkPaint& paintText) {
if(text->textWrap == 0) {
// set maximum for all text
text->textWrap = 40;
}
if(text->text.length() > text->textWrap) {
int start = 0;
int end = text->text.length();
int lastSpace = -1;
int line = 0;
int pos = 0;
int limit = 0;
while(pos < end) {
lastSpace = -1;
limit += text->textWrap;
// in UTF-8 all non ASCII characters has 2 or more characters
int symbolsRead = 0;
int utf8pos = pos;
while(symbolsRead < limit && pos < end) {
if(utf8pos == pos) {
if(text->text.at(pos) <= 128) {
symbolsRead++;
if(!isLetterOrDigit(text->text.at(pos))) {
lastSpace = pos;
}
utf8pos ++;
}
} else {
// here could be code to determine if UTF-8 is ended (currently only 2 chars)
symbolsRead++;
utf8pos = pos + 1;
}
pos++;
}
if(lastSpace == -1) {
PROFILE_NATIVE_OPERATION(rc, drawTextOnCanvas(cv, text->text.substr(start, pos), text->centerX, text->centerY + line * (textSize + 2), paintText, text->textShadow));
start = pos;
} else {
PROFILE_NATIVE_OPERATION(rc, drawTextOnCanvas(cv, text->text.substr(start, lastSpace), text->centerX, text->centerY + line * (textSize + 2), paintText, text->textShadow));
start = lastSpace + 1;
limit += (start - pos) - 1;
}
line++;
}
} else {
PROFILE_NATIVE_OPERATION(rc, drawTextOnCanvas(cv, text->text, text->centerX, text->centerY, paintText, text->textShadow));
}
}
void getNextToken(Lexer *self) {
self->startPos = 0;
skipLayoutAndComments(self);
self->startPos = self->pos;
if (isEndOfInput(self->currentChar)) {
recognizeEndOfInputToken(self);
self->running = false; // stop lexing
return;
}
else if (isDigit(self->currentChar) || (self->currentChar == '.' && isDigit(peekAhead(self, 1)))) {
// number
recognizeNumberToken(self);
}
else if (isLetterOrDigit(self->currentChar) || self->currentChar == '_') {
// ident
recognizeIdentifierToken(self);
}
else if (isString(self->currentChar)) {
// string
recognizeStringToken(self);
}
else if (isCharacter(self->currentChar)) {
// character
recognizeCharacterToken(self);
}
else if (isOperator(self->currentChar)) {
// operator
recognizeOperatorToken(self);
}
else if (isEndOfLine(self->currentChar)) {
recognizeEndOfLineToken(self);
}
else if (isSeparator(self->currentChar)) {
// separator
recognizeSeparatorToken(self);
}
else {
// errorneous
recognizeErroneousToken(self);
}
}
bool Lexer::getNextToken(Tokener &tokener)
{
bool isComment = false;
bool loopMark = true;
while(loopMark)
{
peekNext();
if(endOfFile)
{
if(isComment)
{
cout << line << ',' << row << " 文件末尾出错：缺少'}\n'" << endl;
exit(1);
}
tokener.putToken(new Token(-1),line,row);
return false;
}
switch(peek)
{
case ' ':
case '\t':
case '\n':
break;
case '{':
if (isComment)
{
cout << line << ',' << row << " 错误的输入：嵌套的注释\n" << endl;
exit(1);
}
else
{
isComment = true;
}
break;
case '}':
if (isComment) {
isComment = false;
}
else
{
cout << line << ',' << row << " 错误的输入：嵌套的注释\n" << endl;
exit(1);
}
break;
default:
if (!isComment)
{
loopMark = false;
break;
}
}
}
// 2.识别复合词法符号 ":="
switch(peek)
{
case ':':
if (!checkNext('='))
{
tokener.putToken(new Symbol(':'), line, row);
return true;
}
else
{
tokener.putToken(&Operator::OP_ASSIGN, line, row - 1);
return true;
}
}
//识别各种符号
switch(peek)
{
case '+':
tokener.putToken(&Operator::OP_PLUS, line, row);
return true;
case '-':
tokener.putToken(&Operator::OP_MINUS, line, row);
return true;
case '*':
tokener.putToken(&Operator::OP_MUTL, line, row);
return true;
case '/':
tokener.putToken(&Operator::OP_DIV, line, row);
return true;
case '=':
tokener.putToken(&Operator::OP_EQUAL, line, row);
return true;
case '<':
tokener.putToken(&Operator::OP_LESS, line, row);
return true;
}
//读取整数常量
if(isDigit(peek))
{
int startRow = row;
int value = peek - '0';
while(true)
{
peekNext();
if(endOfFile)
{
break;
}
if(!isDigit(peek))
{
putBack(peek);
break;
}
value = value * 10 + peek - '0';
}
tokener.putToken(new Int(value),line,startRow);
return true;
}
//读取标识符
if(isLetter(peek))
{
int startrow = row;
string str;
str.append(string(&peek));
while(true)
{
peekNext();
if(endOfFile)
{
break;
}
if(!isLetterOrDigit(peek))
{
putBack(peek);
break;
}
str.append(string(&peek));
}
Word *word = Word::getReserve(str);
if( word == NULL)
{
word = new Word(Token::TAG_VARIABLE,str);
}
tokener.putToken((Word*)word,line,startrow);
return true;
}
//剩余的情况
tokener.putToken(new Token(peek),line,row);
return true;
}