void recognizeIdentifierToken(Lexer *self) { consumeCharacter(self); while (isLetterOrDigit(self->currentChar)) { consumeCharacter(self); } while (isUnderscore(self->currentChar) && isLetterOrDigit(peekAhead(self, 1))) { consumeCharacter(self); while (isLetterOrDigit(self->currentChar)) { consumeCharacter(self); } } pushToken(self, TOKEN_IDENTIFIER); }
void drawWrappedText(RenderingContext* rc, SkCanvas* cv, TextDrawInfo* text, float textSize, SkPaint& paintText) { if(text->textWrap == 0) { // set maximum for all text text->textWrap = 40; } if(text->text.length() > text->textWrap) { int start = 0; int end = text->text.length(); int lastSpace = -1; int line = 0; int pos = 0; int limit = 0; while(pos < end) { lastSpace = -1; limit += text->textWrap; // in UTF-8 all non ASCII characters has 2 or more characters int symbolsRead = 0; int utf8pos = pos; while(symbolsRead < limit && pos < end) { if(utf8pos == pos) { if(text->text.at(pos) <= 128) { symbolsRead++; if(!isLetterOrDigit(text->text.at(pos))) { lastSpace = pos; } utf8pos ++; } } else { // here could be code to determine if UTF-8 is ended (currently only 2 chars) symbolsRead++; utf8pos = pos + 1; } pos++; } if(lastSpace == -1) { PROFILE_NATIVE_OPERATION(rc, drawTextOnCanvas(cv, text->text.substr(start, pos), text->centerX, text->centerY + line * (textSize + 2), paintText, text->textShadow)); start = pos; } else { PROFILE_NATIVE_OPERATION(rc, drawTextOnCanvas(cv, text->text.substr(start, lastSpace), text->centerX, text->centerY + line * (textSize + 2), paintText, text->textShadow)); start = lastSpace + 1; limit += (start - pos) - 1; } line++; } } else { PROFILE_NATIVE_OPERATION(rc, drawTextOnCanvas(cv, text->text, text->centerX, text->centerY, paintText, text->textShadow)); } }
void getNextToken(Lexer *self) { self->startPos = 0; skipLayoutAndComments(self); self->startPos = self->pos; if (isEndOfInput(self->currentChar)) { recognizeEndOfInputToken(self); self->running = false; // stop lexing return; } else if (isDigit(self->currentChar) || (self->currentChar == '.' && isDigit(peekAhead(self, 1)))) { // number recognizeNumberToken(self); } else if (isLetterOrDigit(self->currentChar) || self->currentChar == '_') { // ident recognizeIdentifierToken(self); } else if (isString(self->currentChar)) { // string recognizeStringToken(self); } else if (isCharacter(self->currentChar)) { // character recognizeCharacterToken(self); } else if (isOperator(self->currentChar)) { // operator recognizeOperatorToken(self); } else if (isEndOfLine(self->currentChar)) { recognizeEndOfLineToken(self); } else if (isSeparator(self->currentChar)) { // separator recognizeSeparatorToken(self); } else { // errorneous recognizeErroneousToken(self); } }
bool Lexer::getNextToken(Tokener &tokener) { bool isComment = false; bool loopMark = true; while(loopMark) { peekNext(); if(endOfFile) { if(isComment) { cout << line << ',' << row << " 文件末尾出错:缺少'}\n'" << endl; exit(1); } tokener.putToken(new Token(-1),line,row); return false; } switch(peek) { case ' ': case '\t': case '\n': break; case '{': if (isComment) { cout << line << ',' << row << " 错误的输入:嵌套的注释\n" << endl; exit(1); } else { isComment = true; } break; case '}': if (isComment) { isComment = false; } else { cout << line << ',' << row << " 错误的输入:嵌套的注释\n" << endl; exit(1); } break; default: if (!isComment) { loopMark = false; break; } } } // 2.识别复合词法符号 ":=" switch(peek) { case ':': if (!checkNext('=')) { tokener.putToken(new Symbol(':'), line, row); return true; } else { tokener.putToken(&Operator::OP_ASSIGN, line, row - 1); return true; } } //识别各种符号 switch(peek) { case '+': tokener.putToken(&Operator::OP_PLUS, line, row); return true; case '-': tokener.putToken(&Operator::OP_MINUS, line, row); return true; case '*': tokener.putToken(&Operator::OP_MUTL, line, row); return true; case '/': tokener.putToken(&Operator::OP_DIV, line, row); return true; case '=': tokener.putToken(&Operator::OP_EQUAL, line, row); return true; case '<': tokener.putToken(&Operator::OP_LESS, line, row); return true; } //读取整数常量 if(isDigit(peek)) { int startRow = row; int value = peek - '0'; while(true) { peekNext(); if(endOfFile) { break; } if(!isDigit(peek)) { putBack(peek); break; } value = value * 10 + peek - '0'; } tokener.putToken(new Int(value),line,startRow); return true; } //读取标识符 if(isLetter(peek)) { int startrow = row; string str; str.append(string(&peek)); while(true) { peekNext(); if(endOfFile) { break; } if(!isLetterOrDigit(peek)) { putBack(peek); break; } str.append(string(&peek)); } Word *word = Word::getReserve(str); if( word == NULL) { word = new Word(Token::TAG_VARIABLE,str); } tokener.putToken((Word*)word,line,startrow); return true; } //剩余的情况 tokener.putToken(new Token(peek),line,row); return true; }