int ScriptDocLexer::GetNextTokenForParser() { int ret = -1; if (AdvanceOneToken( ret )) { return ret; } return -1; }
VError HTMLLexer::Tokenize( TokenList &outTokens, bool inContinuationOfComment, sLONG inOpenStringType ) { fCommentContinuationMode = inContinuationOfComment; int nToken = 0; while (AdvanceOneToken( nToken, &outTokens )) ; // Infinite loop! return VE_OK; }
int HTMLLexer::GetNextTokenForParser() { // If we peeked at a token, then just return the last one we got if (fPeeked) { int ret = fLastToken; fPeeked = false; // We're no longer in peek mode though! return ret; } // Otherwise, get the next token in the stream int token; if (AdvanceOneToken( token, NULL )) return token; return -1; }
bool ScriptDocLexer::AdvanceOneToken( int &outToken ) { // Clear out our last token's text and any tag values we've stored up fLastElement = NULL; // Start by skipping any whitespaces that we run into bool consumedWhitespace = false; if (!SkipWhitespaces( consumedWhitespace )) return false; // Now snag the next character from the stream so we can determine how // to handle it. if (!fLexerInput->HasMoreChars()) return false; UniChar ch = fLexerInput->PeekAtNextChar(); if (IsScriptDocStart( ch )) { ConsumeScriptDocStart(); outToken = ScriptDocTokenValues::START; return true; } // Now that we've handled the start and stop cases for the tag, we need to consume // some more data from the line. Every ScriptDoc line starts with at least one asterisk, // but possibly more. We will eat all of the asterisks we find. if (ch != CHAR_ASTERISK) return false; do { ch = fLexerInput->MoveToNextChar(); } while (ch == CHAR_ASTERISK); // If the very next character is a solidus, that means we've reached the end of the scriptdoc // comment if (ch == CHAR_SOLIDUS) { outToken = ScriptDocTokenValues::END; return true; } // Put the last token we grabbed during asterisk processing back onto the stream fLexerInput->MoveToPreviousChar(); // Now we can skip any whitespaces that remain if (!SkipWhitespaces( consumedWhitespace )) return false; // Having done that, we can continue to look for semantic information. ch = fLexerInput->HasMoreChars() ? fLexerInput->PeekAtNextChar() : -1; if (-1 == ch) { // This is a santity check for the end of input return false; } else if (IsLineEnding( ch )) { ConsumeLineEnding( fLexerInput->MoveToNextChar() ); // If we hit a line ending, then we just want to start this whole process over again return AdvanceOneToken( outToken ); } else if (IsScriptDocTag( ch )) { // We found a ScriptDoc tag, so let's eat it fFoundTags = true; if (!ConsumeScriptDocTag()) return false; outToken = ScriptDocTokenValues::ELEMENT; return true; } else { // The only thing left is the comment section. Each ScriptDoc can have one and only // one comment to it, and it is supposed to be found before any tags. It can span multiple // lines. if (fFoundComment || fFoundTags) return false; fFoundComment = true; VString text; if (!ConsumeComment( text )) return false; fLastElement = new CommentElement( text ); outToken = ScriptDocTokenValues::ELEMENT; return true; } return true; }