int ScriptDocLexer::GetNextTokenForParser()
{
int ret = -1;
if (AdvanceOneToken( ret )) {
return ret;
}
return -1;
}
VError HTMLLexer::Tokenize( TokenList &outTokens, bool inContinuationOfComment, sLONG inOpenStringType )
{
fCommentContinuationMode = inContinuationOfComment;
int nToken = 0;
while (AdvanceOneToken( nToken, &outTokens ))
; // Infinite loop!
return VE_OK;
}
int HTMLLexer::GetNextTokenForParser()
{
// If we peeked at a token, then just return the last one we got
if (fPeeked) {
int ret = fLastToken;
fPeeked = false; // We're no longer in peek mode though!
return ret;
}
// Otherwise, get the next token in the stream
int token;
if (AdvanceOneToken( token, NULL )) return token;
return -1;
}
bool ScriptDocLexer::AdvanceOneToken( int &outToken )
{
// Clear out our last token's text and any tag values we've stored up
fLastElement = NULL;
// Start by skipping any whitespaces that we run into
bool consumedWhitespace = false;
if (!SkipWhitespaces( consumedWhitespace )) return false;
// Now snag the next character from the stream so we can determine how
// to handle it.
if (!fLexerInput->HasMoreChars()) return false;
UniChar ch = fLexerInput->PeekAtNextChar();
if (IsScriptDocStart( ch )) {
ConsumeScriptDocStart();
outToken = ScriptDocTokenValues::START;
return true;
}
// Now that we've handled the start and stop cases for the tag, we need to consume
// some more data from the line. Every ScriptDoc line starts with at least one asterisk,
// but possibly more. We will eat all of the asterisks we find.
if (ch != CHAR_ASTERISK) return false;
do {
ch = fLexerInput->MoveToNextChar();
} while (ch == CHAR_ASTERISK);
// If the very next character is a solidus, that means we've reached the end of the scriptdoc
// comment
if (ch == CHAR_SOLIDUS) {
outToken = ScriptDocTokenValues::END;
return true;
}
// Put the last token we grabbed during asterisk processing back onto the stream
fLexerInput->MoveToPreviousChar();
// Now we can skip any whitespaces that remain
if (!SkipWhitespaces( consumedWhitespace )) return false;
// Having done that, we can continue to look for semantic information.
ch = fLexerInput->HasMoreChars() ? fLexerInput->PeekAtNextChar() : -1;
if (-1 == ch) {
// This is a santity check for the end of input
return false;
} else if (IsLineEnding( ch )) {
ConsumeLineEnding( fLexerInput->MoveToNextChar() );
// If we hit a line ending, then we just want to start this whole process over again
return AdvanceOneToken( outToken );
} else if (IsScriptDocTag( ch )) {
// We found a ScriptDoc tag, so let's eat it
fFoundTags = true;
if (!ConsumeScriptDocTag()) return false;
outToken = ScriptDocTokenValues::ELEMENT;
return true;
} else {
// The only thing left is the comment section. Each ScriptDoc can have one and only
// one comment to it, and it is supposed to be found before any tags. It can span multiple
// lines.
if (fFoundComment || fFoundTags) return false;
fFoundComment = true;
VString text;
if (!ConsumeComment( text )) return false;
fLastElement = new CommentElement( text );
outToken = ScriptDocTokenValues::ELEMENT;
return true;
}
return true;
}
