bool HTMLLexer::ConsumeMultiLineComment( TokenList *ioTokens, sLONG inType ) { // We have to override the default behavior for multiline comment consumption because we define // a bunch of "special" comment types for 4D. So we will let the default consumer do its job, but // we will look at the comment that was consumed afterwards to see if we have a special comment // type. If we do, we need only modify the token's type accordingly. bool ret = VLexerBase::ConsumeMultiLineComment( ioTokens, inType ); // Look at the last token in the list (it should be a comment or an open comment), and grab its text ILexerToken *token = ioTokens->back(); xbox_assert( token->GetType() == ILexerToken::TT_COMMENT || token->GetType() == ILexerToken::TT_OPEN_COMMENT ); // For right now, we only care about finished comments because it makes the logic easier. We don't have to // worry so much about handling the syntax highlighting aspects of things. VString tokenText = token->GetText(); if (token->GetType() == ILexerToken::TT_COMMENT && tokenText.BeginsWith( CVSTR( "<!--#4D" ) )) { // We probably have one of the special comment types. We're going to remove the old token and create a new // one with the appropriate information, but only after we've verified that this isn't just garbage! if (tokenText.BeginsWith( CVSTR( "<!--#4DVAR" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DHTMLVAR" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DINCLUDE" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DACTION" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DSCRIPT" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DMETHOD" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DIF" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DELSE" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DENDIF" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DLOOP" ) ) || tokenText.BeginsWith( CVSTR( "<!--#4DENDLOOP" ) )) { ioTokens->pop_back(); ioTokens->push_back( new HTMLLexerToken( ILexerToken::TT_SPECIAL_4D_COMMENT, token->GetPosition(), token->GetLength(), tokenText, token->GetValue() ) ); } } return ret; }
bool ScriptDocLexer::ConsumeComment( VString &outComment ) { // We found the start of a comment, so we want to consume it in its entirety. The // only tricky part to comments is that they can span multiple lines, but we don't // want to include the starting asterisk as part of the comment itself. UniChar szUCharBuffer [ 256 ]; VIndex nBufferSize = 0; while (true) { // If we're out of characters, we need to bail out if (!fLexerInput->HasMoreChars()) return false; // Check the next character -- if it's a line ending, we need to handle it specially, // but if it's not, then we can add it to our comment buffer UniChar ch = fLexerInput->MoveToNextChar(); if (IsLineEnding( ch )) { ConsumeLineEnding( ch ); // When we have a line ending, we don't know whether we're dealing with a continuation // of the comment, or the start of a tag section, or perhaps even the end of the // ScriptDoc section itself. So we need to proceed with caution. For anything other than // a continuation of the comment, we want to bail out. // // Start by skipping any whitespaces we find bool toss = false; if (!SkipWhitespaces( toss )) return false; // Store the line ending off so we can possibly stuff it into the user's buffer UniChar lineEnding = ch; // Next, we expect to see the asterisk that is required. However, it could also be that // we have found the end of the script doc itself. ch = fLexerInput->HasMoreChars() ? fLexerInput->PeekAtNextChar() : -1; if (-1 == ch) return false; if (ch != CHAR_ASTERISK) return false; do { ch = fLexerInput->MoveToNextChar(); } while (ch == CHAR_ASTERISK); if (ch == CHAR_SOLIDUS) { fLexerInput->MoveToPreviousChar(); fLexerInput->MoveToPreviousChar(); break; } fLexerInput->MoveToPreviousChar(); // Now we want to skip any whitespaces we find between the asterisk and the rest of the comment. TokenList spaces; if (!SkipWhitespaces( toss, &spaces )) return false; // The next trick is to see whether we've got a user tag, which follows the asterisk ch = fLexerInput->HasMoreChars() ? fLexerInput->PeekAtNextChar() : -1; if (-1 == ch) return false; if (IsScriptDocTag( ch )) { // In order for this to lex properly on the next pass through AdvanceOneToken, we actually need // to put the asterisk back, as well as any of the spaces we've read. for (TokenList::iterator iter = spaces.begin(); iter != spaces.end(); ++iter) { ILexerToken *token = *iter; for (int i = 0; i < token->GetLength(); i++) fLexerInput->MoveToPreviousChar(); } // Put back the asterisk as well fLexerInput->MoveToPreviousChar(); // We've processed the end of the comment, so we want to bail out now break; } // We are going to preserve the user's formatting by including the newlines they use. AppendUniCharWithBuffer( lineEnding, &outComment, szUCharBuffer, nBufferSize, 256 ); } else { AppendUniCharWithBuffer( ch, &outComment, szUCharBuffer, nBufferSize, 256 ); } } if (nBufferSize > 0) outComment.AppendUniChars( szUCharBuffer, nBufferSize ); return true; }