bool ScriptDocLexer::ConsumeTypeTag()
{
bool toss;
if ( ! SkipWhitespaces( toss ) )
return false;
if (!fLexerInput->HasMoreChars() )
return false;
VString typeList;
bool bDoFurtherProcessing;
bool bracketMode = (fLexerInput->PeekAtNextChar() == CHAR_LEFT_CURLY_BRACKET);
ConsumeTypes( typeList, bDoFurtherProcessing );
if ( ! SkipWhitespaces( toss ) )
return false;
UniChar ch = fLexerInput->MoveToNextChar();
if (IsLineEnding( ch ))
ConsumeLineEnding( ch );
else
fLexerInput->MoveToPreviousChar();
fLastElement = new TypeElement( typeList );
return true;
}
bool VLexerBase::ConsumeSingleLineComment( TokenList *ioTokens )
{
VIndex nStart = fLexerInput->GetCurrentPosition();
while (fLexerInput->HasMoreChars()) {
UniChar c = fLexerInput->MoveToNextChar();
if (IsLineEnding( c )) {
ConsumeLineEnding( c );
break;
}
}
// We subtract one from the length that we pass in because we know it's a newline character, and we don't want to
// include that as part of our comment text.
VString commentText;
fLexerInput->GetSubString( nStart + 1, fLexerInput->GetCurrentPosition() - nStart - 1, commentText );
if (ioTokens != 0) {
ioTokens->push_back( new VLexerToken( ILexerToken::TT_COMMENT, nStart, fLexerInput->GetCurrentPosition() - nStart, commentText ) );
}
CommentConsumed( commentText, 0 );
return true;
}
bool ScriptDocLexer::ConsumeName( VString &outName, bool &outIsOptional, bool &outDoFurtherProcessing )
{
outIsOptional = false;
outDoFurtherProcessing = true;
while (true) {
if (!fLexerInput->HasMoreChars()) return false;
UniChar ch = fLexerInput->MoveToNextChar();
if (CHAR_LEFT_SQUARE_BRACKET == ch) {
// If the name is enclosed in [], then it's considered to be an optional
// parameter.
if (outName.GetLength() == 0) {
outIsOptional = true;
} else {
return false;
}
} else if (IsLineEnding( ch )) {
// We've reached the end of the tag, and the user has not specified a comment, which
// is fine. We'll just break out now and not do any further processing.
ConsumeLineEnding( ch );
outDoFurtherProcessing = false;
break;
} else if (IsWhitespace( ch ) || (outIsOptional && CHAR_RIGHT_SQUARE_BRACKET == ch)) {
break;
} else {
outName.AppendUniChar( ch );
}
}
return true;
}
bool ScriptDocLexer::ConsumeExceptionTag()
{
// The exception tag takes the form: @exception {name} description, and we've already
// handled the @exception clause. So we want to skip over whitespace and look for the
// {name} clause next.
UniChar ch;
bool toss;
if (!SkipWhitespaces( toss )) return false;
if (!fLexerInput->HasMoreChars()) return false;
// We expect the next character to be an open curly bracket.
if (CHAR_LEFT_CURLY_BRACKET != fLexerInput->MoveToNextChar()) return false;
if (!fLexerInput->HasMoreChars()) return false;
// Now we expect to read the name of the exception
VString exceptionName;
while (true) {
if (!fLexerInput->HasMoreChars()) return false;
ch = fLexerInput->MoveToNextChar();
if (IsLineEnding( ch )) { ConsumeLineEnding( ch ); return false; }
if (CHAR_RIGHT_CURLY_BRACKET == ch) break;
exceptionName.AppendUniChar( ch );
}
// Now that we have the exception name clause, we can skip whitespaces and look
// for the description
if (!SkipWhitespaces( toss )) return false;
VString description;
if (!ConsumeComment( description )) return false;
fLastElement = new ExceptionElement( exceptionName, description );
return true;
}
bool ScriptDocLexer::ConsumeReturnTag()
{
UniChar ch;
bool toss;
if ( ! SkipWhitespaces( toss ) )
return false;
if ( ! fLexerInput->HasMoreChars() )
return false;
// We expect the next character to be an open curly bracket.
if ( CHAR_LEFT_CURLY_BRACKET != fLexerInput->MoveToNextChar() )
return false;
if ( ! fLexerInput->HasMoreChars() )
return false;
// Now we expect to read the type list
VString typeList;
while (true)
{
if ( ! fLexerInput->HasMoreChars() )
return false;
ch = fLexerInput->MoveToNextChar();
if ( IsLineEnding( ch ) )
{
ConsumeLineEnding( ch );
return false;
}
if (CHAR_RIGHT_CURLY_BRACKET == ch)
break;
typeList.AppendUniChar( ch );
}
if ( ! SkipWhitespaces( toss ) )
return false;
// Now we can find the description of the return value
VString description;
ConsumeComment( description );
fLastElement = new ReturnElement( typeList, description );
return true;
}
bool ScriptDocLexer::ConsumeScriptDocStart()
{
// Consume the /**. We can make these assertions
// because the IsScriptDocStart function has already ascertained that this data
// is true by peeking at characters. We're merely eating them.
if (!testAssert( fLexerInput->MoveToNextChar() == CHAR_SOLIDUS )) return false;
if (!testAssert( fLexerInput->MoveToNextChar() == CHAR_ASTERISK )) return false;
if (!testAssert( fLexerInput->MoveToNextChar() == CHAR_ASTERISK )) return false;
// Anything else up until the end of the line is simply eaten
while (fLexerInput->HasMoreChars()) {
UniChar uChar = fLexerInput->MoveToNextChar();
if (IsLineEnding( uChar )) {
ConsumeLineEnding( uChar );
break;
}
}
return fLexerInput->HasMoreChars();
}
bool HTMLLexer::IsMultiLineCommentEnd( UniChar inChar, sLONG &outCharsToConsume, sLONG inType )
{
// We have to check for newlines for no other reason than bookkeeping. This records when we
// locate a newline so that the fLineNumber property remains properly in-sync. We will consume
// the newline for the caller if that's what we've gotten.
if (IsLineEnding( inChar )) ConsumeLineEnding( inChar );
// We are looking for -->
if (inType != kCommentContinuationType &&
inType != kHTMLMultilineComment) return false;
if (CHAR_HYPHEN_MINUS != inChar) return false;
if (!fLexerInput->HasMoreChars()) return false;
VString subStr;
fLexerInput->GetSubString( fLexerInput->GetCurrentPosition() + 1, 2, subStr );
if (subStr.EqualTo( CVSTR( "->" ), false )) {
// We're at the end of the comment, but the caller still needs to consume two characters
outCharsToConsume = 2;
return true;
}
return false;
}
bool ScriptDocLexer::ConsumeTypes( VString &outTypes, bool &outDoFurtherProcessing )
{
outDoFurtherProcessing = true;
// We expect the next character to be an open curly bracket if the user has specified
// type information for the parameter
bool curlyBracketMode = CHAR_LEFT_CURLY_BRACKET == fLexerInput->PeekAtNextChar();
if ( curlyBracketMode )
{
fLexerInput->MoveToNextChar();
if ( ! fLexerInput->HasMoreChars() )
return false;
}
// Now we expect to read the type list
while (true)
{
if ( ! fLexerInput->HasMoreChars() )
return false;
UniChar ch = fLexerInput->MoveToNextChar();
if ( IsLineEnding( ch ) )
{
ConsumeLineEnding( ch );
if (curlyBracketMode)
outDoFurtherProcessing = false;
break;
}
if (curlyBracketMode && CHAR_RIGHT_CURLY_BRACKET == ch)
break;
outTypes.AppendUniChar( ch );
}
return true;
}
bool ScriptDocLexer::ConsumeSeeTag()
{
UniChar ch;
VString className, methodName;
bool lexingClassName = true;
while (true) {
if (!fLexerInput->HasMoreChars()) return false;
ch = fLexerInput->MoveToNextChar();
if (IsLineEnding( ch )) { ConsumeLineEnding( ch ); break; }
if (CHAR_NUMBER_SIGN == ch) {
lexingClassName = false;
} else if (lexingClassName) {
className.AppendUniChar( ch );
} else {
methodName.AppendUniChar( ch );
}
}
fLastElement = new SeeElement( className, methodName );
return true;
}
bool HTMLLexer::AdvanceOneToken( int &outToken, TokenList *outTokens )
{
outToken = -1;
if (!fLexerInput)
return false;
// There can be any combination of newlines and whitespaces preceeding semantic tokens,
// so we're going to loop until we don't find either.
while (fLexerInput->HasMoreChars())
{
bool consumedWhitespace = false;
bool consumedNewLine = false;
if (!SkipWhitespaces( consumedWhitespace, outTokens ))
return false;
// HTML also treats newlines as a whitespace
while (fLexerInput->HasMoreChars() && IsLineEnding( fLexerInput->PeekAtNextChar() ))
{
// Eat the line ending
ConsumeLineEnding( fLexerInput->MoveToNextChar() );
consumedNewLine = true;
}
// If we're done consuming newlines and whitespaces, then we're done with this loop
if (!consumedWhitespace && !consumedNewLine)
break;
}
if (!fLexerInput->HasMoreChars())
return false;
// Take a peek at what sort of token we're about to deal with.
UniChar uChar = fLexerInput->PeekAtNextChar();
sLONG stringType;
sLONG stringValue;
if( (outToken = ConsumePossiblePunctuation(uChar, outTokens)) != 0 )
{
}
else if (IsStringStart( uChar, stringType, stringValue ))
{
VString vstrQuoted;
if (!ConsumeString( &vstrQuoted, outTokens, stringType, stringValue ))
{
return false;
}
outToken = stringValue;
fLastTokenText = vstrQuoted;
}
else if (IsIdentifierStart( uChar ))
{
// The base class assumes we've consumed the first character already for this call. We should
// rectify this some day, as it's very confusing.
fLexerInput->MoveToNextChar();
VString *vstrNAME = ConsumeIdentifier();
if (!vstrNAME)
{
return false;
}
outToken = HTMLLexemes::TEXT;
if (outTokens) outTokens->push_back( new HTMLLexerToken( ILexerToken::TT_NAME, fLexerInput->GetCurrentPosition() - vstrNAME->GetLength(), vstrNAME->GetLength(), *vstrNAME, outToken ) );
fLastTokenText = *vstrNAME;
delete vstrNAME;
}
else
{
return false;
}
SetLastToken( outToken );
return true;
}
bool ScriptDocLexer::AdvanceOneToken( int &outToken )
{
// Clear out our last token's text and any tag values we've stored up
fLastElement = NULL;
// Start by skipping any whitespaces that we run into
bool consumedWhitespace = false;
if (!SkipWhitespaces( consumedWhitespace )) return false;
// Now snag the next character from the stream so we can determine how
// to handle it.
if (!fLexerInput->HasMoreChars()) return false;
UniChar ch = fLexerInput->PeekAtNextChar();
if (IsScriptDocStart( ch )) {
ConsumeScriptDocStart();
outToken = ScriptDocTokenValues::START;
return true;
}
// Now that we've handled the start and stop cases for the tag, we need to consume
// some more data from the line. Every ScriptDoc line starts with at least one asterisk,
// but possibly more. We will eat all of the asterisks we find.
if (ch != CHAR_ASTERISK) return false;
do {
ch = fLexerInput->MoveToNextChar();
} while (ch == CHAR_ASTERISK);
// If the very next character is a solidus, that means we've reached the end of the scriptdoc
// comment
if (ch == CHAR_SOLIDUS) {
outToken = ScriptDocTokenValues::END;
return true;
}
// Put the last token we grabbed during asterisk processing back onto the stream
fLexerInput->MoveToPreviousChar();
// Now we can skip any whitespaces that remain
if (!SkipWhitespaces( consumedWhitespace )) return false;
// Having done that, we can continue to look for semantic information.
ch = fLexerInput->HasMoreChars() ? fLexerInput->PeekAtNextChar() : -1;
if (-1 == ch) {
// This is a santity check for the end of input
return false;
} else if (IsLineEnding( ch )) {
ConsumeLineEnding( fLexerInput->MoveToNextChar() );
// If we hit a line ending, then we just want to start this whole process over again
return AdvanceOneToken( outToken );
} else if (IsScriptDocTag( ch )) {
// We found a ScriptDoc tag, so let's eat it
fFoundTags = true;
if (!ConsumeScriptDocTag()) return false;
outToken = ScriptDocTokenValues::ELEMENT;
return true;
} else {
// The only thing left is the comment section. Each ScriptDoc can have one and only
// one comment to it, and it is supposed to be found before any tags. It can span multiple
// lines.
if (fFoundComment || fFoundTags) return false;
fFoundComment = true;
VString text;
if (!ConsumeComment( text )) return false;
fLastElement = new CommentElement( text );
outToken = ScriptDocTokenValues::ELEMENT;
return true;
}
return true;
}
bool ScriptDocLexer::ConsumeComment( VString &outComment )
{
// We found the start of a comment, so we want to consume it in its entirety. The
// only tricky part to comments is that they can span multiple lines, but we don't
// want to include the starting asterisk as part of the comment itself.
UniChar szUCharBuffer [ 256 ];
VIndex nBufferSize = 0;
while (true) {
// If we're out of characters, we need to bail out
if (!fLexerInput->HasMoreChars()) return false;
// Check the next character -- if it's a line ending, we need to handle it specially,
// but if it's not, then we can add it to our comment buffer
UniChar ch = fLexerInput->MoveToNextChar();
if (IsLineEnding( ch )) {
ConsumeLineEnding( ch );
// When we have a line ending, we don't know whether we're dealing with a continuation
// of the comment, or the start of a tag section, or perhaps even the end of the
// ScriptDoc section itself. So we need to proceed with caution. For anything other than
// a continuation of the comment, we want to bail out.
//
// Start by skipping any whitespaces we find
bool toss = false;
if (!SkipWhitespaces( toss )) return false;
// Store the line ending off so we can possibly stuff it into the user's buffer
UniChar lineEnding = ch;
// Next, we expect to see the asterisk that is required. However, it could also be that
// we have found the end of the script doc itself.
ch = fLexerInput->HasMoreChars() ? fLexerInput->PeekAtNextChar() : -1;
if (-1 == ch) return false;
if (ch != CHAR_ASTERISK) return false;
do {
ch = fLexerInput->MoveToNextChar();
} while (ch == CHAR_ASTERISK);
if (ch == CHAR_SOLIDUS) { fLexerInput->MoveToPreviousChar(); fLexerInput->MoveToPreviousChar(); break; }
fLexerInput->MoveToPreviousChar();
// Now we want to skip any whitespaces we find between the asterisk and the rest of the comment.
TokenList spaces;
if (!SkipWhitespaces( toss, &spaces )) return false;
// The next trick is to see whether we've got a user tag, which follows the asterisk
ch = fLexerInput->HasMoreChars() ? fLexerInput->PeekAtNextChar() : -1;
if (-1 == ch) return false;
if (IsScriptDocTag( ch )) {
// In order for this to lex properly on the next pass through AdvanceOneToken, we actually need
// to put the asterisk back, as well as any of the spaces we've read.
for (TokenList::iterator iter = spaces.begin(); iter != spaces.end(); ++iter) {
ILexerToken *token = *iter;
for (int i = 0; i < token->GetLength(); i++) fLexerInput->MoveToPreviousChar();
}
// Put back the asterisk as well
fLexerInput->MoveToPreviousChar();
// We've processed the end of the comment, so we want to bail out now
break;
}
// We are going to preserve the user's formatting by including the newlines they use.
AppendUniCharWithBuffer( lineEnding, &outComment, szUCharBuffer, nBufferSize, 256 );
} else {
AppendUniCharWithBuffer( ch, &outComment, szUCharBuffer, nBufferSize, 256 );
}
}
if (nBufferSize > 0) outComment.AppendUniChars( szUCharBuffer, nBufferSize );
return true;
}
bool ScriptDocLexer::ConsumeScriptDocTag()
{
// We need to consume the @ token for the tag. We know it exists because
// we've already peeked at it
if (!testAssert( fLexerInput->MoveToNextChar() == CHAR_COMMERCIAL_AT )) return false;
// After the @ symbol comes the name of the tag, followed by some whitespace. Any
// whitespace between the @ and the tag name will be eaten
bool toss = false;
if (!SkipWhitespaces( toss )) return false;
UniChar ch;
VString text;
bool bLineEnded = false;
while (true) {
if (!fLexerInput->HasMoreChars()) return false;
ch = fLexerInput->MoveToNextChar();
if (IsWhitespace( ch )) break;
if (IsLineEnding( ch )) {
ConsumeLineEnding( ch );
bLineEnded = true;
break;
}
text.AppendUniChar( ch );
}
// Skip any whitespace between the tag name and the tag contents
if (!SkipWhitespaces( toss )) return false;
// Now that we have the tag name, we can create the appropriate class to represent
// it.
VString simpleText;
if (text == "author" && ConsumeComment( simpleText ))
{
fLastElement = new AuthorElement( simpleText );
}
else if (text == "alias" && ConsumeComment( simpleText ))
{
fLastElement = new AliasElement( simpleText );
}
else if (text == "attributes" && ConsumeComment( simpleText ))
{
fLastElement = new AttributesElement( simpleText );
}
else if (text == "class" && ConsumeComment( simpleText ))
{
fLastElement = new ClassElement( simpleText );
}
else if (text == "classDescription" && ConsumeComment( simpleText ))
{
fLastElement = new ClassDescriptionElement( simpleText );
}
else if (text == "constructor")
{
fLastElement = new ConstructorElement();
}
else if (text == "default" && ConsumeComment( simpleText ))
{
fLastElement = new DefaultElement(simpleText);
}
else if (text == "deprecated" && ConsumeComment( simpleText ))
{
fLastElement = new DeprecatedElement(simpleText);
}
else if (text == "exception")
{
return ConsumeExceptionTag();
}
else if ( (text == "extends" || text == "augments" || text == "inherits") && ConsumeComment( simpleText ))
{
fLastElement = new ExtendsElement(simpleText);
}
else if (text == "id" && ConsumeComment( simpleText ))
{
fLastElement = new IdElement( simpleText );
}
else if (text == "inherits" && ConsumeComment( simpleText ))
{
fLastElement = new InheritsElement( simpleText );
}
else if (text == "memberOf" && ConsumeComment( simpleText ))
{
fLastElement = new MemberOfElement( simpleText );
}
else if (text == "method")
{
return ConsumeMethodTag();
}
else if (text == "module" && ConsumeComment( simpleText ))
{
fLastElement = new ModuleElement( simpleText );
}
else if (text == "namespace" && ConsumeComment( simpleText ))
{
fLastElement = new NamespaceElement( simpleText );
}
else if (text == "param")
{
return ConsumeParamTag();
}
else if (text == "paramValue")
{
return ConsumeParamValueTag();
}
else if (text == "private")
{
fLastElement = new PrivateElement();
}
else if (text == "projectDescription" && ConsumeComment( simpleText ))
{
fLastElement = new ProjectDescriptionElement( simpleText );
}
else if (text == "property")
{
return ConsumePropertyTag();
}
else if ( (text == "require" ||text == "requires") && ConsumeComment( simpleText ))
{
fLastElement = new RequiresElement( simpleText );
}
else if (text == "return" || text == "returns" || text == "result")
{
return ConsumeReturnTag();
}
else if (text == "sdoc" && ConsumeComment( simpleText ))
{
fLastElement = new SDocElement( simpleText );
}
else if (text == "see")
{
return ConsumeSeeTag();
}
else if (text == "since" && ConsumeComment( simpleText ))
{
fLastElement = new SinceElement( simpleText );
}
else if (text == "static")
{
fLastElement = new StaticElement();
}
else if (text == "throws" || text == "throw")
{
return ConsumeThrowsTag();
}
else if (text == "type")
{
return ConsumeTypeTag();
}
else if (text == "version" && ConsumeComment( simpleText ))
{
fLastElement = new VersionElement( simpleText );
}
else
{
// This is an unsupported tag, so make an Unknown tag to handle it. Unknown tags can
// have comments following them, or just a newline signifying it's time for another tag.
// We don't know which it is supposed to be, so we will skip whitespaces and check to see
// if there's a newline. If there is, then it's assumed to be a tag with no comments.
if (bLineEnded || ConsumeComment( simpleText ))
{
fLastElement = new UnknownElement( text, simpleText );
}
else
{
return false;
}
}
return true;
}
