bool ScriptDocLexer::ConsumePropertyTag()
{
bool toss;
if (!SkipWhitespaces( toss )) return false;
if (!fLexerInput->HasMoreChars()) return false;
VString typeList, name, description;
bool isOptional, bDoFurtherProcessing;
// Look for the optional property type(s)
if (ConsumeTypes(typeList, bDoFurtherProcessing))
if (!SkipWhitespaces( toss )) return false;
// Look for the required property name.
if (!bDoFurtherProcessing) return false;
if (!ConsumeName( name, isOptional, bDoFurtherProcessing )) return false;
// Now we can find the optional property description
if (bDoFurtherProcessing)
ConsumeComment( description );
fLastElement = new PropertyElement( typeList, name, description );
return true;
}
bool ScriptDocLexer::ConsumeExceptionTag()
{
// The exception tag takes the form: @exception {name} description, and we've already
// handled the @exception clause. So we want to skip over whitespace and look for the
// {name} clause next.
UniChar ch;
bool toss;
if (!SkipWhitespaces( toss )) return false;
if (!fLexerInput->HasMoreChars()) return false;
// We expect the next character to be an open curly bracket.
if (CHAR_LEFT_CURLY_BRACKET != fLexerInput->MoveToNextChar()) return false;
if (!fLexerInput->HasMoreChars()) return false;
// Now we expect to read the name of the exception
VString exceptionName;
while (true) {
if (!fLexerInput->HasMoreChars()) return false;
ch = fLexerInput->MoveToNextChar();
if (IsLineEnding( ch )) { ConsumeLineEnding( ch ); return false; }
if (CHAR_RIGHT_CURLY_BRACKET == ch) break;
exceptionName.AppendUniChar( ch );
}
// Now that we have the exception name clause, we can skip whitespaces and look
// for the description
if (!SkipWhitespaces( toss )) return false;
VString description;
if (!ConsumeComment( description )) return false;
fLastElement = new ExceptionElement( exceptionName, description );
return true;
}
bool ScriptDocLexer::ConsumeReturnTag()
{
UniChar ch;
bool toss;
if ( ! SkipWhitespaces( toss ) )
return false;
if ( ! fLexerInput->HasMoreChars() )
return false;
// We expect the next character to be an open curly bracket.
if ( CHAR_LEFT_CURLY_BRACKET != fLexerInput->MoveToNextChar() )
return false;
if ( ! fLexerInput->HasMoreChars() )
return false;
// Now we expect to read the type list
VString typeList;
while (true)
{
if ( ! fLexerInput->HasMoreChars() )
return false;
ch = fLexerInput->MoveToNextChar();
if ( IsLineEnding( ch ) )
{
ConsumeLineEnding( ch );
return false;
}
if (CHAR_RIGHT_CURLY_BRACKET == ch)
break;
typeList.AppendUniChar( ch );
}
if ( ! SkipWhitespaces( toss ) )
return false;
// Now we can find the description of the return value
VString description;
ConsumeComment( description );
fLastElement = new ReturnElement( typeList, description );
return true;
}
bool ScriptDocLexer::ConsumeMethodTag()
{
UniChar ch;
// So it turns out that sometimes there will be a method name here. So we will eat anything up to the
// end of the line and just ignore it.
fLastElement = new MethodElement();
if (fLexerInput->HasMoreChars()) {
ch = fLexerInput->PeekAtNextChar();
if (CHAR_ASTERISK != ch && !IsLineEnding( ch )) {
VString toss;
ConsumeComment( toss );
}
}
return true;
}
bool ScriptDocLexer::ConsumeParamTag()
{
// Param tags are one of the most complex ScriptDoc tags. It consists of a list of
// types followed by a name and a description. We've already processed the @param
// part of the element, so we want to find the type list.
UniChar ch;
bool toss;
if (!SkipWhitespaces( toss )) return false;
if (!fLexerInput->HasMoreChars()) return false;
// Unfortunately, there's no real concensus on how this tag is handled. There's the
// ScriptDoc Way, but then there's also a very common variation. Basically, it can
// either be @param {types} Name Comments or @param Name {types} Comments, and we
// really want to support both versions. Normal order is the ScriptDoc way.
bool bIsNormalOrder = (fLexerInput->PeekAtNextChar() == CHAR_LEFT_CURLY_BRACKET);
bool bDoFurtherProcessing;
VString typeList, paramName, description;
bool isOptional;
if (bIsNormalOrder)
{
ConsumeTypes( typeList, bDoFurtherProcessing );
if (!SkipWhitespaces( toss ))
return false;
if (!ConsumeName( paramName, isOptional, bDoFurtherProcessing ) || paramName.IsEmpty() )
return false;
}
else
{
if (!ConsumeName( paramName, isOptional, bDoFurtherProcessing ))
return false;
if (bDoFurtherProcessing)
ConsumeTypes( typeList, bDoFurtherProcessing );
}
if (bDoFurtherProcessing)
{
if (!SkipWhitespaces( toss ))
return false;
// Now we can find the description of the parameter
ConsumeComment( description );
}
fLastElement = new ParamElement( paramName, typeList, description, isOptional );
return true;
}
bool ScriptDocLexer::ConsumeThrowsTag()
{
bool toss;
if (!SkipWhitespaces( toss )) return false;
if (!fLexerInput->HasMoreChars()) return false;
VString type, description;
bool isOptional, bDoFurtherProcessing;
// Look for the optional type
if (ConsumeTypes(type, bDoFurtherProcessing))
if (!SkipWhitespaces( toss )) return false;
// Now we can find the optional description
if (bDoFurtherProcessing)
ConsumeComment( description );
fLastElement = new ThrowsElement( type, description );
return true;
}
/**
* This method is called just after a "<" has been consumed
* and we know we're at the start of some kind of tagged
* element. We don't know yet if it's a tag or a comment.
*
* @param aChar is the last char read
* @param aToken is the out arg holding our new token (the function allocates
* the return token using mTokenAllocator).
* @param aScanner represents our input source
* @param aFlushTokens is an OUT parameter use to tell consumers to flush
* the current tokens after processing the current one.
* @return error code.
*/
nsresult
nsHTMLTokenizer::ConsumeTag(PRUnichar aChar,
CToken*& aToken,
nsScanner& aScanner,
bool& aFlushTokens)
{
PRUnichar theNextChar, oldChar;
nsresult result = aScanner.Peek(aChar, 1);
if (NS_OK == result) {
switch (aChar) {
case kForwardSlash:
result = aScanner.Peek(theNextChar, 2);
if (NS_OK == result) {
// Get the original "<" (we've already seen it with a Peek)
aScanner.GetChar(oldChar);
// XML allows non ASCII tag names, consume this as an end tag. This
// is needed to make XML view source work
bool isXML = !!(mFlags & NS_IPARSER_FLAG_XML);
if (nsCRT::IsAsciiAlpha(theNextChar) ||
kGreaterThan == theNextChar ||
(isXML && !nsCRT::IsAscii(theNextChar))) {
result = ConsumeEndTag(aChar, aToken, aScanner);
} else {
result = ConsumeComment(aChar, aToken, aScanner);
}
}
break;
case kExclamation:
result = aScanner.Peek(theNextChar, 2);
if (NS_OK == result) {
// Get the original "<" (we've already seen it with a Peek)
aScanner.GetChar(oldChar);
if (kMinus == theNextChar || kGreaterThan == theNextChar) {
result = ConsumeComment(aChar, aToken, aScanner);
} else {
result = ConsumeSpecialMarkup(aChar, aToken, aScanner);
}
}
break;
case kQuestionMark:
// It must be a processing instruction...
// Get the original "<" (we've already seen it with a Peek)
aScanner.GetChar(oldChar);
result = ConsumeProcessingInstruction(aChar, aToken, aScanner);
break;
default:
// XML allows non ASCII tag names, consume this as a start tag.
bool isXML = !!(mFlags & NS_IPARSER_FLAG_XML);
if (nsCRT::IsAsciiAlpha(aChar) ||
(isXML && !nsCRT::IsAscii(aChar))) {
// Get the original "<" (we've already seen it with a Peek)
aScanner.GetChar(oldChar);
result = ConsumeStartTag(aChar, aToken, aScanner, aFlushTokens);
} else {
// We are not dealing with a tag. So, don't consume the original
// char and leave the decision to ConsumeText().
result = ConsumeText(aToken, aScanner);
}
}
}
// Last ditch attempt to make sure we don't lose data.
if (kEOF == result && !aScanner.IsIncremental()) {
// Whoops, we don't want to lose any data! Consume the rest as text.
// This normally happens for either a trailing < or </
result = ConsumeText(aToken, aScanner);
}
return result;
}
bool ScriptDocLexer::AdvanceOneToken( int &outToken )
{
// Clear out our last token's text and any tag values we've stored up
fLastElement = NULL;
// Start by skipping any whitespaces that we run into
bool consumedWhitespace = false;
if (!SkipWhitespaces( consumedWhitespace )) return false;
// Now snag the next character from the stream so we can determine how
// to handle it.
if (!fLexerInput->HasMoreChars()) return false;
UniChar ch = fLexerInput->PeekAtNextChar();
if (IsScriptDocStart( ch )) {
ConsumeScriptDocStart();
outToken = ScriptDocTokenValues::START;
return true;
}
// Now that we've handled the start and stop cases for the tag, we need to consume
// some more data from the line. Every ScriptDoc line starts with at least one asterisk,
// but possibly more. We will eat all of the asterisks we find.
if (ch != CHAR_ASTERISK) return false;
do {
ch = fLexerInput->MoveToNextChar();
} while (ch == CHAR_ASTERISK);
// If the very next character is a solidus, that means we've reached the end of the scriptdoc
// comment
if (ch == CHAR_SOLIDUS) {
outToken = ScriptDocTokenValues::END;
return true;
}
// Put the last token we grabbed during asterisk processing back onto the stream
fLexerInput->MoveToPreviousChar();
// Now we can skip any whitespaces that remain
if (!SkipWhitespaces( consumedWhitespace )) return false;
// Having done that, we can continue to look for semantic information.
ch = fLexerInput->HasMoreChars() ? fLexerInput->PeekAtNextChar() : -1;
if (-1 == ch) {
// This is a santity check for the end of input
return false;
} else if (IsLineEnding( ch )) {
ConsumeLineEnding( fLexerInput->MoveToNextChar() );
// If we hit a line ending, then we just want to start this whole process over again
return AdvanceOneToken( outToken );
} else if (IsScriptDocTag( ch )) {
// We found a ScriptDoc tag, so let's eat it
fFoundTags = true;
if (!ConsumeScriptDocTag()) return false;
outToken = ScriptDocTokenValues::ELEMENT;
return true;
} else {
// The only thing left is the comment section. Each ScriptDoc can have one and only
// one comment to it, and it is supposed to be found before any tags. It can span multiple
// lines.
if (fFoundComment || fFoundTags) return false;
fFoundComment = true;
VString text;
if (!ConsumeComment( text )) return false;
fLastElement = new CommentElement( text );
outToken = ScriptDocTokenValues::ELEMENT;
return true;
}
return true;
}
bool ScriptDocLexer::ConsumeScriptDocTag()
{
// We need to consume the @ token for the tag. We know it exists because
// we've already peeked at it
if (!testAssert( fLexerInput->MoveToNextChar() == CHAR_COMMERCIAL_AT )) return false;
// After the @ symbol comes the name of the tag, followed by some whitespace. Any
// whitespace between the @ and the tag name will be eaten
bool toss = false;
if (!SkipWhitespaces( toss )) return false;
UniChar ch;
VString text;
bool bLineEnded = false;
while (true) {
if (!fLexerInput->HasMoreChars()) return false;
ch = fLexerInput->MoveToNextChar();
if (IsWhitespace( ch )) break;
if (IsLineEnding( ch )) {
ConsumeLineEnding( ch );
bLineEnded = true;
break;
}
text.AppendUniChar( ch );
}
// Skip any whitespace between the tag name and the tag contents
if (!SkipWhitespaces( toss )) return false;
// Now that we have the tag name, we can create the appropriate class to represent
// it.
VString simpleText;
if (text == "author" && ConsumeComment( simpleText ))
{
fLastElement = new AuthorElement( simpleText );
}
else if (text == "alias" && ConsumeComment( simpleText ))
{
fLastElement = new AliasElement( simpleText );
}
else if (text == "attributes" && ConsumeComment( simpleText ))
{
fLastElement = new AttributesElement( simpleText );
}
else if (text == "class" && ConsumeComment( simpleText ))
{
fLastElement = new ClassElement( simpleText );
}
else if (text == "classDescription" && ConsumeComment( simpleText ))
{
fLastElement = new ClassDescriptionElement( simpleText );
}
else if (text == "constructor")
{
fLastElement = new ConstructorElement();
}
else if (text == "default" && ConsumeComment( simpleText ))
{
fLastElement = new DefaultElement(simpleText);
}
else if (text == "deprecated" && ConsumeComment( simpleText ))
{
fLastElement = new DeprecatedElement(simpleText);
}
else if (text == "exception")
{
return ConsumeExceptionTag();
}
else if ( (text == "extends" || text == "augments" || text == "inherits") && ConsumeComment( simpleText ))
{
fLastElement = new ExtendsElement(simpleText);
}
else if (text == "id" && ConsumeComment( simpleText ))
{
fLastElement = new IdElement( simpleText );
}
else if (text == "inherits" && ConsumeComment( simpleText ))
{
fLastElement = new InheritsElement( simpleText );
}
else if (text == "memberOf" && ConsumeComment( simpleText ))
{
fLastElement = new MemberOfElement( simpleText );
}
else if (text == "method")
{
return ConsumeMethodTag();
}
else if (text == "module" && ConsumeComment( simpleText ))
{
fLastElement = new ModuleElement( simpleText );
}
else if (text == "namespace" && ConsumeComment( simpleText ))
{
fLastElement = new NamespaceElement( simpleText );
}
else if (text == "param")
{
return ConsumeParamTag();
}
else if (text == "paramValue")
{
return ConsumeParamValueTag();
}
else if (text == "private")
{
fLastElement = new PrivateElement();
}
else if (text == "projectDescription" && ConsumeComment( simpleText ))
{
fLastElement = new ProjectDescriptionElement( simpleText );
}
else if (text == "property")
{
return ConsumePropertyTag();
}
else if ( (text == "require" ||text == "requires") && ConsumeComment( simpleText ))
{
fLastElement = new RequiresElement( simpleText );
}
else if (text == "return" || text == "returns" || text == "result")
{
return ConsumeReturnTag();
}
else if (text == "sdoc" && ConsumeComment( simpleText ))
{
fLastElement = new SDocElement( simpleText );
}
else if (text == "see")
{
return ConsumeSeeTag();
}
else if (text == "since" && ConsumeComment( simpleText ))
{
fLastElement = new SinceElement( simpleText );
}
else if (text == "static")
{
fLastElement = new StaticElement();
}
else if (text == "throws" || text == "throw")
{
return ConsumeThrowsTag();
}
else if (text == "type")
{
return ConsumeTypeTag();
}
else if (text == "version" && ConsumeComment( simpleText ))
{
fLastElement = new VersionElement( simpleText );
}
else
{
// This is an unsupported tag, so make an Unknown tag to handle it. Unknown tags can
// have comments following them, or just a newline signifying it's time for another tag.
// We don't know which it is supposed to be, so we will skip whitespaces and check to see
// if there's a newline. If there is, then it's assumed to be a tag with no comments.
if (bLineEnded || ConsumeComment( simpleText ))
{
fLastElement = new UnknownElement( text, simpleText );
}
else
{
return false;
}
}
return true;
}
void CFile::SkipWhitespace()
{
while (ConsumeHorizontalWhitespace() || ConsumeNewline() || ConsumeComment())
;
}
