void DwRfc1521Tokenizer::ParseToken()
{
// Assume the field body has already been extracted. That is, we don't
// have to watch for the end of the field body or folding. We just
// treat any CRs or LFs as white space.
mTokenStart = mNextStart;
mTokenLength = 0;
mTkType = eTkNull;
if (mTokenStart >= mString.length()) {
return;
}
// Skip leading space. Also, since control chars are not permitted
// in atoms, skip these, too.
while (1) {
if (mTokenStart >= mString.length()) {
return;
}
if (!isspace(mString[mTokenStart]) && !iscntrl(mString[mTokenStart]))
break;
++mTokenStart;
}
char ch = mString[mTokenStart];
// Quoted string
if (ch == '"') {
mTkType = eTkQuotedString;
ParseQuotedString();
}
// Comment
else if (ch == '(') {
mTkType = eTkComment;
ParseComment();
}
// Domain literal
else if (ch == '[') {
mTkType = eTkDomainLiteral;
ParseDomainLiteral();
}
// Special
else if (istspecial(ch)) {
mTkType = eTkTspecial;
mTokenLength = 1;
mToken = mString.substr(mTokenStart, 1);
mNextStart = mTokenStart + 1;
}
// Atom
else {
mTkType = eTkToken;
ParseAtom();
}
if (mDebugOut) PrintToken(mDebugOut);
}
void DwRfc1521Tokenizer::ParseToken()
{
// Assume the field body has already been extracted. That is, we don't
// have to watch for the end of the field body or folding. We just
// treat any CRs or LFs as white space.
mTokenStart = mNextStart;
mTokenLength = 0;
mTkType = eTkNull;
// Skip leading space. Also, since control chars are not permitted
// in atoms, skip these, too.
while(1)
{
if(mTokenStart >= mString.length())
{
return;
}
if(isnotspaceorcntrl(mString[mTokenStart]))
break;
++mTokenStart;
}
char ch = mString[mTokenStart];
switch(ch)
{
// Quoted string
case '"':
mTkType = eTkQuotedString;
ParseQuotedString();
break;
// Comment
case '(':
mTkType = eTkComment;
ParseComment();
break;
// Domain literal
case '[':
mTkType = eTkDomainLiteral;
ParseDomainLiteral();
break;
// Special
case ')':
case '<':
case '>':
case '@':
case ',':
case ';':
case ':':
case '\\':
case '/':
case ']':
case '?':
case '=':
mTkType = eTkTspecial;
mTokenLength = 1;
mToken = mString.substr(mTokenStart, 1);
mNextStart = mTokenStart + 1;
break;
default:
mTkType = eTkToken;
ParseAtom();
break;
}
if(mDebugOut) PrintToken(mDebugOut);
}
