void Lexer::token(Token &t) { if ( file.fd() < 0) t.Type( Token::eof); else { token2(t); if (t.Type() == Token::eof) file.Close(); } lasttokentype=t.Type(); if (maildrop.embedded_mode) switch (lasttokentype) { case Token::tokento: case Token::tokencc: case Token::btstring: case Token::tokenxfilter: case Token::dotlock: case Token::flock: case Token::logfile: case Token::log: { Buffer errmsg; errmsg="maildrop: '"; errmsg += t.Name(); errmsg += "' disabled in embedded mode.\n"; errmsg += '\0'; error((const char *)errmsg); t.Type( Token::error ); break; } default: break; } if (VerboseLevel() > 8) { Buffer debug; debug="Tokenized "; debug += t.Name(); debug += '\n'; debug += '\0'; error((const char *)debug); } }
TokenList BracketMatcher::ReadCombination() { // TODO: split into smaller functions enum EMode { eOther , eOperator , eFirstOperand }; TokenList ret; unsigned int bracket_level = 0; EMode mode = eOther; while( true ) { Token token = lexer_.NextToken(); // If the stream ended before a combo was closed, return an empty // list - the user pressed Ctrl-D and doesn't want any output. if( token.IsEndOfStream() ) { return TokenList(); } token.AddToColumn( newline_processor_.GetIndent() ); // If we have got to the first operand of a combination // store the indent level so that we can indent correctly // in the repl when the user presses return. switch( mode ) { case eOperator: { mode = eFirstOperand; break; } case eFirstOperand: { newline_processor_.PushIndent( bracket_level, token.Column() ); mode = eOther; break; } default: { break; } } ret.AddToken( token ); // If we've finished this combination, return now if( token.Name() == ")" && bracket_level == 1 ) { break; } // Otherwise, adjust our bracket level if there is a ( or ) if( token.Name() == "(" ) { ++bracket_level; mode = eOperator; // The next token will be the operator } else if( token.Name() == ")" ) { newline_processor_.PopIndent( bracket_level ); --bracket_level; mode = eOther; } // If we got a bare symbol (no brackets) we exit here // (except that we skip straight past quotes) if( bracket_level == 0 && token.Name() != "'" ) { break; } // If the lexer's token ended with newline, it couldn't call // newline (because we hadn't processed the token yet). We // have now processed it, so we can call NewLine now. if( print_continuation_ && lexer_.EndedWithNewLine() ) { newline_processor_.NewLine(); } } // When the combination (or bare symbol) is finished, we don't // need to track indentation any more, so reset the newline processor, // and we don't care about any newlines before the next token, so // we skip whitespace in the lexer. newline_processor_.Reset(); return ret; }