/** caller handles locking */
static bool PREPLOGBUFFER(BufBuilder& bb) {
if( writes.empty() )
return false;
bb.reset();
unsigned *lenInBlockHeader;
{
// JSectHeader
bb.appendStr("\nHH\n", false);
lenInBlockHeader = (unsigned *) bb.skip(4);
}
string lastFilePath;
{
scoped_lock lk(privateViews._mutex());
for( vector<WriteIntent>::iterator i = writes.begin(); i != writes.end(); i++ ) {
size_t ofs;
MongoMMF *mmf = privateViews._find(i->p, ofs);
if( mmf == 0 ) {
journalingFailure("view pointer cannot be resolved");
}
else {
if( mmf->filePath() != lastFilePath ) {
lastFilePath = mmf->filePath();
JDbContext c;
bb.appendStruct(c);
bb.appendStr(lastFilePath);
}
JEntry e;
e.len = i->len;
e.fileNo = mmf->fileSuffixNo();
bb.appendStruct(e);
bb.appendBuf(i->p, i->len);
}
}
}
{
JSectFooter f;
f.hash = 0;
bb.appendStruct(f);
}
{
unsigned L = (bb.len() + 8191) & 0xffffe000; // fill to alignment
dassert( L >= (unsigned) bb.len() );
*lenInBlockHeader = L;
unsigned padding = L - bb.len();
bb.skip(padding);
dassert( bb.len() % 8192 == 0 );
}
writes.clear();
alreadyNoted.clear();
return true;
}
std::unique_ptr<Message> LegacyReplyBuilder::done() {
invariant(_state == State::kOutputDocs);
BSONObj reply = uassertStatusOK(rpc::downconvertReplyMetadata(_commandReply, _metadata));
BufBuilder bufBuilder;
bufBuilder.skip(sizeof(QueryResult::Value));
if (_allowAddingOutputDocs) {
BSONObjBuilder topBuilder(bufBuilder);
for (const auto& el : reply) {
if (kCursorTag != el.fieldNameStringData()) {
topBuilder.append(el);
continue;
}
invariant(el.isABSONObj());
BSONObjBuilder curBuilder(topBuilder.subobjStart(kCursorTag));
for (const auto& insideEl : el.Obj()) {
if (kFirstBatchTag != insideEl.fieldNameStringData()) {
curBuilder.append(insideEl);
continue;
}
invariant(insideEl.isABSONObj());
BSONArrayBuilder arrBuilder(curBuilder.subarrayStart(kFirstBatchTag));
for (const auto& doc : _outputDocs) {
arrBuilder.append(doc);
}
arrBuilder.doneFast();
}
curBuilder.doneFast();
}
topBuilder.doneFast();
} else {
reply.appendSelfToBufBuilder(bufBuilder);
}
auto msgHeaderSz = static_cast<std::size_t>(MsgData::MsgDataHeaderSize);
invariant(static_cast<std::size_t>(bufBuilder.len()) + msgHeaderSz <=
mongol::MaxMessageSizeBytes);
QueryResult::View qr = bufBuilder.buf();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bufBuilder.len());
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
_message->setData(qr.view2ptr(), true);
bufBuilder.decouple();
_state = State::kDone;
return std::move(_message);
}
void DBClientMultiCommand::sendAll() {
for ( deque<PendingCommand*>::iterator it = _pendingCommands.begin();
it != _pendingCommands.end(); ++it ) {
PendingCommand* command = *it;
dassert( NULL == command->conn );
try {
// TODO: Figure out how to handle repl sets, configs
dassert( command->endpoint.type() == ConnectionString::MASTER ||
command->endpoint.type() == ConnectionString::CUSTOM );
command->conn = shardConnectionPool.get( command->endpoint, 0 /*timeout*/);
Message toSend;
// see query.h for the protocol we are using here.
BufBuilder bufB;
bufB.appendNum( 0 ); // command/query options
bufB.appendStr( command->dbName + ".$cmd" ); // write command ns
bufB.appendNum( 0 ); // ntoskip (0 for command)
bufB.appendNum( 1 ); // ntoreturn (1 for command)
command->cmdObj.appendSelfToBufBuilder( bufB );
toSend.setData( dbQuery, bufB.buf(), bufB.len() );
// Send our command
command->conn->say( toSend );
}
catch ( const DBException& ex ) {
command->status = ex.toStatus();
if ( NULL != command->conn ) delete command->conn;
command->conn = NULL;
}
}
}
void DBClientCursor::_assembleInit(Message& toSend) {
// If we haven't gotten a cursorId yet, we need to issue a new query or command.
if (!cursorId) {
// HACK:
// Unfortunately, this code is used by the shell to run commands,
// so we need to allow the shell to send invalid options so that we can
// test that the server rejects them. Thus, to allow generating commands with
// invalid options, we validate them here, and fall back to generating an OP_QUERY
// through assembleQueryRequest if the options are invalid.
bool hasValidNToReturnForCommand = (nToReturn == 1 || nToReturn == -1);
bool hasValidFlagsForCommand = !(opts & mongo::QueryOption_Exhaust);
if (_isCommand && hasValidNToReturnForCommand && hasValidFlagsForCommand) {
toSend = assembleCommandRequest(_client, nsToDatabaseSubstring(ns), opts, query);
return;
}
assembleQueryRequest(ns, query, nextBatchSize(), nToSkip, fieldsToReturn, opts, toSend);
return;
}
// Assemble a legacy getMore request.
BufBuilder b;
b.appendNum(opts);
b.appendStr(ns);
b.appendNum(nToReturn);
b.appendNum(cursorId);
toSend.setData(dbGetMore, b.buf(), b.len());
}
void DBClientCursor::requestMore() {
assert( cursorId && pos == nReturned );
if (haveLimit) {
nToReturn -= nReturned;
assert(nToReturn > 0);
}
BufBuilder b;
b.append(opts);
b.append(ns.c_str());
b.append(nextBatchSize());
b.append(cursorId);
Message toSend;
toSend.setData(dbGetMore, b.buf(), b.len());
auto_ptr<Message> response(new Message());
if ( connector ) {
connector->call( toSend, *response );
m = response;
dataReceived();
}
else {
assert( _scopedHost.size() );
ScopedDbConnection conn( _scopedHost );
conn->call( toSend , *response );
connector = conn.get();
m = response;
dataReceived();
connector = 0;
conn.done();
}
}
DBClientCursor::~DBClientCursor() {
if (!this)
return;
DESTRUCTOR_GUARD (
if ( cursorId && _ownCursor ) {
BufBuilder b;
b.appendNum( (int)0 ); // reserved
b.appendNum( (int)1 ); // number
b.appendNum( cursorId );
Message m;
m.setData( dbKillCursors , b.buf() , b.len() );
if ( connector ){
connector->sayPiggyBack( m );
}
else {
assert( _scopedHost.size() );
ScopedDbConnection conn( _scopedHost );
conn->sayPiggyBack( m );
conn.done();
}
}
);
void DBClientCursor::requestMore() {
verify( cursorId && batch.pos == batch.nReturned );
if (haveLimit) {
nToReturn -= batch.nReturned;
verify(nToReturn > 0);
}
BufBuilder b;
b.appendNum(opts);
b.appendStr(ns);
b.appendNum(nextBatchSize());
b.appendNum(cursorId);
Message toSend;
toSend.setData(dbGetMore, b.buf(), b.len());
auto_ptr<Message> response(new Message());
if ( _client ) {
_client->call( toSend, *response );
this->batch.m = response;
dataReceived();
}
else {
verify( _scopedHost.size() );
ScopedDbConnection conn(_scopedHost);
conn->call( toSend , *response );
_client = conn.get();
this->batch.m = response;
dataReceived();
_client = 0;
conn.done();
}
}
static bool receivedQuery(Client& c, DbResponse& dbresponse, Message& m ){
bool ok = true;
MSGID responseTo = m.header()->id;
DbMessage d(m);
QueryMessage q(d);
auto_ptr< Message > resp( new Message() );
CurOp& op = *(c.curop());
try {
dbresponse.exhaust = runQuery(m, q, op, *resp);
assert( !resp->empty() );
}
catch ( AssertionException& e ) {
ok = false;
op.debug().str << " exception ";
LOGSOME {
log() << "assertion " << e.toString() << " ns:" << q.ns << " query:" <<
(q.query.valid() ? q.query.toString() : "query object is corrupt") << endl;
if( q.ntoskip || q.ntoreturn )
log() << " ntoskip:" << q.ntoskip << " ntoreturn:" << q.ntoreturn << endl;
}
BSONObjBuilder err;
e.getInfo().append( err );
BSONObj errObj = err.done();
BufBuilder b;
b.skip(sizeof(QueryResult));
b.appendBuf((void*) errObj.objdata(), errObj.objsize());
// todo: call replyToQuery() from here instead of this!!! see dbmessage.h
QueryResult * msgdata = (QueryResult *) b.buf();
b.decouple();
QueryResult *qr = msgdata;
qr->_resultFlags() = ResultFlag_ErrSet;
if ( e.getCode() == StaleConfigInContextCode )
qr->_resultFlags() |= ResultFlag_ShardConfigStale;
qr->len = b.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
resp.reset( new Message() );
resp->setData( msgdata, true );
}
if ( op.shouldDBProfile( 0 ) ){
op.debug().str << " bytes:" << resp->header()->dataLen();
}
dbresponse.response = resp.release();
dbresponse.responseTo = responseTo;
return ok;
}
// Test if the reserved field is short of 4 bytes
TEST(DBMessage1, ShortFlags) {
BufBuilder b;
string ns("test");
b.appendChar(1);
Message toSend;
toSend.setData(dbDelete, b.buf(), b.len());
ASSERT_THROWS(DbMessage d1(toSend), UserException);
}
void DBClientConnection::killCursor( long long cursorId ){
BufBuilder b;
b.append( (int)0 ); // reserved
b.append( (int)1 ); // number
b.append( cursorId );
Message m;
m.setData( dbKillCursors , b.buf() , b.len() );
sayPiggyBack( m );
}
void DBClientBase::insert( const string & ns , BSONObj obj , int flags) {
Message toSend;
BufBuilder b;
b.appendNum( flags );
b.appendStr( ns );
obj.appendSelfToBufBuilder( b );
toSend.setData( dbInsert , b.buf() , b.len() );
say( toSend );
}
// Try a bad read of a type too large
TEST(DBMessage1, GoodKill2) {
BufBuilder b;
b.appendNum(static_cast<int>(1));
b.appendNum(static_cast<int>(3));
Message toSend;
toSend.setData(dbKillCursors, b.buf(), b.len());
DbMessage d1(toSend);
ASSERT_THROWS(d1.pullInt64(), UserException);
}
void generateLegacyQueryErrorResponse(const AssertionException* exception,
const QueryMessage& queryMessage,
CurOp* curop,
Message* response) {
curop->debug().exceptionInfo = exception->getInfo();
log(LogComponent::kQuery) << "assertion " << exception->toString() << " ns:" << queryMessage.ns
<< " query:" << (queryMessage.query.valid()
? queryMessage.query.toString()
: "query object is corrupt");
if (queryMessage.ntoskip || queryMessage.ntoreturn) {
log(LogComponent::kQuery) << " ntoskip:" << queryMessage.ntoskip
<< " ntoreturn:" << queryMessage.ntoreturn;
}
const SendStaleConfigException* scex = (exception->getCode() == ErrorCodes::SendStaleConfig)
? static_cast<const SendStaleConfigException*>(exception)
: NULL;
BSONObjBuilder err;
exception->getInfo().append(err);
if (scex) {
err.append("ok", 0.0);
err.append("ns", scex->getns());
scex->getVersionReceived().addToBSON(err, "vReceived");
scex->getVersionWanted().addToBSON(err, "vWanted");
}
BSONObj errObj = err.done();
if (scex) {
log(LogComponent::kQuery) << "stale version detected during query over " << queryMessage.ns
<< " : " << errObj;
}
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
bb.appendBuf((void*)errObj.objdata(), errObj.objsize());
// TODO: call replyToQuery() from here instead of this!!! see dbmessage.h
QueryResult::View msgdata = bb.buf();
bb.decouple();
QueryResult::View qr = msgdata;
qr.setResultFlags(ResultFlag_ErrSet);
if (scex)
qr.setResultFlags(qr.getResultFlags() | ResultFlag_ShardConfigStale);
qr.msgdata().setLen(bb.len());
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
response->setData(msgdata.view2ptr(), true);
}
bool ShardedClientCursor::sendNextBatch( Request& r , int ntoreturn ,
BufBuilder& buffer, int& docCount ) {
uassert( 10191 , "cursor already done" , ! _done );
int maxSize = 1024 * 1024;
if ( _totalSent > 0 )
maxSize *= 3;
docCount = 0;
// Send more if ntoreturn is 0, or any value > 1
// (one is assumed to be a single doc return, with no cursor)
bool sendMore = ntoreturn == 0 || ntoreturn > 1;
ntoreturn = abs( ntoreturn );
while ( _cursor->more() ) {
BSONObj o = _cursor->next();
buffer.appendBuf( (void*)o.objdata() , o.objsize() );
docCount++;
if ( buffer.len() > maxSize ) {
break;
}
if ( docCount == ntoreturn ) {
// soft limit aka batch size
break;
}
if ( ntoreturn == 0 && _totalSent == 0 && docCount >= 100 ) {
// first batch should be max 100 unless batch size specified
break;
}
}
bool hasMore = sendMore && _cursor->more();
LOG(5) << "\t hasMore: " << hasMore
<< " sendMore: " << sendMore
<< " cursorMore: " << _cursor->more()
<< " ntoreturn: " << ntoreturn
<< " num: " << docCount
<< " wouldSendMoreIfHad: " << sendMore
<< " id:" << getId()
<< " totalSent: " << _totalSent << endl;
_totalSent += docCount;
_done = ! hasMore;
return hasMore;
}
void DBClientBase::insert( const string & ns , BSONObj obj ) {
Message toSend;
BufBuilder b;
int opts = 0;
b.append( opts );
b.append( ns );
obj.appendSelfToBufBuilder( b );
toSend.setData( dbInsert , b.buf() , b.len() );
say( toSend );
}
void DBClientBase::insert( const string & ns , const vector< BSONObj > &v , int flags) {
Message toSend;
BufBuilder b;
b.appendNum( flags );
b.appendStr( ns );
for( vector< BSONObj >::const_iterator i = v.begin(); i != v.end(); ++i )
i->appendSelfToBufBuilder( b );
toSend.setData( dbInsert, b.buf(), b.len() );
say( toSend );
}
// Test a short NS missing a trailing null
TEST(DBMessage1, BadNS) {
BufBuilder b;
b.appendNum(static_cast<int>(1));
b.appendChar('b');
b.appendChar('a');
b.appendChar('d');
// Forget to append \0
Message toSend;
toSend.setData(dbDelete, b.buf(), b.len());
ASSERT_THROWS(DbMessage d1(toSend), UserException);
}
void assembleRequest( const string &ns, BSONObj query, int nToReturn, int nToSkip, const BSONObj *fieldsToReturn, int queryOptions, Message &toSend ) {
CHECK_OBJECT( query , "assembleRequest query" );
// see query.h for the protocol we are using here.
BufBuilder b;
int opts = queryOptions;
b.append(opts);
b.append(ns.c_str());
b.append(nToSkip);
b.append(nToReturn);
query.appendSelfToBufBuilder(b);
if ( fieldsToReturn )
fieldsToReturn->appendSelfToBufBuilder(b);
toSend.setData(dbQuery, b.buf(), b.len());
}
DBClientCursor::~DBClientCursor() {
if ( cursorId && ownCursor_ ) {
BufBuilder b;
b.append( (int)0 ); // reserved
b.append( (int)1 ); // number
b.append( cursorId );
Message m;
m.setData( dbKillCursors , b.buf() , b.len() );
connector->sayPiggyBack( m );
}
}
// THROWS
static void sayAsCmd( DBClientBase* conn, const StringData& dbName, const BSONObj& cmdObj ) {
Message toSend;
// see query.h for the protocol we are using here.
BufBuilder bufB;
bufB.appendNum( 0 ); // command/query options
bufB.appendStr( dbName.toString() + ".$cmd" ); // write command ns
bufB.appendNum( 0 ); // ntoskip (0 for command)
bufB.appendNum( 1 ); // ntoreturn (1 for command)
cmdObj.appendSelfToBufBuilder( bufB );
toSend.setData( dbQuery, bufB.buf(), bufB.len() );
// Send our command
conn->say( toSend );
}
void DBClientBase::update( const string & ns , Query query , BSONObj obj , bool upsert ) {
BufBuilder b;
b.append( (int)0 ); // reserverd
b.append( ns );
b.append( (int)upsert );
query.obj.appendSelfToBufBuilder( b );
obj.appendSelfToBufBuilder( b );
Message toSend;
toSend.setData( dbUpdate , b.buf() , b.len() );
say( toSend );
}
// Test a basic good insert, and an extra read
TEST(DBMessage1, GoodInsert) {
BufBuilder b;
string ns("test");
b.appendNum(static_cast<int>(1));
b.appendStr(ns);
b.appendNum(static_cast<int>(3));
b.appendNum(static_cast<int>(39));
Message toSend;
toSend.setData(dbInsert, b.buf(), b.len());
DbMessage d1(toSend);
ASSERT_EQUALS(3, d1.pullInt());
ASSERT_EQUALS(39, d1.pullInt());
ASSERT_THROWS(d1.pullInt(), UserException);
}
Message opCommandRequestFromOpMsgRequest(const OpMsgRequest& request) {
const auto commandName = request.getCommandName();
BufBuilder builder;
builder.skip(mongo::MsgData::MsgDataHeaderSize); // Leave room for message header.
builder.appendStr(request.getDatabase());
builder.appendStr(commandName);
// OP_COMMAND is only used when communicating with 3.4 nodes and they serialize their metadata
// fields differently. In addition to field-level differences, some generic arguments are pulled
// out to a metadata object, separate from the body. We do all down-conversion here so that the
// rest of the code only has to deal with the current format.
BSONObjBuilder metadataBuilder; // Will be appended to the message after we finish the body.
{
BSONObjBuilder bodyBuilder(builder);
for (auto elem : request.body) {
const auto fieldName = elem.fieldNameStringData();
if (fieldName == "$configServerState") {
metadataBuilder.appendAs(elem, "configsvr");
} else if (fieldName == "$readPreference") {
BSONObjBuilder ssmBuilder(metadataBuilder.subobjStart("$ssm"));
ssmBuilder.append(elem);
ssmBuilder.append("$secondaryOk",
uassertStatusOK(ReadPreferenceSetting::fromInnerBSON(elem))
.canRunOnSecondary());
} else if (fieldName == "$db") {
// skip
} else if (fieldGoesInMetadata(commandName, fieldName)) {
metadataBuilder.append(elem);
} else {
bodyBuilder.append(elem);
}
}
for (auto&& seq : request.sequences) {
invariant(seq.name.find('.') == std::string::npos); // Only support top-level for now.
dassert(!bodyBuilder.asTempObj().hasField(seq.name));
bodyBuilder.append(seq.name, seq.objs);
}
}
metadataBuilder.obj().appendSelfToBufBuilder(builder);
MsgData::View msg = builder.buf();
msg.setLen(builder.len());
msg.setOperation(dbCommand);
return Message(builder.release());
}
void DBClientCursor::requestMore() {
assert( cursorId && pos == nReturned );
BufBuilder b;
b.append(opts);
b.append(ns.c_str());
b.append(nToReturn);
b.append(cursorId);
Message toSend;
toSend.setData(dbGetMore, b.buf(), b.len());
auto_ptr<Message> response(new Message());
connector->call( toSend, *response );
m = response;
dataReceived();
}
bool DBClientCursor::init() {
Message toSend;
if ( !cursorId ) {
assembleRequest( ns, query, nToReturn, nToSkip, fieldsToReturn, opts, toSend );
} else {
BufBuilder b;
b.append( opts );
b.append( ns.c_str() );
b.append( nToReturn );
b.append( cursorId );
toSend.setData( dbGetMore, b.buf(), b.len() );
}
if ( !connector->call( toSend, *m, false ) )
return false;
dataReceived();
return true;
}
void replyToQuery( int queryResultFlags, Message& response, const BSONObj& resultObj ) {
BufBuilder bufBuilder;
bufBuilder.skip( sizeof( QueryResult ));
bufBuilder.appendBuf( reinterpret_cast< void *>(
const_cast< char* >( resultObj.objdata() )), resultObj.objsize() );
QueryResult* queryResult = reinterpret_cast< QueryResult* >( bufBuilder.buf() );
bufBuilder.decouple();
queryResult->_resultFlags() = queryResultFlags;
queryResult->len = bufBuilder.len();
queryResult->setOperation( opReply );
queryResult->cursorId = 0;
queryResult->startingFrom = 0;
queryResult->nReturned = 1;
response.setData( queryResult, true ); // transport will free
}
void replyToQuery(int queryResultFlags, Message &m, DbResponse &dbresponse, BSONObj obj) {
BufBuilder b;
b.skip(sizeof(QueryResult));
b.appendBuf((void*) obj.objdata(), obj.objsize());
QueryResult* msgdata = (QueryResult *) b.buf();
b.decouple();
QueryResult *qr = msgdata;
qr->_resultFlags() = queryResultFlags;
qr->len = b.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
Message *resp = new Message();
resp->setData(msgdata, true); // transport will free
dbresponse.response = resp;
dbresponse.responseTo = m.header()->id;
}
void replyToQuery( int queryResultFlags, Message& response, const BSONObj& resultObj ) {
BufBuilder bufBuilder;
bufBuilder.skip( sizeof( QueryResult::Value ));
bufBuilder.appendBuf( reinterpret_cast< void *>(
const_cast< char* >( resultObj.objdata() )), resultObj.objsize() );
QueryResult::View queryResult = bufBuilder.buf();
bufBuilder.decouple();
queryResult.setResultFlags(queryResultFlags);
queryResult.msgdata().setLen(bufBuilder.len());
queryResult.msgdata().setOperation( opReply );
queryResult.setCursorId(0);
queryResult.setStartingFrom(0);
queryResult.setNReturned(1);
response.setData( queryResult.view2ptr(), true ); // transport will free
}
void DBClientBase::remove( const string & ns , Query obj , bool justOne ) {
Message toSend;
BufBuilder b;
int opts = 0;
b.append( opts );
b.append( ns );
int flags = 0;
if ( justOne || obj.obj.hasField( "_id" ) )
flags |= 1;
b.append( flags );
obj.obj.appendSelfToBufBuilder( b );
toSend.setData( dbDelete , b.buf() , b.len() );
say( toSend );
}
void DBClientBase::remove( const string & ns , Query obj , bool justOne ) {
Message toSend;
BufBuilder b;
int opts = 0;
b.appendNum( opts );
b.appendStr( ns );
int flags = 0;
if ( justOne )
flags |= RemoveOption_JustOne;
b.appendNum( flags );
obj.obj.appendSelfToBufBuilder( b );
toSend.setData( dbDelete , b.buf() , b.len() );
say( toSend );
}
