/** caller handles locking */
static bool PREPLOGBUFFER(BufBuilder& bb) {
if( writes.empty() )
return false;
bb.reset();
unsigned *lenInBlockHeader;
{
// JSectHeader
bb.appendStr("\nHH\n", false);
lenInBlockHeader = (unsigned *) bb.skip(4);
}
string lastFilePath;
{
scoped_lock lk(privateViews._mutex());
for( vector<WriteIntent>::iterator i = writes.begin(); i != writes.end(); i++ ) {
size_t ofs;
MongoMMF *mmf = privateViews._find(i->p, ofs);
if( mmf == 0 ) {
journalingFailure("view pointer cannot be resolved");
}
else {
if( mmf->filePath() != lastFilePath ) {
lastFilePath = mmf->filePath();
JDbContext c;
bb.appendStruct(c);
bb.appendStr(lastFilePath);
}
JEntry e;
e.len = i->len;
e.fileNo = mmf->fileSuffixNo();
bb.appendStruct(e);
bb.appendBuf(i->p, i->len);
}
}
}
{
JSectFooter f;
f.hash = 0;
bb.appendStruct(f);
}
{
unsigned L = (bb.len() + 8191) & 0xffffe000; // fill to alignment
dassert( L >= (unsigned) bb.len() );
*lenInBlockHeader = L;
unsigned padding = L - bb.len();
bb.skip(padding);
dassert( bb.len() % 8192 == 0 );
}
writes.clear();
alreadyNoted.clear();
return true;
}
void DBClientCursor::requestMore() {
verify( cursorId && batch.pos == batch.nReturned );
if (haveLimit) {
nToReturn -= batch.nReturned;
verify(nToReturn > 0);
}
BufBuilder b;
b.appendNum(opts);
b.appendStr(ns);
b.appendNum(nextBatchSize());
b.appendNum(cursorId);
Message toSend;
toSend.setData(dbGetMore, b.buf(), b.len());
auto_ptr<Message> response(new Message());
if ( _client ) {
_client->call( toSend, *response );
this->batch.m = response;
dataReceived();
}
else {
verify( _scopedHost.size() );
ScopedDbConnection conn(_scopedHost);
conn->call( toSend , *response );
_client = conn.get();
this->batch.m = response;
dataReceived();
_client = 0;
conn.done();
}
}
void DBClientMultiCommand::sendAll() {
for ( deque<PendingCommand*>::iterator it = _pendingCommands.begin();
it != _pendingCommands.end(); ++it ) {
PendingCommand* command = *it;
dassert( NULL == command->conn );
try {
// TODO: Figure out how to handle repl sets, configs
dassert( command->endpoint.type() == ConnectionString::MASTER ||
command->endpoint.type() == ConnectionString::CUSTOM );
command->conn = shardConnectionPool.get( command->endpoint, 0 /*timeout*/);
Message toSend;
// see query.h for the protocol we are using here.
BufBuilder bufB;
bufB.appendNum( 0 ); // command/query options
bufB.appendStr( command->dbName + ".$cmd" ); // write command ns
bufB.appendNum( 0 ); // ntoskip (0 for command)
bufB.appendNum( 1 ); // ntoreturn (1 for command)
command->cmdObj.appendSelfToBufBuilder( bufB );
toSend.setData( dbQuery, bufB.buf(), bufB.len() );
// Send our command
command->conn->say( toSend );
}
catch ( const DBException& ex ) {
command->status = ex.toStatus();
if ( NULL != command->conn ) delete command->conn;
command->conn = NULL;
}
}
}
void Timestamp::append(BufBuilder& builder, const StringData& fieldName) const {
// No endian conversions needed, since we store in-memory representation
// in little endian format, regardless of target endian.
builder.appendNum(static_cast<char>(bsonTimestamp));
builder.appendStr(fieldName);
builder.appendNum(asULL());
}
void DBClientCursor::_assembleInit(Message& toSend) {
// If we haven't gotten a cursorId yet, we need to issue a new query or command.
if (!cursorId) {
// HACK:
// Unfortunately, this code is used by the shell to run commands,
// so we need to allow the shell to send invalid options so that we can
// test that the server rejects them. Thus, to allow generating commands with
// invalid options, we validate them here, and fall back to generating an OP_QUERY
// through assembleQueryRequest if the options are invalid.
bool hasValidNToReturnForCommand = (nToReturn == 1 || nToReturn == -1);
bool hasValidFlagsForCommand = !(opts & mongo::QueryOption_Exhaust);
if (_isCommand && hasValidNToReturnForCommand && hasValidFlagsForCommand) {
toSend = assembleCommandRequest(_client, nsToDatabaseSubstring(ns), opts, query);
return;
}
assembleQueryRequest(ns, query, nextBatchSize(), nToSkip, fieldsToReturn, opts, toSend);
return;
}
// Assemble a legacy getMore request.
BufBuilder b;
b.appendNum(opts);
b.appendStr(ns);
b.appendNum(nToReturn);
b.appendNum(cursorId);
toSend.setData(dbGetMore, b.buf(), b.len());
}
void DBClientCursor::requestMore() {
assert( cursorId && pos == nReturned );
if (haveLimit){
nToReturn -= nReturned;
assert(nToReturn > 0);
}
BufBuilder b;
b.appendNum(opts);
b.appendStr(ns);
b.appendNum(nextBatchSize());
b.appendNum(cursorId);
Message toSend;
toSend.setData(dbGetMore, b.buf(), b.len());
auto_ptr<Message> response(new Message());
if ( connector ){
connector->call( toSend, *response );
m = response;
dataReceived();
}
else {
assert( _scopedHost.size() );
ScopedDbConnection conn( _scopedHost );
conn->call( toSend , *response );
connector = conn.get();
m = response;
dataReceived();
connector = 0;
conn.done();
}
}
Message opCommandRequestFromOpMsgRequest(const OpMsgRequest& request) {
const auto commandName = request.getCommandName();
BufBuilder builder;
builder.skip(mongo::MsgData::MsgDataHeaderSize); // Leave room for message header.
builder.appendStr(request.getDatabase());
builder.appendStr(commandName);
// OP_COMMAND is only used when communicating with 3.4 nodes and they serialize their metadata
// fields differently. In addition to field-level differences, some generic arguments are pulled
// out to a metadata object, separate from the body. We do all down-conversion here so that the
// rest of the code only has to deal with the current format.
BSONObjBuilder metadataBuilder; // Will be appended to the message after we finish the body.
{
BSONObjBuilder bodyBuilder(builder);
for (auto elem : request.body) {
const auto fieldName = elem.fieldNameStringData();
if (fieldName == "$configServerState") {
metadataBuilder.appendAs(elem, "configsvr");
} else if (fieldName == "$readPreference") {
BSONObjBuilder ssmBuilder(metadataBuilder.subobjStart("$ssm"));
ssmBuilder.append(elem);
ssmBuilder.append("$secondaryOk",
uassertStatusOK(ReadPreferenceSetting::fromInnerBSON(elem))
.canRunOnSecondary());
} else if (fieldName == "$db") {
// skip
} else if (fieldGoesInMetadata(commandName, fieldName)) {
metadataBuilder.append(elem);
} else {
bodyBuilder.append(elem);
}
}
for (auto&& seq : request.sequences) {
invariant(seq.name.find('.') == std::string::npos); // Only support top-level for now.
dassert(!bodyBuilder.asTempObj().hasField(seq.name));
bodyBuilder.append(seq.name, seq.objs);
}
}
metadataBuilder.obj().appendSelfToBufBuilder(builder);
MsgData::View msg = builder.buf();
msg.setLen(builder.len());
msg.setOperation(dbCommand);
return Message(builder.release());
}
void Document::serializeForSorter(BufBuilder& buf) const {
const int numElems = size();
buf.appendNum(numElems);
for (DocumentStorageIterator it = storage().iterator(); !it.atEnd(); it.advance()) {
buf.appendStr(it->nameSD(), /*NUL byte*/ true);
it->val.serializeForSorter(buf);
}
}
TEST( Builder, String1 ) {
const char * big = "eliot was here";
StringData small( big, 5 );
ASSERT_EQUALS( small, "eliot" );
BufBuilder bb;
bb.appendStr( small );
ASSERT_EQUALS( 0, strcmp( bb.buf(), "eliot" ) );
ASSERT_EQUALS( 0, strcmp( "eliot", bb.buf() ) );
}
void DBClientBase::insert( const string & ns , BSONObj obj , int flags) {
Message toSend;
BufBuilder b;
b.appendNum( flags );
b.appendStr( ns );
obj.appendSelfToBufBuilder( b );
toSend.setData( dbInsert , b.buf() , b.len() );
say( toSend );
}
void DBClientBase::insert( const string & ns , const vector< BSONObj > &v , int flags) {
Message toSend;
BufBuilder b;
b.appendNum( flags );
b.appendStr( ns );
for( vector< BSONObj >::const_iterator i = v.begin(); i != v.end(); ++i )
i->appendSelfToBufBuilder( b );
toSend.setData( dbInsert, b.buf(), b.len() );
say( toSend );
}
void assembleRequest( const string &ns, BSONObj query, int nToReturn, int nToSkip, const BSONObj *fieldsToReturn, int queryOptions, Message &toSend ) {
CHECK_OBJECT( query , "assembleRequest query" );
// see query.h for the protocol we are using here.
BufBuilder b;
int opts = queryOptions;
b.appendNum(opts);
b.appendStr(ns);
b.appendNum(nToSkip);
b.appendNum(nToReturn);
query.appendSelfToBufBuilder(b);
if ( fieldsToReturn )
fieldsToReturn->appendSelfToBufBuilder(b);
toSend.setData(dbQuery, b.buf(), b.len());
}
// THROWS
static void sayAsCmd( DBClientBase* conn, const StringData& dbName, const BSONObj& cmdObj ) {
Message toSend;
// see query.h for the protocol we are using here.
BufBuilder bufB;
bufB.appendNum( 0 ); // command/query options
bufB.appendStr( dbName.toString() + ".$cmd" ); // write command ns
bufB.appendNum( 0 ); // ntoskip (0 for command)
bufB.appendNum( 1 ); // ntoreturn (1 for command)
cmdObj.appendSelfToBufBuilder( bufB );
toSend.setData( dbQuery, bufB.buf(), bufB.len() );
// Send our command
conn->say( toSend );
}
// Test a basic good insert, and an extra read
TEST(DBMessage1, GoodInsert) {
BufBuilder b;
string ns("test");
b.appendNum(static_cast<int>(1));
b.appendStr(ns);
b.appendNum(static_cast<int>(3));
b.appendNum(static_cast<int>(39));
Message toSend;
toSend.setData(dbInsert, b.buf(), b.len());
DbMessage d1(toSend);
ASSERT_EQUALS(3, d1.pullInt());
ASSERT_EQUALS(39, d1.pullInt());
ASSERT_THROWS(d1.pullInt(), UserException);
}
void DBClientBase::remove( const string & ns , Query obj , bool justOne ) {
Message toSend;
BufBuilder b;
int opts = 0;
b.appendNum( opts );
b.appendStr( ns );
int flags = 0;
if ( justOne )
flags |= RemoveOption_JustOne;
b.appendNum( flags );
obj.obj.appendSelfToBufBuilder( b );
toSend.setData( dbDelete , b.buf() , b.len() );
say( toSend );
}
StatusWith<Message> downconvertGetMoreCommandRequest(const RemoteCommandRequest& request) {
auto swGetMoreRequest = GetMoreRequest::parseFromBSON(request.dbname, request.cmdObj);
if (!swGetMoreRequest.isOK()) {
return swGetMoreRequest.getStatus();
}
auto getMoreRequest = std::move(swGetMoreRequest.getValue());
BufBuilder b;
b.appendNum(std::int32_t{0}); // reserved bits
b.appendStr(getMoreRequest.nss.ns());
// Without this static cast, we will append batchSize as an int64 and get an invalid message.
b.appendNum(static_cast<std::int32_t>(getMoreRequest.batchSize.value_or(0)));
b.appendNum(getMoreRequest.cursorid);
Message m;
m.setData(dbGetMore, b.buf(), b.len());
return {std::move(m)};
}
void Document::serializeForSorter(BufBuilder& buf) const {
const int numElems = size();
buf.appendNum(numElems);
for (DocumentStorageIterator it = storage().iterator(); !it.atEnd(); it.advance()) {
buf.appendStr(it->nameSD(), /*NUL byte*/ true);
it->val.serializeForSorter(buf);
}
if (hasTextScore()) {
buf.appendNum(char(DocumentStorage::MetaType::TEXT_SCORE + 1));
buf.appendNum(getTextScore());
}
if (hasRandMetaField()) {
buf.appendNum(char(DocumentStorage::MetaType::RAND_VAL + 1));
buf.appendNum(getRandMetaField());
}
buf.appendNum(char(0));
}
void DBClientBase::update( const string & ns , Query query , BSONObj obj , bool upsert , bool multi ) {
BufBuilder b;
b.appendNum( (int)0 ); // reserved
b.appendStr( ns );
int flags = 0;
if ( upsert ) flags |= UpdateOption_Upsert;
if ( multi ) flags |= UpdateOption_Multi;
b.appendNum( flags );
query.obj.appendSelfToBufBuilder( b );
obj.appendSelfToBufBuilder( b );
Message toSend;
toSend.setData( dbUpdate , b.buf() , b.len() );
say( toSend );
}
// Test a basic good insert, and an extra read
TEST(DBMessage1, GoodInsert2) {
BufBuilder b;
string ns("test");
b.appendNum(static_cast<int>(1));
b.appendStr(ns);
b.appendNum(static_cast<int>(3));
b.appendNum(static_cast<int>(39));
BSONObj bo = BSON("ts" << 0);
bo.appendSelfToBufBuilder(b);
Message toSend;
toSend.setData(dbInsert, b.buf(), b.len());
DbMessage d1(toSend);
ASSERT_EQUALS(3, d1.pullInt());
ASSERT_EQUALS(39, d1.pullInt());
BSONObj bo2 = d1.nextJsObj();
ASSERT_THROWS(d1.nextJsObj(), MsgAssertionException);
}
void assembleQueryRequest(const string& ns,
BSONObj query,
int nToReturn,
int nToSkip,
const BSONObj* fieldsToReturn,
int queryOptions,
Message& toSend) {
if (kDebugBuild) {
massert(10337, (string) "object not valid assembleRequest query", query.isValid());
}
// see query.h for the protocol we are using here.
BufBuilder b;
int opts = queryOptions;
b.appendNum(opts);
b.appendStr(ns);
b.appendNum(nToSkip);
b.appendNum(nToReturn);
query.appendSelfToBufBuilder(b);
if (fieldsToReturn)
fieldsToReturn->appendSelfToBufBuilder(b);
toSend.setData(dbQuery, b.buf(), b.len());
}
void NetworkInterfaceASIO::_messageFromRequest(const RemoteCommandRequest& request,
Message* toSend,
bool useOpCommand) {
BSONObj query = request.cmdObj;
invariant(query.isValid());
// TODO: Once OP_COMMAND work is complete,
// look at client to see if it supports OP_COMMAND.
// TODO: Investigate whether we can use CommandRequestBuilder here.
BufBuilder b;
b.appendNum(0); // opts
b.appendStr(request.dbname + ".$cmd");
b.appendNum(0); // toSkip
b.appendNum(1); // toReturn, don't care about responses
query.appendSelfToBufBuilder(b);
// TODO: If AsyncOp can own this buffer, we can avoid copying it in setData().
toSend->setData(dbQuery, b.buf(), b.len());
toSend->header().setId(nextMessageId());
toSend->header().setResponseTo(0);
}
