void DBClientCursor::dataReceived() {
QueryResult *qr = (QueryResult *) m->singleData();
resultFlags = qr->resultFlags();
if ( qr->resultFlags() & QueryResult::ResultFlag_CursorNotFound ) {
// cursor id no longer valid at the server.
assert( qr->cursorId == 0 );
cursorId = 0; // 0 indicates no longer valid (dead)
if ( ! ( opts & QueryOption_CursorTailable ) )
throw UserException( 13127 , "getMore: cursor didn't exist on server, possible restart or timeout?" );
}
if ( cursorId == 0 || ! ( opts & QueryOption_CursorTailable ) ) {
// only set initially: we don't want to kill it on end of data
// if it's a tailable cursor
cursorId = qr->cursorId;
}
nReturned = qr->nReturned;
pos = 0;
data = qr->data();
connector->checkResponse( data, nReturned );
/* this assert would fire the way we currently work:
assert( nReturned || cursorId == 0 );
*/
}
void Strategy::doQuery( Request& r , const Shard& shard ) {
r.checkAuth( Auth::READ );
ShardConnection dbcon( shard , r.getns() );
DBClientBase &c = dbcon.conn();
string actualServer;
Message response;
bool ok = c.call( r.m(), response, true , &actualServer );
uassert( 10200 , "mongos: error calling db", ok );
{
QueryResult *qr = (QueryResult *) response.singleData();
if ( qr->resultFlags() & ResultFlag_ShardConfigStale ) {
dbcon.done();
// Version is zero b/c this is deprecated codepath
throw RecvStaleConfigException( r.getns() , "Strategy::doQuery", ShardChunkVersion( 0 ), ShardChunkVersion( 0 ) );
}
}
r.reply( response , actualServer.size() ? actualServer : c.getServerAddress() );
dbcon.done();
}
void Strategy::doQuery( Request& r , const Shard& shard ){
try{
ShardConnection dbcon( shard , r.getns() );
DBClientBase &c = dbcon.conn();
Message response;
bool ok = c.call( r.m(), response);
{
QueryResult *qr = (QueryResult *) response.singleData();
if ( qr->resultFlags() & ResultFlag_ShardConfigStale ){
dbcon.done();
throw StaleConfigException( r.getns() , "Strategy::doQuery" );
}
}
uassert( 10200 , "mongos: error calling db", ok);
r.reply( response , c.getServerAddress() );
dbcon.done();
}
catch ( AssertionException& e ) {
BSONObjBuilder err;
e.getInfo().append( err );
BSONObj errObj = err.done();
replyToQuery(ResultFlag_ErrSet, r.p() , r.m() , errObj);
}
}
void DBClientCursor::dataReceived( bool& retry, string& host ) {
QueryResult *qr = (QueryResult *) b.m->singleData();
resultFlags = qr->resultFlags();
if ( qr->resultFlags() & ResultFlag_ErrSet ) {
wasError = true;
}
if ( qr->resultFlags() & ResultFlag_CursorNotFound ) {
// cursor id no longer valid at the server.
assert( qr->cursorId == 0 );
cursorId = 0; // 0 indicates no longer valid (dead)
if ( ! ( opts & QueryOption_CursorTailable ) )
throw UserException( 13127 , "getMore: cursor didn't exist on server, possible restart or timeout?" );
}
if ( cursorId == 0 || ! ( opts & QueryOption_CursorTailable ) ) {
// only set initially: we don't want to kill it on end of data
// if it's a tailable cursor
cursorId = qr->cursorId;
}
b.nReturned = qr->nReturned;
b.pos = 0;
b.data = qr->data();
_client->checkResponse( b.data, b.nReturned, &retry, &host ); // watches for "not master"
if( qr->resultFlags() & ResultFlag_ShardConfigStale ) {
BSONObj error;
assert( peekError( &error ) );
throw RecvStaleConfigException( error["ns"].String(),
(string)"stale config on lazy receive" + causedBy( getErrField( error ) ) );
}
/* this assert would fire the way we currently work:
assert( nReturned || cursorId == 0 );
*/
}
void DBClientCursor::dataReceived() {
QueryResult *qr = (QueryResult *) m->data;
resultFlags = qr->resultFlags();
if ( qr->resultFlags() & QueryResult::ResultFlag_CursorNotFound ) {
// cursor id no longer valid at the server.
assert( qr->cursorId == 0 );
cursorId = 0; // 0 indicates no longer valid (dead)
}
if ( cursorId == 0 || ! ( opts & Option_CursorTailable ) ) {
// only set initially: we don't want to kill it on end of data
// if it's a tailable cursor
cursorId = qr->cursorId;
}
nReturned = qr->nReturned;
pos = 0;
data = qr->data();
connector->checkResponse( data, nReturned );
/* this assert would fire the way we currently work:
assert( nReturned || cursorId == 0 );
*/
}
/**
* Returns true if request is a query for sharded indexes.
*/
static bool doShardedIndexQuery( Request& r, const QuerySpec& qSpec ) {
// Extract the ns field from the query, which may be embedded within the "query" or
// "$query" field.
string indexNSQuery(qSpec.filter()["ns"].str());
DBConfigPtr config = grid.getDBConfig( r.getns() );
if ( !config->isSharded( indexNSQuery )) {
return false;
}
// if you are querying on system.indexes, we need to make sure we go to a shard
// that actually has chunks. This is not a perfect solution (what if you just
// look at all indexes), but better than doing nothing.
ShardPtr shard;
ChunkManagerPtr cm;
config->getChunkManagerOrPrimary( indexNSQuery, cm, shard );
if ( cm ) {
set<Shard> shards;
cm->getAllShards( shards );
verify( shards.size() > 0 );
shard.reset( new Shard( *shards.begin() ) );
}
ShardConnection dbcon( *shard , r.getns() );
DBClientBase &c = dbcon.conn();
string actualServer;
Message response;
bool ok = c.call( r.m(), response, true , &actualServer );
uassert( 10200 , "mongos: error calling db", ok );
{
QueryResult *qr = (QueryResult *) response.singleData();
if ( qr->resultFlags() & ResultFlag_ShardConfigStale ) {
dbcon.done();
// Version is zero b/c this is deprecated codepath
throw RecvStaleConfigException( r.getns(),
"Strategy::doQuery",
ChunkVersion( 0, 0, OID() ),
ChunkVersion( 0, 0, OID() ));
}
}
r.reply( response , actualServer.size() ? actualServer : c.getServerAddress() );
dbcon.done();
return true;
}
bool handlePossibleShardedMessage( Message &m, DbResponse &dbresponse ){
int op = m.data->operation();
if ( op < 2000 || op >= 3000 )
return false;
const char *ns = m.data->_data + 4;
string errmsg;
if ( shardVersionOk( ns , errmsg ) )
return false;
if ( doesOpGetAResponse( op ) ){
BufBuilder b( 32768 );
b.skip( sizeof( QueryResult ) );
{
BSONObj obj = BSON( "$err" << errmsg );
b.append( obj.objdata() , obj.objsize() );
}
QueryResult *qr = (QueryResult*)b.buf();
qr->resultFlags() = QueryResult::ResultFlag_ErrSet | QueryResult::ResultFlag_ShardConfigStale;
qr->len = b.len();
qr->setOperation( opReply );
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
b.decouple();
Message * resp = new Message();
resp->setData( qr , true );
dbresponse.response = resp;
dbresponse.responseTo = m.data->id;
return true;
}
OID * clientID = clientServerIds.get();
massert( "write with bad shard config and no server id!" , clientID );
log() << "got write with an old config - writing back" << endl;
BSONObjBuilder b;
b.appendBool( "writeBack" , true );
b.append( "ns" , ns );
b.appendBinData( "msg" , m.data->len , bdtCustom , (char*)(m.data) );
log() << "writing back msg with len: " << m.data->len << " op: " << m.data->_operation << endl;
clientQueues[clientID->str()]->push( b.obj() );
return true;
}
void Strategy::doQuery( Request& r , const Shard& shard ) {
ShardConnection dbcon( shard , r.getns() );
DBClientBase &c = dbcon.conn();
string actualServer;
Message response;
bool ok = c.call( r.m(), response, true , &actualServer );
uassert( 10200 , "mongos: error calling db", ok );
{
QueryResult *qr = (QueryResult *) response.singleData();
if ( qr->resultFlags() & ResultFlag_ShardConfigStale ) {
dbcon.done();
throw StaleConfigException( r.getns() , "Strategy::doQuery" );
}
}
r.reply( response , actualServer.size() ? actualServer : c.getServerAddress() );
dbcon.done();
}
void receivedQuery(DbResponse& dbresponse, /*AbstractMessagingPort& dbMsgPort, */Message& m, stringstream& ss, bool logit) {
MSGID responseTo = m.data->id;
DbMessage d(m);
QueryMessage q(d);
QueryResult* msgdata;
try {
if (q.fields.get() && q.fields->errmsg)
uassert(q.fields->errmsg, false);
/* note these are logged BEFORE authentication -- which is sort of ok */
if ( _diaglog.level && logit ) {
if ( strstr(q.ns, ".$cmd") ) {
/* $cmd queries are "commands" and usually best treated as write operations */
OPWRITE;
}
else {
OPREAD;
}
}
setClient( q.ns );
Client& client = cc();
client.top.setRead();
strncpy(client.curop()->ns, q.ns, Namespace::MaxNsLen);
msgdata = runQuery(m, ss ).release();
}
catch ( AssertionException& e ) {
ss << " exception ";
LOGSOME problem() << " Caught Assertion in runQuery ns:" << q.ns << ' ' << e.toString() << '\n';
log() << " ntoskip:" << q.ntoskip << " ntoreturn:" << q.ntoreturn << '\n';
if ( q.query.valid() )
log() << " query:" << q.query.toString() << endl;
else
log() << " query object is not valid!" << endl;
BSONObjBuilder err;
err.append("$err", e.msg.empty() ? "assertion during query" : e.msg);
BSONObj errObj = err.done();
BufBuilder b;
b.skip(sizeof(QueryResult));
b.append((void*) errObj.objdata(), errObj.objsize());
// todo: call replyToQuery() from here instead of this!!! see dbmessage.h
msgdata = (QueryResult *) b.buf();
b.decouple();
QueryResult *qr = msgdata;
qr->resultFlags() = QueryResult::ResultFlag_ErrSet;
qr->len = b.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
}
Message *resp = new Message();
resp->setData(msgdata, true); // transport will free
dbresponse.response = resp;
dbresponse.responseTo = responseTo;
Database *database = cc().database();
if ( database ) {
if ( database->profile )
ss << " bytes:" << resp->data->dataLen();
}
else {
if ( strstr(q.ns, "$cmd") == 0 ) // (this condition is normal for $cmd dropDatabase)
log() << "ERROR: receiveQuery: database is null; ns=" << q.ns << endl;
}
}
void processMessage( Connection& c , Message& m ) {
AuditingDbMessage d(m);
if ( m.operation() == mongo::opReply )
out() << " - " << (unsigned)m.header()->responseTo;
out() << '\n';
try {
switch( m.operation() ) {
case mongo::opReply: {
mongo::QueryResult* r = (mongo::QueryResult*)m.singleData();
out() << "\treply" << " n:" << r->nReturned << " cursorId: " << r->cursorId << endl;
if ( r->nReturned ) {
mongo::BSONObj o( r->data() );
out() << "\t" << o << endl;
}
break;
}
case mongo::dbQuery: {
mongo::QueryMessage q(d);
out() << "\tquery: " << q.query << " ntoreturn: " << q.ntoreturn << " ntoskip: " << q.ntoskip;
if( !q.fields.isEmpty() )
out() << " hasfields";
if( q.queryOptions & mongo::QueryOption_SlaveOk )
out() << " SlaveOk";
if( q.queryOptions & mongo::QueryOption_NoCursorTimeout )
out() << " NoCursorTimeout";
if( q.queryOptions & ~(mongo::QueryOption_SlaveOk | mongo::QueryOption_NoCursorTimeout) )
out() << " queryOptions:" << hex << q.queryOptions;
out() << endl;
break;
}
case mongo::dbUpdate: {
int flags = d.pullInt();
BSONObj q = d.nextJsObj( "update" );
BSONObj o = d.nextJsObj( "update" );
out() << "\tupdate flags:" << flags << " q:" << q << " o:" << o << endl;
break;
}
case mongo::dbInsert: {
out() << "\tinsert: " << d.nextJsObj( "insert" ) << endl;
while ( d.moreJSObjs() ) {
out() << "\t\t" << d.nextJsObj( "insert" ) << endl;
}
break;
}
case mongo::dbGetMore: {
int nToReturn = d.pullInt();
long long cursorId = d.pullInt64();
out() << "\tgetMore nToReturn: " << nToReturn << " cursorId: " << cursorId << endl;
break;
}
case mongo::dbDelete: {
int flags = d.pullInt();
BSONObj q = d.nextJsObj( "delete" );
out() << "\tdelete flags: " << flags << " q: " << q << endl;
break;
}
case mongo::dbKillCursors: {
int *x = (int *) m.singleData()->_data;
x++; // reserved
int n = *x;
out() << "\tkillCursors n: " << n << endl;
break;
}
default:
out() << "\tunknown opcode " << m.operation() << endl;
cerr << "*** CANNOT HANDLE TYPE: " << m.operation() << endl;
}
}
catch ( ... ) {
cerr << "Error parsing message for operation: " << m.operation() << endl;
}
if ( !forwardAddress.empty() ) {
if ( m.operation() != mongo::opReply ) {
boost::shared_ptr<DBClientConnection> conn = forwarder[ c ];
if ( !conn ) {
conn.reset(new DBClientConnection( true ));
conn->connect( forwardAddress );
forwarder[ c ] = conn;
}
if ( m.operation() == mongo::dbQuery || m.operation() == mongo::dbGetMore ) {
if ( m.operation() == mongo::dbGetMore ) {
DbMessage d( m );
d.pullInt();
mongo::little<long long> &cId = d.pullInt64();
cId = mapCursor[ c ][ cId ];
}
Message response;
conn->port().call( m, response );
QueryResult *qr = (QueryResult *) response.singleData();
if ( !( qr->resultFlags() & mongo::ResultFlag_CursorNotFound ) ) {
if ( qr->cursorId != 0 ) {
lastCursor[ c ] = qr->cursorId;
return;
}
}
lastCursor[ c ] = 0;
}
else {
conn->port().say( m );
}
}
else {
Connection r = c.reverse();
long long myCursor = lastCursor[ r ];
QueryResult *qr = (QueryResult *) m.singleData();
long long yourCursor = qr->cursorId;
if ( ( qr->resultFlags() & mongo::ResultFlag_CursorNotFound ) )
yourCursor = 0;
if ( myCursor && !yourCursor )
cerr << "Expected valid cursor in sniffed response, found none" << endl;
if ( !myCursor && yourCursor )
cerr << "Sniffed valid cursor when none expected" << endl;
if ( myCursor && yourCursor ) {
mapCursor[ r ][ qr->cursorId ] = lastCursor[ r ];
lastCursor[ r ] = 0;
}
}
}
}
void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *packet){
const struct sniff_ip* ip = (struct sniff_ip*)(packet + captureHeaderSize);
int size_ip = IP_HL(ip)*4;
if ( size_ip < 20 ){
cerr << "*** Invalid IP header length: " << size_ip << " bytes" << endl;
return;
}
assert( ip->ip_p == IPPROTO_TCP );
const struct sniff_tcp* tcp = (struct sniff_tcp*)(packet + captureHeaderSize + size_ip);
int size_tcp = TH_OFF(tcp)*4;
if (size_tcp < 20){
cerr << "*** Invalid TCP header length: " << size_tcp << " bytes" << endl;
return;
}
if ( ! ( serverPorts.count( ntohs( tcp->th_sport ) ) ||
serverPorts.count( ntohs( tcp->th_dport ) ) ) ){
return;
}
const u_char * payload = (const u_char*)(packet + captureHeaderSize + size_ip + size_tcp);
unsigned totalSize = ntohs(ip->ip_len);
assert( totalSize <= header->caplen );
int size_payload = totalSize - (size_ip + size_tcp);
if (size_payload <= 0 )
return;
Connection c;
c.srcAddr = ip->ip_src;
c.srcPort = tcp->th_sport;
c.dstAddr = ip->ip_dst;
c.dstPort = tcp->th_dport;
if ( seen[ c ] ) {
if ( expectedSeq[ c ] != ntohl( tcp->th_seq ) ) {
cerr << "Warning: sequence # mismatch, there may be dropped packets" << endl;
}
} else {
seen[ c ] = true;
}
expectedSeq[ c ] = ntohl( tcp->th_seq ) + size_payload;
Message m;
if ( bytesRemainingInMessage[ c ] == 0 ) {
m.setData( (MsgData*)payload , false );
if ( !m.data->valid() ) {
cerr << "Invalid message start, skipping packet." << endl;
return;
}
if ( size_payload > m.data->len ) {
cerr << "Multiple messages in packet, skipping packet." << endl;
return;
}
if ( size_payload < m.data->len ) {
bytesRemainingInMessage[ c ] = m.data->len - size_payload;
messageBuilder[ c ].reset( new BufBuilder() );
messageBuilder[ c ]->append( (void*)payload, size_payload );
return;
}
} else {
bytesRemainingInMessage[ c ] -= size_payload;
messageBuilder[ c ]->append( (void*)payload, size_payload );
if ( bytesRemainingInMessage[ c ] < 0 ) {
cerr << "Received too many bytes to complete message, resetting buffer" << endl;
bytesRemainingInMessage[ c ] = 0;
messageBuilder[ c ].reset();
return;
}
if ( bytesRemainingInMessage[ c ] > 0 )
return;
m.setData( (MsgData*)messageBuilder[ c ]->buf(), true );
messageBuilder[ c ]->decouple();
messageBuilder[ c ].reset();
}
DbMessage d( m );
cout << inet_ntoa(ip->ip_src) << ":" << ntohs( tcp->th_sport )
<< ( serverPorts.count( ntohs( tcp->th_dport ) ) ? " -->> " : " <<-- " )
<< inet_ntoa(ip->ip_dst) << ":" << ntohs( tcp->th_dport )
<< " " << d.getns()
<< " " << m.data->len << " bytes "
<< m.data->id;
if ( m.data->operation() == mongo::opReply )
cout << " - " << m.data->responseTo;
cout << endl;
switch( m.data->operation() ){
case mongo::opReply:{
mongo::QueryResult* r = (mongo::QueryResult*)m.data;
cout << "\treply" << " n:" << r->nReturned << " cursorId: " << r->cursorId << endl;
if ( r->nReturned ){
mongo::BSONObj o( r->data() , 0 );
cout << "\t" << o << endl;
}
break;
}
case mongo::dbQuery:{
mongo::QueryMessage q(d);
cout << "\tquery: " << q.query << endl;
break;
}
case mongo::dbUpdate:{
int flags = d.pullInt();
BSONObj q = d.nextJsObj();
BSONObj o = d.nextJsObj();
cout << "\tupdate flags:" << flags << " q:" << q << " o:" << o << endl;
break;
}
case mongo::dbInsert:{
cout << "\tinsert: " << d.nextJsObj() << endl;
while ( d.moreJSObjs() )
cout << "\t\t" << d.nextJsObj() << endl;
break;
}
case mongo::dbGetMore:{
int nToReturn = d.pullInt();
long long cursorId = d.pullInt64();
cout << "\tgetMore nToReturn: " << nToReturn << " cursorId: " << cursorId << endl;
break;
}
case mongo::dbDelete:{
int flags = d.pullInt();
BSONObj q = d.nextJsObj();
cout << "\tdelete flags: " << flags << " q: " << q << endl;
break;
}
case mongo::dbKillCursors:{
int *x = (int *) m.data->_data;
x++; // reserved
int n = *x;
cout << "\tkillCursors n: " << n << endl;
break;
}
default:
cerr << "*** CANNOT HANDLE TYPE: " << m.data->operation() << endl;
}
if ( !forwardAddress.empty() ) {
if ( m.data->operation() != mongo::opReply ) {
DBClientConnection *conn = forwarder[ c ];
if ( !conn ) {
// These won't get freed on error, oh well hopefully we'll just
// abort in that case anyway.
conn = new DBClientConnection( true );
conn->connect( forwardAddress );
forwarder[ c ] = conn;
}
if ( m.data->operation() == mongo::dbQuery || m.data->operation() == mongo::dbGetMore ) {
if ( m.data->operation() == mongo::dbGetMore ) {
DbMessage d( m );
d.pullInt();
long long &cId = d.pullInt64();
cId = mapCursor[ c ][ cId ];
}
Message response;
conn->port().call( m, response );
QueryResult *qr = (QueryResult *) response.data;
if ( !( qr->resultFlags() & QueryResult::ResultFlag_CursorNotFound ) ) {
if ( qr->cursorId != 0 ) {
lastCursor[ c ] = qr->cursorId;
return;
}
}
lastCursor[ c ] = 0;
} else {
conn->port().say( m );
}
} else {
Connection r = c.reverse();
long long myCursor = lastCursor[ r ];
QueryResult *qr = (QueryResult *) m.data;
long long yourCursor = qr->cursorId;
if ( ( qr->resultFlags() & QueryResult::ResultFlag_CursorNotFound ) )
yourCursor = 0;
if ( myCursor && !yourCursor )
cerr << "Expected valid cursor in sniffed response, found none" << endl;
if ( !myCursor && yourCursor )
cerr << "Sniffed valid cursor when none expected" << endl;
if ( myCursor && yourCursor ) {
mapCursor[ r ][ qr->cursorId ] = lastCursor[ r ];
lastCursor[ r ] = 0;
}
}
}
}
QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid , stringstream& ss) {
ClientCursor *cc = ClientCursor::find(cursorid);
int bufSize = 512;
if ( cc ){
bufSize += sizeof( QueryResult );
bufSize += ( ntoreturn ? 4 : 1 ) * 1024 * 1024;
}
BufBuilder b( bufSize );
b.skip(sizeof(QueryResult));
int resultFlags = 0;
int start = 0;
int n = 0;
if ( !cc ) {
log() << "getMore: cursorid not found " << ns << " " << cursorid << endl;
cursorid = 0;
resultFlags = QueryResult::ResultFlag_CursorNotFound;
}
else {
ss << " query: " << cc->query << " ";
start = cc->pos;
Cursor *c = cc->c.get();
c->checkLocation();
while ( 1 ) {
if ( !c->ok() ) {
if ( c->tailable() ) {
if ( c->advance() ) {
continue;
}
break;
}
bool ok = ClientCursor::erase(cursorid);
assert(ok);
cursorid = 0;
cc = 0;
break;
}
if ( !cc->matcher->matches(c->currKey(), c->currLoc() ) ) {
}
else {
//out() << "matches " << c->currLoc().toString() << '\n';
if( c->getsetdup(c->currLoc()) ) {
//out() << " but it's a dup \n";
}
else {
BSONObj js = c->current();
fillQueryResultFromObj(b, cc->filter.get(), js);
n++;
if ( (ntoreturn>0 && (n >= ntoreturn || b.len() > MaxBytesToReturnToClientAtOnce)) ||
(ntoreturn==0 && b.len()>1*1024*1024) ) {
c->advance();
cc->pos += n;
//cc->updateLocation();
break;
}
}
}
c->advance();
}
if ( cc ) {
cc->updateLocation();
cc->mayUpgradeStorage();
}
}
QueryResult *qr = (QueryResult *) b.buf();
qr->len = b.len();
qr->setOperation(opReply);
qr->resultFlags() = resultFlags;
qr->cursorId = cursorid;
qr->startingFrom = start;
qr->nReturned = n;
b.decouple();
return qr;
}