void rollbackTransactionFromOplog(BSONObj entry, bool purgeEntry) {
bool transactionAlreadyApplied = entry["a"].Bool();
Client::Transaction transaction(DB_SERIALIZABLE);
if (transactionAlreadyApplied) {
if (entry.hasElement("ref")) {
rollbackRefOp(entry);
} else if (entry.hasElement("ops")) {
rollbackOps(entry["ops"].Array());
} else {
verify(0);
}
}
{
LOCK_REASON(lockReason, "repl: purging entry from oplog");
Lock::DBRead lk1("local", lockReason);
if (purgeEntry) {
purgeEntryFromOplog(entry);
}
else {
// set the applied bool to false, to let the oplog know that
// this entry has not been applied to collections
BSONElementManipulator(entry["a"]).setBool(false);
writeEntryToOplog(entry, false);
}
}
transaction.commit(DB_TXN_NOSYNC);
}
bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) {
bool all = *cmdObj.firstElement().valuestrsafe() == '*';
int before = result.len();
// TODO: convert to ServerParameters -- SERVER-10515
if (isJournalingEnabled() && (all || cmdObj.hasElement("journalCommitInterval")) &&
!isMongos()) {
result.append("journalCommitInterval",
getJournalCommitInterval());
}
if( all || cmdObj.hasElement( "traceExceptions" ) ) {
result.append("traceExceptions",
DBException::traceExceptions);
}
if( all || cmdObj.hasElement( "replMonitorMaxFailedChecks" ) ) {
result.append("replMonitorMaxFailedChecks",
ReplicaSetMonitor::getMaxFailedChecks());
}
const ServerParameter::Map& m = ServerParameterSet::getGlobal()->getMap();
for ( ServerParameter::Map::const_iterator i = m.begin(); i != m.end(); ++i ) {
if ( all || cmdObj.hasElement( i->first.c_str() ) ) {
i->second->append( result, i->second->name() );
}
}
if ( before == result.len() ) {
errmsg = "no option found to get";
return false;
}
return true;
}
virtual bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
log() << "replSet replSetTest command received: " << cmdObj.toString() << rsLog;
if (!checkAuth(errmsg, result)) {
return false;
}
if( cmdObj.hasElement("forceInitialSyncFailure") ) {
replSetForceInitialSyncFailure = (unsigned) cmdObj["forceInitialSyncFailure"].Number();
return true;
}
if( !check(errmsg, result) )
return false;
if( cmdObj.hasElement("blind") ) {
replSetBlind = cmdObj.getBoolField("blind");
return true;
}
if (cmdObj.hasElement("sethbmsg")) {
replset::sethbmsg(cmdObj["sethbmsg"].String());
return true;
}
return false;
}
// takes an entry that was written _logTransactionOps
// and applies them to collections
//
// TODO: possibly improve performance of this. We create and destroy a
// context for each operation. Find a way to amortize it out if necessary
//
void applyTransactionFromOplog(BSONObj entry) {
bool transactionAlreadyApplied = entry["a"].Bool();
if (!transactionAlreadyApplied) {
Client::Transaction transaction(DB_SERIALIZABLE);
if (entry.hasElement("ref")) {
applyRefOp(entry);
} else if (entry.hasElement("ops")) {
applyOps(entry["ops"].Array());
} else {
verify(0);
}
// set the applied bool to true, to let the oplog know that
// this entry has been applied to collections
BSONElementManipulator(entry["a"]).setBool(true);
{
LOCK_REASON(lockReason, "repl: setting oplog entry's applied bit");
Lock::DBRead lk1("local", lockReason);
writeEntryToOplog(entry, false);
}
// If this code fails, it is impossible to recover from
// because we don't know if the transaction successfully committed
// so we might as well crash
// There is currently no known way this code can throw an exception
try {
// we are operating as a secondary. We don't have to fsync
transaction.commit(DB_TXN_NOSYNC);
}
catch (std::exception &e) {
log() << "exception during commit of applyTransactionFromOplog, aborting system: " << e.what() << endl;
printStackTrace();
logflush();
::abort();
}
}
}
bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ) {
bool all = *cmdObj.firstElement().valuestrsafe() == '*';
int before = result.len();
if( all || cmdObj.hasElement("quiet") ) {
result.append("quiet", cmdLine.quiet );
}
if( all || cmdObj.hasElement("notablescan") ) {
result.append("notablescan", cmdLine.noTableScan);
}
if( all || cmdObj.hasElement("logLevel") ) {
result.append("logLevel", logLevel);
}
if( all || cmdObj.hasElement("syncdelay") ) {
result.append("syncdelay", cmdLine.syncdelay);
}
if( all || cmdObj.hasElement("replApplyBatchSize") ) {
result.append("replApplyBatchSize", replApplyBatchSize);
}
if ( before == result.len() ) {
errmsg = "no option found to get";
return false;
}
return true;
}
Config::OutputOptions Config::parseOutputOptions(const std::string& dbname,
const BSONObj& cmdObj) {
Config::OutputOptions outputOptions;
outputOptions.outNonAtomic = false;
if (cmdObj["out"].type() == String) {
outputOptions.collectionName = cmdObj["out"].String();
outputOptions.outType = REPLACE;
}
else if (cmdObj["out"].type() == Object) {
BSONObj o = cmdObj["out"].embeddedObject();
BSONElement e = o.firstElement();
string t = e.fieldName();
if (t == "normal" || t == "replace") {
outputOptions.outType = REPLACE;
outputOptions.collectionName = e.String();
}
else if (t == "merge") {
outputOptions.outType = MERGE;
outputOptions.collectionName = e.String();
}
else if (t == "reduce") {
outputOptions.outType = REDUCE;
outputOptions.collectionName = e.String();
}
else if (t == "inline") {
outputOptions.outType = INMEMORY;
}
else {
uasserted(13522,
mongoutils::str::stream() << "unknown out specifier [" << t << "]");
}
if (o.hasElement("db")) {
outputOptions.outDB = o["db"].String();
}
if (o.hasElement("nonAtomic")) {
outputOptions.outNonAtomic = o["nonAtomic"].Bool();
if (outputOptions.outNonAtomic)
uassert(15895,
"nonAtomic option cannot be used with this output type",
(outputOptions.outType == REDUCE ||
outputOptions.outType == MERGE));
}
}
else {
uasserted(13606 , "'out' has to be a string or an object");
}
if (outputOptions.outType != INMEMORY) {
outputOptions.finalNamespace = mongoutils::str::stream()
<< (outputOptions.outDB.empty() ? dbname : outputOptions.outDB)
<< "." << outputOptions.collectionName;
}
return outputOptions;
}
virtual bool run(const string& db, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
string coll = cmdObj.firstElement().valuestr();
if( coll.empty() || db.empty() ) {
errmsg = "no collection name specified";
return false;
}
if( isCurrentlyAReplSetPrimary() && !cmdObj["force"].trueValue() ) {
errmsg = "will not run compact on an active replica set primary as this is a slow blocking operation. use force:true to force";
return false;
}
string ns = db + '.' + coll;
if ( ! NamespaceString::normal(ns.c_str()) ) {
errmsg = "bad namespace name";
return false;
}
// parameter validation to avoid triggering assertions in compact()
if ( str::contains(ns, ".system.") ) {
errmsg = "can't compact a system namespace";
return false;
}
{
writelock lk;
Client::Context ctx(ns);
NamespaceDetails *d = nsdetails(ns.c_str());
if( ! d ) {
errmsg = "namespace does not exist";
return false;
}
if ( d->capped ) {
errmsg = "cannot compact a capped collection";
return false;
}
}
double pf = 1.0;
int pb = 0;
if( cmdObj.hasElement("paddingFactor") ) {
pf = cmdObj["paddingFactor"].Number();
assert( pf >= 1.0 && pf <= 4.0 );
}
if( cmdObj.hasElement("paddingBytes") ) {
pb = (int) cmdObj["paddingBytes"].Number();
assert( pb >= 0 && pb <= 1024 * 1024 );
}
bool validate = !cmdObj.hasElement("validate") || cmdObj["validate"].trueValue(); // default is true at the moment
bool ok = compact(ns, errmsg, validate, result, pf, pb);
return ok;
}
void up(const BSONObj& info, HeartbeatInfo& mem) {
HeartbeatInfo::numPings++;
mem.authIssue = false;
if( mem.upSince == 0 ) {
log() << "replSet member " << h.toString() << " is up" << rsLog;
mem.upSince = mem.lastHeartbeat;
}
mem.health = 1.0;
mem.lastHeartbeatMsg = info["hbmsg"].String();
if (info.hasElement("syncingTo")) {
mem.syncingTo = info["syncingTo"].String();
}
if( info.hasElement("opTime") )
mem.opTime = info["opTime"].Date();
// see if this member is in the electable set
if( info["e"].eoo() ) {
// for backwards compatibility
const Member *member = theReplSet->findById(mem.id());
if (member && member->config().potentiallyHot()) {
theReplSet->addToElectable(mem.id());
}
else {
theReplSet->rmFromElectable(mem.id());
}
}
// add this server to the electable set if it is within 10
// seconds of the latest optime we know of
else if( info["e"].trueValue() &&
mem.opTime >= theReplSet->lastOpTimeWritten.getSecs() - 10) {
unsigned lastOp = theReplSet->lastOtherOpTime().getSecs();
if (lastOp > 0 && mem.opTime >= lastOp - 10) {
theReplSet->addToElectable(mem.id());
}
}
else {
theReplSet->rmFromElectable(mem.id());
}
be cfg = info["config"];
if( cfg.ok() ) {
// received a new config
boost::function<void()> f =
boost::bind(&Manager::msgReceivedNewConfig, theReplSet->mgr, cfg.Obj().copy());
theReplSet->mgr->send(f);
}
}
bool Query::isComplex(const BSONObj& obj, bool* hasDollar) {
if (obj.hasElement("query")) {
if (hasDollar)
*hasDollar = false;
return true;
}
if (obj.hasElement("$query")) {
if (hasDollar)
*hasDollar = true;
return true;
}
return false;
}
virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
log() << "replSet replSetTest command received: " << cmdObj.toString() << rsLog;
if( cmdObj.hasElement("forceInitialSyncFailure") ) {
replSetForceInitialSyncFailure = (unsigned) cmdObj["forceInitialSyncFailure"].Number();
return true;
}
// may not need this, but if removed check all tests still work:
if( !check(errmsg, result) )
return false;
if( cmdObj.hasElement("blind") ) {
replSetBlind = cmdObj.getBoolField("blind");
return true;
}
return false;
}
virtual bool errmsgRun(OperationContext* opCtx,
const string& db,
const BSONObj& cmdObj,
string& errmsg,
BSONObjBuilder& result) {
NamespaceString nss = CommandHelpers::parseNsCollectionRequired(db, cmdObj);
repl::ReplicationCoordinator* replCoord = repl::ReplicationCoordinator::get(opCtx);
if (replCoord->getMemberState().primary() && !cmdObj["force"].trueValue()) {
errmsg =
"will not run compact on an active replica set primary as this is a slow blocking "
"operation. use force:true to force";
return false;
}
if (!nss.isNormal()) {
errmsg = "bad namespace name";
return false;
}
if (nss.isSystem()) {
// Items in system.* cannot be moved as there might be pointers to them.
errmsg = "can't compact a system namespace";
return false;
}
CompactOptions compactOptions;
if (cmdObj.hasElement("validate"))
compactOptions.validateDocuments = cmdObj["validate"].trueValue();
AutoGetDb autoDb(opCtx, db, MODE_X);
Database* const collDB = autoDb.getDb();
Collection* collection = collDB ? collDB->getCollection(opCtx, nss) : nullptr;
auto view =
collDB && !collection ? ViewCatalog::get(collDB)->lookup(opCtx, nss.ns()) : nullptr;
// If db/collection does not exist, short circuit and return.
if (!collDB || !collection) {
if (view)
uasserted(ErrorCodes::CommandNotSupportedOnView, "can't compact a view");
else
uasserted(ErrorCodes::NamespaceNotFound, "collection does not exist");
}
OldClientContext ctx(opCtx, nss.ns());
BackgroundOperation::assertNoBgOpInProgForNs(nss.ns());
log() << "compact " << nss.ns() << " begin, options: " << compactOptions;
StatusWith<CompactStats> status = compactCollection(opCtx, collection, &compactOptions);
uassertStatusOK(status.getStatus());
log() << "compact " << nss.ns() << " end";
return true;
}
void replicateFullTransactionToOplog(BSONObj& o, OplogReader& r, bool* bigTxn) {
*bigTxn = false;
if (o.hasElement("ref")) {
OID oid = o["ref"].OID();
LOG(3) << "oplog ref " << oid << endl;
copyOplogRefsRange(r, oid);
*bigTxn = true;
}
Client::ReadContext ctx(rsoplog);
replicateTransactionToOplog(o);
}
bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
int s = 0;
if( cmdObj.hasElement("notablescan") ) {
result.append("was", cmdLine.noTableScan);
cmdLine.noTableScan = cmdObj["notablescan"].Bool();
s++;
}
if( cmdObj.hasElement("quiet") ) {
result.append("was", cmdLine.quiet );
cmdLine.quiet = cmdObj["quiet"].Bool();
s++;
}
if( cmdObj.hasElement("syncdelay") ) {
result.append("was", cmdLine.syncdelay );
cmdLine.syncdelay = cmdObj["syncdelay"].Number();
s++;
}
if( cmdObj.hasElement( "logLevel" ) ) {
result.append("was", logLevel );
logLevel = cmdObj["logLevel"].numberInt();
s++;
}
if( cmdObj.hasElement( "replApplyBatchSize" ) ) {
result.append("was", replApplyBatchSize );
BSONElement e = cmdObj["replApplyBatchSize"];
ParameterValidator * v = ParameterValidator::get( e.fieldName() );
assert( v );
if ( ! v->isValid( e , errmsg ) )
return false;
replApplyBatchSize = e.numberInt();
s++;
}
if( s == 0 ) {
errmsg = "no option found to set, use '*' to get all ";
return false;
}
return true;
}
bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) {
int s = 0;
bool found = setParmsMongodSpecific(dbname, cmdObj, errmsg, result, fromRepl);
if( cmdObj.hasElement("journalCommitInterval") ) {
if( !cmdLine.dur ) {
errmsg = "journaling is off";
return false;
}
int x = (int) cmdObj["journalCommitInterval"].Number();
assert( x > 1 && x < 500 );
cmdLine.journalCommitInterval = x;
log() << "setParameter journalCommitInterval=" << x << endl;
s++;
}
if( cmdObj.hasElement("notablescan") ) {
assert( !cmdLine.isMongos() );
if( s == 0 )
result.append("was", cmdLine.noTableScan);
cmdLine.noTableScan = cmdObj["notablescan"].Bool();
s++;
}
if( cmdObj.hasElement("quiet") ) {
if( s == 0 )
result.append("was", cmdLine.quiet );
cmdLine.quiet = cmdObj["quiet"].Bool();
s++;
}
if( cmdObj.hasElement("syncdelay") ) {
assert( !cmdLine.isMongos() );
if( s == 0 )
result.append("was", cmdLine.syncdelay );
cmdLine.syncdelay = cmdObj["syncdelay"].Number();
s++;
}
if( cmdObj.hasElement( "logLevel" ) ) {
if( s == 0 )
result.append("was", logLevel );
logLevel = cmdObj["logLevel"].numberInt();
s++;
}
if( cmdObj.hasElement( "replApplyBatchSize" ) ) {
if( s == 0 )
result.append("was", replApplyBatchSize );
BSONElement e = cmdObj["replApplyBatchSize"];
ParameterValidator * v = ParameterValidator::get( e.fieldName() );
assert( v );
if ( ! v->isValid( e , errmsg ) )
return false;
replApplyBatchSize = e.numberInt();
s++;
}
if( s == 0 && !found ) {
errmsg = "no option found to set, use help:true to see options ";
return false;
}
return true;
}
void replicateFullTransactionToOplog(BSONObj& o, OplogReader& r, bool* bigTxn) {
*bigTxn = false;
if (o.hasElement("ref")) {
OID oid = o["ref"].OID();
LOG(3) << "oplog ref " << oid << endl;
copyOplogRefsRange(r, oid);
*bigTxn = true;
}
LOCK_REASON(lockReason, "repl: copying entry to local oplog");
Client::ReadContext ctx(rsoplog, lockReason);
replicateTransactionToOplog(o);
}
/**********************************************************
*reads from db and converts bson to FileRec object
*
***********************************************************/
void FileRec::readFromDB(mongo::DBClientConnection& conn, string filename) {
boost::filesystem::path p(filename); //get filename from path
string file(p.filename().c_str());
auto_ptr<mongo::DBClientCursor> cursor = conn.query("fileRecords.Filerec", MONGO_QUERY("filename" << file));
if (cursor->more()) {
BSONObj record = cursor->next();
//get data from db and store in the FileRec
this->filename = record.getStringField("filename");
this->tempname = record.getStringField("Tempname");
this->recentHash = record.getStringField("curhash");
this->origHash = record.getStringField("ovhash");
this->length = record.getIntField("length");
this->versionCount = record.getIntField("nversions");
this->modifytime.tv_nsec = record.getField("Mtnsec").numberLong();
this->modifytime.tv_sec = record.getField("mtsec").numberLong();
this->refNum = record.getIntField("currentversion");
vector<BSONElement> hashes(record.getField("FileBlkHashes").Array());
for (vector<BSONElement>::iterator it = hashes.begin(); it != hashes.end(); ++it) {
appendBlock((*it).String());
}
//comments is an array of objects so it takes a bit of nesting to convert
vector<BSONElement> array = record["comments"].Array(); //convert to array
for (vector<BSONElement>::iterator ar = array.begin(); ar != array.end(); ++ar) {
BSONObj commentdata = ar->Obj(); //store object at array[x] into BSONObj
BSONElement version = commentdata.getField("version"); //convert
BSONElement commentdb = commentdata.getField("comment");
comment data;
data.comment = commentdb.String();
data.version = version.Int();
appendComment(data);
}
if (record.hasElement("versionrec")) { //again an array of objects
vector<BSONElement> array = record["versionrec"].Array();
for (vector<BSONElement>::iterator it = array.begin(); it != array.end(); ++it) {
BSONObj versionRecord = it->Obj();
BSONElement id = versionRecord.getField("id");
appendVersion(id.String());
}
}
}
}
virtual bool run(OperationContext* txn, const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
log() << "replSet replSetTest command received: " << cmdObj.toString() << rsLog;
if( cmdObj.hasElement("forceInitialSyncFailure") ) {
replSetForceInitialSyncFailure = (unsigned) cmdObj["forceInitialSyncFailure"].Number();
return true;
}
Status status = getGlobalReplicationCoordinator()->checkReplEnabledForCommand(&result);
if (!status.isOK())
return appendCommandStatus(result, status);
return false;
}
virtual bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
if( cmdObj.hasElement("expireOplogDays") || cmdObj.hasElement("expireOplogHours") ) {
uint32_t expireOplogDays = cmdLine.expireOplogDays;
uint32_t expireOplogHours = cmdLine.expireOplogHours;
if (cmdObj.hasElement("expireOplogHours")) {
BSONElement e = cmdObj["expireOplogHours"];
if (!e.isNumber()) {
errmsg = "bad expireOplogHours";
return false;
}
expireOplogHours = e.numberLong();
}
if (cmdObj.hasElement("expireOplogDays")) {
BSONElement e = cmdObj["expireOplogDays"];
if (!e.isNumber()) {
errmsg = "bad expireOplogDays";
return false;
}
expireOplogDays = e.numberLong();
}
theReplSet->changeExpireOplog(expireOplogDays, expireOplogHours);
}
return true;
}
bool run(const string& dbname, BSONObj& obj, int options, string& errmsg, BSONObjBuilder& result, bool fromRepl ) {
if( !obj.hasElement("servers") ) {
vector<string> ips;
obj["servers"].Obj().Vals(ips);
{
SimpleMutex::scoped_lock lk(cloud::mtx);
cloud::ips.ref(lk).swap(ips);
if( !cloud::startedThread ) {
cloud::startedThread = true;
boost::thread thr(cloud::thread);
}
}
}
return true;
}
virtual bool run(const string& db, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
string coll = cmdObj.firstElement().valuestr();
if( coll.empty() || db.empty() ) {
errmsg = "no collection name specified";
return false;
}
if( isCurrentlyAReplSetPrimary() && !cmdObj["force"].trueValue() ) {
errmsg = "will not run compact on an active replica set primary as this is a slow blocking operation. use force:true to force";
return false;
}
string ns = db + '.' + coll;
bool validate = !cmdObj.hasElement("validate") || cmdObj["validate"].trueValue(); // default is true at the moment
bool ok = compact(ns, errmsg, validate, result);
return ok;
}
void purgeEntryFromOplog(BSONObj entry) {
verify(rsOplogDetails);
if (entry.hasElement("ref")) {
OID oid = entry["ref"].OID();
Helpers::removeRange(
rsOplogRefs,
BSON("_id" << BSON("oid" << oid << "seq" << 0)),
BSON("_id" << BSON("oid" << oid << "seq" << LLONG_MAX)),
BSON("_id" << 1),
true,
false
);
}
BSONObj pk = entry["_id"].wrap("");
uint64_t flags = (NamespaceDetails::NO_LOCKTREE);
rsOplogDetails->deleteObject(pk, entry, flags);
}
bool _requestHeartbeat(HeartbeatInfo& mem, BSONObj& info, int& theirConfigVersion) {
if (tries++ % threshold == (threshold - 1)) {
ScopedConn conn(h.toString());
conn.reconnect();
}
Timer timer;
time_t before = curTimeMicros64() / 1000000;
bool ok = requestHeartbeat(theReplSet->name(), theReplSet->selfFullName(),
h.toString(), info, theReplSet->config().version, theirConfigVersion);
mem.ping = (unsigned int)timer.millis();
// we set this on any response - we don't get this far if
// couldn't connect because exception is thrown
time_t after = mem.lastHeartbeat = before + (mem.ping / 1000);
if ( info["time"].isNumber() ) {
long long t = info["time"].numberLong();
if( t > after )
mem.skew = (int) (t - after);
else if( t < before )
mem.skew = (int) (t - before); // negative
}
else {
// it won't be there if remote hasn't initialized yet
if( info.hasElement("time") )
warning() << "heatbeat.time isn't a number: " << info << endl;
mem.skew = INT_MIN;
}
{
be state = info["state"];
if( state.ok() )
mem.hbstate = MemberState(state.Int());
}
return ok;
}
void SyncClusterConnection::_checkLast() {
_lastErrors.clear();
vector<string> errors;
for ( size_t i=0; i<_conns.size(); i++ ) {
BSONObj res;
string err;
try {
if ( ! _conns[i]->runCommand( "admin" , BSON( "getlasterror" << 1 << "fsync" << 1 ) , res ) )
err = "cmd failed: ";
}
catch ( std::exception& e ) {
err += e.what();
}
catch ( ... ) {
err += "unknown failure";
}
_lastErrors.push_back( res.getOwned() );
errors.push_back( err );
}
verify( _lastErrors.size() == errors.size() && _lastErrors.size() == _conns.size() );
stringstream err;
bool ok = true;
for ( size_t i = 0; i<_conns.size(); i++ ) {
BSONObj res = _lastErrors[i];
if ( res["ok"].trueValue() && (res["fsyncFiles"].numberInt() > 0 ||
res.hasElement("waited") ||
res["syncMillis"].numberInt() >= 0 ) )
continue;
ok = false;
err << _conns[i]->toString() << ": " << res << " " << errors[i];
}
if ( ok )
return;
throw UserException( 8001 , (string)"SyncClusterConnection write op failed: " + err.str() );
}
void purgeEntryFromOplog(BSONObj entry) {
verify(rsOplogDetails);
if (entry.hasElement("ref")) {
OID oid = entry["ref"].OID();
LOCK_REASON(lockReason, "repl: purging oplog.refs for oplog entry");
Client::ReadContext ctx(rsOplogRefs, lockReason);
Client::Transaction txn(DB_SERIALIZABLE);
deleteIndexRange(
rsOplogRefs,
BSON("_id" << BSON("oid" << oid << "seq" << 0)),
BSON("_id" << BSON("oid" << oid << "seq" << LLONG_MAX)),
BSON("_id" << 1),
true,
false
);
txn.commit();
}
BSONObj pk = entry["_id"].wrap("");
const uint64_t flags = Collection::NO_LOCKTREE;
rsOplogDetails->deleteObject(pk, entry, flags);
}
bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result) {
bool all = *cmdObj.firstElement().valuestrsafe() == '*';
int before = result.len();
const ServerParameter::Map& m = ServerParameterSet::getGlobal()->getMap();
for (ServerParameter::Map::const_iterator i = m.begin(); i != m.end(); ++i) {
if (all || cmdObj.hasElement(i->first.c_str())) {
i->second->append(txn, result, i->second->name());
}
}
if (before == result.len()) {
errmsg = "no option found to get";
return false;
}
return true;
}
void doWork() {
if ( !theReplSet ) {
log(2) << "theReplSet not initialized yet, skipping health poll this round" << rsLog;
return;
}
HeartbeatInfo mem = m;
HeartbeatInfo old = mem;
try {
BSONObj info;
int theirConfigVersion = -10000;
Timer timer;
bool ok = requestHeartbeat(theReplSet->name(), theReplSet->selfFullName(), h.toString(), info, theReplSet->config().version, theirConfigVersion);
mem.ping = (unsigned int)timer.micros();
time_t before = timer.startTime() / 1000000;
// we set this on any response - we don't get this far if
// couldn't connect because exception is thrown
time_t after = mem.lastHeartbeat = before + (mem.ping / 1000000);
if ( info["time"].isNumber() ) {
long long t = info["time"].numberLong();
if( t > after )
mem.skew = (int) (t - after);
else if( t < before )
mem.skew = (int) (t - before); // negative
}
else {
// it won't be there if remote hasn't initialized yet
if( info.hasElement("time") )
warning() << "heatbeat.time isn't a number: " << info << endl;
mem.skew = INT_MIN;
}
{
be state = info["state"];
if( state.ok() )
mem.hbstate = MemberState(state.Int());
}
if( ok ) {
if( mem.upSince == 0 ) {
log() << "replSet info " << h.toString() << " is up" << rsLog;
mem.upSince = mem.lastHeartbeat;
}
mem.health = 1.0;
mem.lastHeartbeatMsg = info["hbmsg"].String();
if( info.hasElement("opTime") )
mem.opTime = info["opTime"].Date();
be cfg = info["config"];
if( cfg.ok() ) {
// received a new config
boost::function<void()> f =
boost::bind(&Manager::msgReceivedNewConfig, theReplSet->mgr, cfg.Obj().copy());
theReplSet->mgr->send(f);
}
}
else {
down(mem, info.getStringField("errmsg"));
}
}
catch(DBException& e) {
down(mem, e.what());
}
catch(...) {
down(mem, "something unusual went wrong");
}
m = mem;
theReplSet->mgr->send( boost::bind(&ReplSet::msgUpdateHBInfo, theReplSet, mem) );
static time_t last = 0;
time_t now = time(0);
bool changed = mem.changed(old);
if( changed ) {
if( old.hbstate != mem.hbstate )
log() << "replSet member " << h.toString() << ' ' << mem.hbstate.toString() << rsLog;
}
if( changed || now-last>4 ) {
last = now;
theReplSet->mgr->send( boost::bind(&Manager::msgCheckNewState, theReplSet->mgr) );
}
}
/** Note: if the object shrinks a lot, we don't free up space, we leave extra at end of the record.
*/
const DiskLoc DataFileMgr::updateRecord(
const char *ns,
Collection* collection,
Record *toupdate, const DiskLoc& dl,
const char *_buf, int _len, OpDebug& debug, bool god) {
dassert( toupdate == dl.rec() );
BSONObj objOld = BSONObj::make(toupdate);
BSONObj objNew(_buf);
DEV verify( objNew.objsize() == _len );
DEV verify( objNew.objdata() == _buf );
if( !objNew.hasElement("_id") && objOld.hasElement("_id") ) {
/* add back the old _id value if the update removes it. Note this implementation is slow
(copies entire object multiple times), but this shouldn't happen often, so going for simple
code, not speed.
*/
BSONObjBuilder b;
BSONElement e;
verify( objOld.getObjectID(e) );
b.append(e); // put _id first, for best performance
b.appendElements(objNew);
objNew = b.obj();
}
NamespaceString nsstring(ns);
if (nsstring.coll() == "system.users") {
V2UserDocumentParser parser;
uassertStatusOK(parser.checkValidUserDocument(objNew));
}
uassert( 13596 , str::stream() << "cannot change _id of a document old:" << objOld << " new:" << objNew,
objNew["_id"] == objOld["_id"]);
/* duplicate key check. we descend the btree twice - once for this check, and once for the actual inserts, further
below. that is suboptimal, but it's pretty complicated to do it the other way without rollbacks...
*/
OwnedPointerVector<UpdateTicket> updateTickets;
updateTickets.mutableVector().resize(collection->details()->getTotalIndexCount());
for (int i = 0; i < collection->details()->getTotalIndexCount(); ++i) {
auto_ptr<IndexDescriptor> descriptor(CatalogHack::getDescriptor(collection->details(), i));
auto_ptr<IndexAccessMethod> iam(CatalogHack::getIndex(descriptor.get()));
InsertDeleteOptions options;
options.logIfError = false;
options.dupsAllowed = !(KeyPattern::isIdKeyPattern(descriptor->keyPattern())
|| descriptor->unique())
|| ignoreUniqueIndex(descriptor->getOnDisk());
updateTickets.mutableVector()[i] = new UpdateTicket();
Status ret = iam->validateUpdate(objOld, objNew, dl, options,
updateTickets.mutableVector()[i]);
if (Status::OK() != ret) {
uasserted(ASSERT_ID_DUPKEY, "Update validation failed: " + ret.toString());
}
}
if ( toupdate->netLength() < objNew.objsize() ) {
// doesn't fit. reallocate -----------------------------------------------------
moveCounter.increment();
uassert( 10003,
"failing update: objects in a capped ns cannot grow",
!(collection && collection->details()->isCapped()));
collection->details()->paddingTooSmall();
deleteRecord(ns, toupdate, dl);
DiskLoc res = insert(ns, objNew.objdata(), objNew.objsize(), false, god);
if (debug.nmoved == -1) // default of -1 rather than 0
debug.nmoved = 1;
else
debug.nmoved += 1;
return res;
}
collection->infoCache()->notifyOfWriteOp();
collection->details()->paddingFits();
debug.keyUpdates = 0;
for (int i = 0; i < collection->details()->getTotalIndexCount(); ++i) {
auto_ptr<IndexDescriptor> descriptor(CatalogHack::getDescriptor(collection->details(), i));
auto_ptr<IndexAccessMethod> iam(CatalogHack::getIndex(descriptor.get()));
int64_t updatedKeys;
Status ret = iam->update(*updateTickets.vector()[i], &updatedKeys);
if (Status::OK() != ret) {
// This shouldn't happen unless something disastrous occurred.
massert(16799, "update failed: " + ret.toString(), false);
}
debug.keyUpdates += updatedKeys;
}
// update in place
int sz = objNew.objsize();
memcpy(getDur().writingPtr(toupdate->data(), sz), objNew.objdata(), sz);
return dl;
}
Config::Config( const string& _dbname , const BSONObj& cmdObj ) {
dbname = _dbname;
ns = dbname + "." + cmdObj.firstElement().valuestr();
verbose = cmdObj["verbose"].trueValue();
uassert( 13602 , "outType is no longer a valid option" , cmdObj["outType"].eoo() );
if ( cmdObj["out"].type() == String ) {
finalShort = cmdObj["out"].String();
outType = REPLACE;
}
else if ( cmdObj["out"].type() == Object ) {
BSONObj o = cmdObj["out"].embeddedObject();
BSONElement e = o.firstElement();
string t = e.fieldName();
if ( t == "normal" || t == "replace" ) {
outType = REPLACE;
finalShort = e.String();
}
else if ( t == "merge" ) {
outType = MERGE;
finalShort = e.String();
}
else if ( t == "reduce" ) {
outType = REDUCE;
finalShort = e.String();
}
else if ( t == "inline" ) {
outType = INMEMORY;
}
else {
uasserted( 13522 , str::stream() << "unknown out specifier [" << t << "]" );
}
if (o.hasElement("db")) {
outDB = o["db"].String();
}
}
else {
uasserted( 13606 , "'out' has to be a string or an object" );
}
if ( outType != INMEMORY ) { // setup names
tempLong = str::stream() << (outDB.empty() ? dbname : outDB) << ".tmp.mr." << cmdObj.firstElement().String() << "_" << finalShort << "_" << JOB_NUMBER++;
incLong = tempLong + "_inc";
finalLong = str::stream() << (outDB.empty() ? dbname : outDB) << "." << finalShort;
}
{
// scope and code
if ( cmdObj["scope"].type() == Object )
scopeSetup = cmdObj["scope"].embeddedObjectUserCheck();
mapper.reset( new JSMapper( cmdObj["map"] ) );
reducer.reset( new JSReducer( cmdObj["reduce"] ) );
if ( cmdObj["finalize"].type() && cmdObj["finalize"].trueValue() )
finalizer.reset( new JSFinalizer( cmdObj["finalize"] ) );
if ( cmdObj["mapparams"].type() == Array ) {
mapParams = cmdObj["mapparams"].embeddedObjectUserCheck();
}
}
{
// query options
BSONElement q = cmdObj["query"];
if ( q.type() == Object )
filter = q.embeddedObjectUserCheck();
else
uassert( 13608 , "query has to be blank or an Object" , ! q.trueValue() );
BSONElement s = cmdObj["sort"];
if ( s.type() == Object )
sort = s.embeddedObjectUserCheck();
else
uassert( 13609 , "sort has to be blank or an Object" , ! s.trueValue() );
if ( cmdObj["limit"].isNumber() )
limit = cmdObj["limit"].numberLong();
else
limit = 0;
}
}
void up(const BSONObj& info, HeartbeatInfo& mem, bool* needsNewStateChecked) {
HeartbeatInfo::numPings++;
mem.authIssue = false;
if( mem.upSince == 0 ) {
log() << "replSet member " << h.toString() << " is up" << rsLog;
mem.upSince = mem.lastHeartbeat;
}
mem.health = 1.0;
mem.lastHeartbeatMsg = info["hbmsg"].String();
if (info.hasElement("syncingTo")) {
mem.syncingTo = info["syncingTo"].String();
}
if( info.hasElement("opTime") ) {
mem.opTime = info["opTime"].Date();
}
if ( info.hasElement("GTID")) {
mem.gtid = getGTIDFromBSON("GTID", info);
}
if ( info.hasElement("lastUnappliedGTID")) {
mem.lastUnappliedGTID = getGTIDFromBSON("lastUnappliedGTID", info);
}
if ( info.hasElement("minLiveGTID")) {
mem.minLiveGTID= getGTIDFromBSON("minLiveGTID", info);
}
if ( info.hasElement("minUnappliedGTID")) {
mem.minUnappliedGTID= getGTIDFromBSON("minUnappliedGTID", info);
}
if ( info.hasElement("oplogVersion")) {
mem.oplogVersion = info["oplogVersion"].numberLong();
}
else {
mem.oplogVersion = 0;
}
// for "highest known primary"
if ( info.hasElement("hkp")) {
mem.highestKnownPrimaryInSet = info["hkp"].numberLong();
// if the highest known primary across the replica set has changed,
// communicate that to the caller so that Manager::msgCheckNewState
// eventually gets called
*needsNewStateChecked = theReplSet->handleHighestKnownPrimaryOfMember(mem.highestKnownPrimaryInSet);
}
else {
mem.highestKnownPrimaryInSet = 0;
}
// see if this member is in the electable set
if( info["e"].eoo() ) {
// for backwards compatibility
const Member *member = theReplSet->findById(mem.id());
if (member && member->config().potentiallyHot()) {
theReplSet->addToElectable(mem.id());
}
else {
theReplSet->rmFromElectable(mem.id());
}
}
// add this server to the electable set if it is within 10
// seconds of the latest optime we know of
else if( info["e"].trueValue() &&
mem.opTime + 10000 >= (theReplSet->gtidManager ? theReplSet->gtidManager->getCurrTimestamp() : 0))
{
unsigned lastOp = theReplSet->lastOtherOpTime();
if (lastOp > 0 && mem.opTime + 10000 >= lastOp) {
theReplSet->addToElectable(mem.id());
}
}
else {
theReplSet->rmFromElectable(mem.id());
}
be cfg = info["config"];
if( cfg.ok() ) {
// received a new config
boost::function<void()> f =
boost::bind(&Manager::msgReceivedNewConfig, theReplSet->mgr, cfg.Obj().copy());
theReplSet->mgr->send(f);
}
}
bool _requestHeartbeat(HeartbeatInfo& mem, BSONObj& info, int& theirConfigVersion) {
{
ScopedConn conn(h.toString());
conn.setTimeout(_timeout);
if (tries++ % threshold == (threshold - 1)) {
conn.reconnect();
}
}
Timer timer;
time_t before = curTimeMicros64() / 1000000;
bool ok = tryHeartbeat(&info, &theirConfigVersion);
mem.ping = static_cast<unsigned int>(timer.millis());
time_t totalSecs = mem.ping / 1000;
// if that didn't work and we have more time, lower timeout and try again
if (!ok && totalSecs < _timeout) {
log() << "replset info " << h.toString() << " heartbeat failed, retrying" << rsLog;
// lower timeout to remaining ping time
{
ScopedConn conn(h.toString());
conn.setTimeout(_timeout - totalSecs);
}
int checkpoint = timer.millis();
timer.reset();
ok = tryHeartbeat(&info, &theirConfigVersion);
mem.ping = static_cast<unsigned int>(timer.millis());
totalSecs = (checkpoint + mem.ping)/1000;
// set timeout back to default
{
ScopedConn conn(h.toString());
conn.setTimeout(_timeout);
}
}
// we set this on any response - we don't get this far if
// couldn't connect because exception is thrown
time_t after = mem.lastHeartbeat = before + totalSecs;
if ( info["time"].isNumber() ) {
long long t = info["time"].numberLong();
if( t > after )
mem.skew = (int) (t - after);
else if( t < before )
mem.skew = (int) (t - before); // negative
}
else {
// it won't be there if remote hasn't initialized yet
if( info.hasElement("time") )
warning() << "heatbeat.time isn't a number: " << info << endl;
mem.skew = INT_MIN;
}
{
be state = info["state"];
if( state.ok() )
mem.hbstate = MemberState(state.Int());
}
if (info.hasField("stateDisagreement") && info["stateDisagreement"].trueValue()) {
log() << "replset info " << h.toString() << " thinks that we are down" << endl;
}
return ok;
}
