Status SimpleRecordStoreV1::truncate(OperationContext* txn) {
const DiskLoc firstExtLoc = _details->firstExtent(txn);
if (firstExtLoc.isNull() || !firstExtLoc.isValid()) {
// Already empty
return Status::OK();
}
// Free all extents except the first.
Extent* firstExt = _extentManager->getExtent(firstExtLoc);
if (!firstExt->xnext.isNull()) {
const DiskLoc extNextLoc = firstExt->xnext;
const DiskLoc oldLastExtLoc = _details->lastExtent(txn);
Extent* const nextExt = _extentManager->getExtent(extNextLoc);
// Unlink other extents;
*txn->recoveryUnit()->writing(&nextExt->xprev) = DiskLoc();
*txn->recoveryUnit()->writing(&firstExt->xnext) = DiskLoc();
_details->setLastExtent(txn, firstExtLoc);
_details->setLastExtentSize(txn, firstExt->length);
_extentManager->freeExtents(txn, extNextLoc, oldLastExtLoc);
}
// Make the first (now only) extent a single large deleted record.
*txn->recoveryUnit()->writing(&firstExt->firstRecord) = DiskLoc();
*txn->recoveryUnit()->writing(&firstExt->lastRecord) = DiskLoc();
_details->orphanDeletedList(txn);
addDeletedRec(txn, _findFirstSpot(txn, firstExtLoc, firstExt));
// Make stats reflect that there are now no documents in this record store.
_details->setStats(txn, 0, 0);
return Status::OK();
}
Status addKey(const BSONObj& key, const DiskLoc& loc) {
// inserts should be in ascending (key, DiskLoc) order.
if ( key.objsize() >= TempKeyMaxSize ) {
return Status(ErrorCodes::KeyTooLong, "key too big");
}
invariant(!loc.isNull());
invariant(loc.isValid());
invariant(!hasFieldNames(key));
if (!_data->empty()) {
// Compare specified key with last inserted key, ignoring its DiskLoc
int cmp = _comparator.compare(IndexKeyEntry(key, DiskLoc()), *_last);
if (cmp < 0 || (_dupsAllowed && cmp == 0 && loc < _last->loc)) {
return Status(ErrorCodes::InternalError,
"expected ascending (key, DiskLoc) order in bulk builder");
}
else if (!_dupsAllowed && cmp == 0 && loc != _last->loc) {
return dupKeyError(key);
}
}
BSONObj owned = key.getOwned();
_last = _data->insert(_data->end(), IndexKeyEntry(owned, loc));
*_currentKeySize += key.objsize();
return Status::OK();
}
virtual Status insert(OperationContext* txn,
const BSONObj& key,
const DiskLoc& loc,
bool dupsAllowed) {
invariant(!loc.isNull());
invariant(loc.isValid());
invariant(!hasFieldNames(key));
if ( key.objsize() >= TempKeyMaxSize ) {
string msg = mongoutils::str::stream()
<< "Heap1Btree::insert: key too large to index, failing "
<< ' ' << key.objsize() << ' ' << key;
return Status(ErrorCodes::KeyTooLong, msg);
}
// TODO optimization: save the iterator from the dup-check to speed up insert
if (!dupsAllowed && isDup(*_data, key, loc))
return dupKeyError(key);
BSONObj owned = key.getOwned();
if ( _data->insert(IndexKeyEntry(owned, loc)).second ) {
_currentKeySize += key.objsize();
Heap1RecoveryUnit::notifyIndexInsert( txn, this, owned, loc );
}
return Status::OK();
}
virtual bool unindex(OperationContext* txn, const BSONObj& key, const DiskLoc& loc) {
invariant(!loc.isNull());
invariant(loc.isValid());
invariant(!hasFieldNames(key));
const size_t numDeleted = _data->erase(IndexEntry(key, loc));
invariant(numDeleted <= 1);
if ( numDeleted == 1 )
_currentKeySize -= key.objsize();
return numDeleted == 1;
}
virtual bool unindex(OperationContext* txn, const BSONObj& key, const DiskLoc& loc) {
invariant(!loc.isNull());
invariant(loc.isValid());
invariant(!hasFieldNames(key));
const size_t numDeleted = _data->erase(IndexKeyEntry(key, loc));
invariant(numDeleted <= 1);
if ( numDeleted == 1 ) {
_currentKeySize -= key.objsize();
Heap1RecoveryUnit::notifyIndexRemove( txn, this, key, loc );
}
return numDeleted == 1;
}
Status addKey(const BSONObj& key, const DiskLoc& loc) {
// inserts should be in ascending order.
if ( key.objsize() >= TempKeyMaxSize ) {
return Status(ErrorCodes::KeyTooLong, "key too big");
}
invariant(!loc.isNull());
invariant(loc.isValid());
invariant(!hasFieldNames(key));
// TODO optimization: dup check can assume dup is only possible with last inserted key
// and avoid the log(n) lookup.
if (!_dupsAllowed && isDup(*_data, key, loc))
return dupKeyError(key);
_data->insert(_data->end(), IndexEntry(key.getOwned(), loc));
*_currentKeySize += key.objsize();
return Status::OK();
}
Status getOplogStartHack(OperationContext* txn,
Collection* collection,
CanonicalQuery* cq,
PlanExecutor** execOut) {
invariant(cq);
auto_ptr<CanonicalQuery> autoCq(cq);
if ( collection == NULL )
return Status(ErrorCodes::InternalError,
"getOplogStartHack called with a NULL collection" );
// A query can only do oplog start finding if it has a top-level $gt or $gte predicate over
// the "ts" field (the operation's timestamp). Find that predicate and pass it to
// the OplogStart stage.
MatchExpression* tsExpr = NULL;
if (MatchExpression::AND == cq->root()->matchType()) {
// The query has an AND at the top-level. See if any of the children
// of the AND are $gt or $gte predicates over 'ts'.
for (size_t i = 0; i < cq->root()->numChildren(); ++i) {
MatchExpression* me = cq->root()->getChild(i);
if (isOplogTsPred(me)) {
tsExpr = me;
break;
}
}
}
else if (isOplogTsPred(cq->root())) {
// The root of the tree is a $gt or $gte predicate over 'ts'.
tsExpr = cq->root();
}
if (NULL == tsExpr) {
return Status(ErrorCodes::OplogOperationUnsupported,
"OplogReplay query does not contain top-level "
"$gt or $gte over the 'ts' field.");
}
DiskLoc startLoc = DiskLoc().setInvalid();
// See if the RecordStore supports the oplogStartHack
const BSONElement tsElem = extractOplogTsOptime(tsExpr);
if (tsElem.type() == Timestamp) {
StatusWith<DiskLoc> goal = oploghack::keyForOptime(tsElem._opTime());
if (goal.isOK()) {
startLoc = collection->getRecordStore()->oplogStartHack(txn, goal.getValue());
}
}
if (startLoc.isValid()) {
LOG(3) << "Using direct oplog seek";
}
else {
LOG(3) << "Using OplogStart stage";
// Fallback to trying the OplogStart stage.
WorkingSet* oplogws = new WorkingSet();
OplogStart* stage = new OplogStart(txn, collection, tsExpr, oplogws);
PlanExecutor* rawExec;
// Takes ownership of oplogws and stage.
Status execStatus = PlanExecutor::make(txn, oplogws, stage, collection,
PlanExecutor::YIELD_AUTO, &rawExec);
invariant(execStatus.isOK());
scoped_ptr<PlanExecutor> exec(rawExec);
// The stage returns a DiskLoc of where to start.
PlanExecutor::ExecState state = exec->getNext(NULL, &startLoc);
// This is normal. The start of the oplog is the beginning of the collection.
if (PlanExecutor::IS_EOF == state) {
return getExecutor(txn, collection, autoCq.release(), PlanExecutor::YIELD_AUTO,
execOut);
}
// This is not normal. An error was encountered.
if (PlanExecutor::ADVANCED != state) {
return Status(ErrorCodes::InternalError,
"quick oplog start location had error...?");
}
}
// cout << "diskloc is " << startLoc.toString() << endl;
// Build our collection scan...
CollectionScanParams params;
params.collection = collection;
params.start = startLoc;
params.direction = CollectionScanParams::FORWARD;
params.tailable = cq->getParsed().getOptions().tailable;
WorkingSet* ws = new WorkingSet();
CollectionScan* cs = new CollectionScan(txn, params, ws, cq->root());
// Takes ownership of 'ws', 'cs', and 'cq'.
return PlanExecutor::make(txn, ws, cs, autoCq.release(), collection,
PlanExecutor::YIELD_AUTO, execOut);
}
string validateNS(const char *ns, NamespaceDetails *d, BSONObj *cmdObj) {
bool scanData = true;
if( cmdObj && cmdObj->hasElement("scandata") && !cmdObj->getBoolField("scandata") )
scanData = false;
bool valid = true;
stringstream ss;
ss << "\nvalidate\n";
//ss << " details: " << hex << d << " ofs:" << nsindex(ns)->detailsOffset(d) << dec << endl;
if ( d->capped )
ss << " capped:" << d->capped << " max:" << d->max << '\n';
ss << " firstExtent:" << d->firstExtent.toString() << " ns:" << d->firstExtent.ext()->nsDiagnostic.toString()<< '\n';
ss << " lastExtent:" << d->lastExtent.toString() << " ns:" << d->lastExtent.ext()->nsDiagnostic.toString() << '\n';
try {
d->firstExtent.ext()->assertOk();
d->lastExtent.ext()->assertOk();
DiskLoc el = d->firstExtent;
int ne = 0;
while( !el.isNull() ) {
Extent *e = el.ext();
e->assertOk();
el = e->xnext;
ne++;
killCurrentOp.checkForInterrupt();
}
ss << " # extents:" << ne << '\n';
}
catch (...) {
valid=false;
ss << " extent asserted ";
}
ss << " datasize?:" << d->stats.datasize << " nrecords?:" << d->stats.nrecords << " lastExtentSize:" << d->lastExtentSize << '\n';
ss << " padding:" << d->paddingFactor << '\n';
try {
try {
ss << " first extent:\n";
d->firstExtent.ext()->dump(ss);
valid = valid && d->firstExtent.ext()->validates();
}
catch (...) {
ss << "\n exception firstextent\n" << endl;
}
set<DiskLoc> recs;
if( scanData ) {
shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
int n = 0;
long long len = 0;
long long nlen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
while ( c->ok() ) {
n++;
DiskLoc cl = c->currLoc();
if ( n < 1000000 )
recs.insert(cl);
if ( d->capped ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = c->_current();
len += r->lengthWithHeaders;
nlen += r->netLength();
c->advance();
}
if ( d->capped && !d->capLooped() ) {
ss << " capped outOfOrder:" << outOfOrder;
if ( outOfOrder > 1 ) {
valid = false;
ss << " ???";
}
else ss << " (OK)";
ss << '\n';
}
ss << " " << n << " objects found, nobj:" << d->stats.nrecords << '\n';
ss << " " << len << " bytes data w/headers\n";
ss << " " << nlen << " bytes data wout/headers\n";
}
ss << " deletedList: ";
for ( int i = 0; i < Buckets; i++ ) {
ss << (d->deletedList[i].isNull() ? '0' : '1');
}
ss << endl;
int ndel = 0;
long long delSize = 0;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = d->deletedList[i];
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( d->capped && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
if ( loc.a() <= 0 || strstr(ns, "hudsonSmall") == 0 ) {
ss << " ?bad deleted loc: " << loc.toString() << " bucket:" << i << " k:" << k << endl;
valid = false;
break;
}
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders;
loc = d->nextDeleted;
k++;
killCurrentOp.checkForInterrupt();
}
}
catch (...) {
ss <<" ?exception in deleted chain for bucket " << i << endl;
valid = false;
}
}
ss << " deleted: n: " << ndel << " size: " << delSize << endl;
if ( incorrect ) {
ss << " ?corrupt: " << incorrect << " records from datafile are in deleted list\n";
valid = false;
}
int idxn = 0;
try {
ss << " nIndexes:" << d->nIndexes << endl;
NamespaceDetails::IndexIterator i = d->ii();
while( i.more() ) {
IndexDetails& id = i.next();
ss << " " << id.indexNamespace() << " keys:" <<
id.head.btree()->fullValidate(id.head, id.keyPattern()) << endl;
}
}
catch (...) {
ss << "\n exception during index validate idxn:" << idxn << endl;
valid=false;
}
}
catch (AssertionException) {
ss << "\n exception during validate\n" << endl;
valid = false;
}
if ( !valid )
ss << " ns corrupt, requires dbchk\n";
return ss.str();
}
void validateNS(const string& ns,
Collection* collection,
const BSONObj& cmdObj,
BSONObjBuilder& result) {
const bool full = cmdObj["full"].trueValue();
const bool scanData = full || cmdObj["scandata"].trueValue();
NamespaceDetails* nsd = collection->details();
bool valid = true;
BSONArrayBuilder errors; // explanation(s) for why valid = false
if ( collection->isCapped() ){
result.append("capped", nsd->isCapped());
result.appendNumber("max", nsd->maxCappedDocs());
}
if ( nsd->firstExtent().isNull() )
result.append( "firstExtent", "null" );
else
result.append( "firstExtent", str::stream() << nsd->firstExtent().toString()
<< " ns:" << nsd->firstExtent().ext()->nsDiagnostic.toString());
if ( nsd->lastExtent().isNull() )
result.append( "lastExtent", "null" );
else
result.append( "lastExtent", str::stream() << nsd->lastExtent().toString()
<< " ns:" << nsd->lastExtent().ext()->nsDiagnostic.toString());
BSONArrayBuilder extentData;
int extentCount = 0;
try {
if ( !nsd->firstExtent().isNull() ) {
nsd->firstExtent().ext()->assertOk();
nsd->lastExtent().ext()->assertOk();
}
DiskLoc extentDiskLoc = nsd->firstExtent();
while (!extentDiskLoc.isNull()) {
Extent* thisExtent = extentDiskLoc.ext();
if (full) {
extentData << thisExtent->dump();
}
if (!thisExtent->validates(extentDiskLoc, &errors)) {
valid = false;
}
DiskLoc nextDiskLoc = thisExtent->xnext;
if (extentCount > 0 && !nextDiskLoc.isNull()
&& nextDiskLoc.ext()->xprev != extentDiskLoc) {
StringBuilder sb;
sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString()
<< " in extent " << nextDiskLoc.toString()
<< " does not point to extent " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
if (nextDiskLoc.isNull() && extentDiskLoc != nsd->lastExtent()) {
StringBuilder sb;
sb << "'lastExtent' pointer " << nsd->lastExtent().toString()
<< " does not point to last extent in list " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
extentDiskLoc = nextDiskLoc;
extentCount++;
killCurrentOp.checkForInterrupt();
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception validating extent " << extentCount
<< ": " << e.what();
errors << sb.str();
valid = false;
}
result.append("extentCount", extentCount);
if ( full )
result.appendArray( "extents" , extentData.arr() );
result.appendNumber("datasize", nsd->dataSize());
result.appendNumber("nrecords", nsd->numRecords());
result.appendNumber("lastExtentSize", nsd->lastExtentSize());
result.appendNumber("padding", nsd->paddingFactor());
try {
bool testingLastExtent = false;
try {
if (nsd->firstExtent().isNull()) {
// this is ok
}
else {
result.append("firstExtentDetails", nsd->firstExtent().ext()->dump());
if (!nsd->firstExtent().ext()->xprev.isNull()) {
StringBuilder sb;
sb << "'xprev' pointer in 'firstExtent' " << nsd->firstExtent().toString()
<< " is " << nsd->firstExtent().ext()->xprev.toString()
<< ", should be null";
errors << sb.str();
valid=false;
}
}
testingLastExtent = true;
if (nsd->lastExtent().isNull()) {
// this is ok
}
else {
if (nsd->firstExtent() != nsd->lastExtent()) {
result.append("lastExtentDetails", nsd->lastExtent().ext()->dump());
if (!nsd->lastExtent().ext()->xnext.isNull()) {
StringBuilder sb;
sb << "'xnext' pointer in 'lastExtent' " << nsd->lastExtent().toString()
<< " is " << nsd->lastExtent().ext()->xnext.toString()
<< ", should be null";
errors << sb.str();
valid = false;
}
}
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception processing '"
<< (testingLastExtent ? "lastExtent" : "firstExtent")
<< "': " << e.what();
errors << sb.str();
valid = false;
}
set<DiskLoc> recs;
if( scanData ) {
int n = 0;
int nInvalid = 0;
long long nQuantizedSize = 0;
long long nPowerOf2QuantizedSize = 0;
long long len = 0;
long long nlen = 0;
long long bsonLen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
DiskLoc cl;
Runner::RunnerState state;
auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns));
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &cl))) {
n++;
if ( n < 1000000 )
recs.insert(cl);
if ( nsd->isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = cl.rec();
len += r->lengthWithHeaders();
nlen += r->netLength();
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizeAllocationSpace
( r->lengthWithHeaders() ) ) {
// Count the number of records having a size consistent with
// the quantizeAllocationSpace quantization implementation.
++nQuantizedSize;
}
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizePowerOf2AllocationSpace
( r->lengthWithHeaders() - 1 ) ) {
// Count the number of records having a size consistent with the
// quantizePowerOf2AllocationSpace quantization implementation.
// Because of SERVER-8311, power of 2 quantization is not idempotent and
// r->lengthWithHeaders() - 1 must be checked instead of the record
// length itself.
++nPowerOf2QuantizedSize;
}
if (full){
BSONObj obj = BSONObj::make(r);
if (!obj.isValid() || !obj.valid()){ // both fast and deep checks
valid = false;
if (nInvalid == 0) // only log once;
errors << "invalid bson object detected (see logs for more info)";
nInvalid++;
if (strcmp("_id", obj.firstElementFieldName()) == 0){
try {
obj.firstElement().validate(); // throws on error
log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl;
}
catch(...){
log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl;
}
}
else {
log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl;
}
}
else {
bsonLen += obj.objsize();
}
}
}
if (Runner::RUNNER_EOF != state) {
// TODO: more descriptive logging.
warning() << "Internal error while reading collection " << ns << endl;
}
if ( nsd->isCapped() && !nsd->capLooped() ) {
result.append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
valid = false;
errors << "too many out of order records";
}
}
result.append("objectsFound", n);
if (full) {
result.append("invalidObjects", nInvalid);
}
result.appendNumber("nQuantizedSize", nQuantizedSize);
result.appendNumber("nPowerOf2QuantizedSize", nPowerOf2QuantizedSize);
result.appendNumber("bytesWithHeaders", len);
result.appendNumber("bytesWithoutHeaders", nlen);
if (full) {
result.appendNumber("bytesBson", bsonLen);
}
}
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << nsd->deletedListEntry(i).isNull();
}
int ndel = 0;
long long delSize = 0;
BSONArrayBuilder delBucketSizes;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = nsd->deletedListEntry(i);
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( nsd->isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
string err( str::stream() << "bad pointer in deleted record list: "
<< loc.toString()
<< " bucket: " << i
<< " k: " << k );
errors << err;
valid = false;
break;
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
killCurrentOp.checkForInterrupt();
}
delBucketSizes << k;
}
catch (...) {
errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i));
valid = false;
}
}
result.appendNumber("deletedCount", ndel);
result.appendNumber("deletedSize", delSize);
if ( full ) {
result << "delBucketSizes" << delBucketSizes.arr();
}
if ( incorrect ) {
errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list");
valid = false;
}
int idxn = 0;
try {
IndexCatalog* indexCatalog = collection->getIndexCatalog();
result.append("nIndexes", nsd->getCompletedIndexCount());
BSONObjBuilder indexes; // not using subObjStart to be exception safe
NamespaceDetails::IndexIterator i = nsd->ii();
while( i.more() ) {
IndexDetails& id = i.next();
log() << "validating index " << idxn << ": " << id.indexNamespace() << endl;
IndexDescriptor* descriptor = indexCatalog->getDescriptor( idxn );
verify( descriptor );
IndexAccessMethod* iam = indexCatalog->getIndex( descriptor );
verify( iam );
int64_t keys;
iam->validate(&keys);
indexes.appendNumber(id.indexNamespace(), static_cast<long long>(keys));
idxn++;
}
result.append("keysPerIndex", indexes.done());
}
catch (...) {
errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn));
valid=false;
}
}
catch (AssertionException) {
errors << "exception during validate";
valid = false;
}
result.appendBool("valid", valid);
result.append("errors", errors.arr());
if ( !full ){
result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
}
if ( !valid ) {
result.append("advice", "ns corrupt, requires repair");
}
}
Status RecordStoreV1Base::validate( OperationContext* txn,
bool full, bool scanData,
ValidateAdaptor* adaptor,
ValidateResults* results, BSONObjBuilder* output ) const {
// 1) basic status that require no iteration
// 2) extent level info
// 3) check extent start and end
// 4) check each non-deleted record
// 5) check deleted list
// -------------
// 1111111111111111111
if ( isCapped() ){
output->appendBool("capped", true);
output->appendNumber("max", _details->maxCappedDocs());
}
output->appendNumber("datasize", _details->dataSize());
output->appendNumber("nrecords", _details->numRecords());
output->appendNumber("lastExtentSize", _details->lastExtentSize(txn));
output->appendNumber("padding", _details->paddingFactor());
if ( _details->firstExtent(txn).isNull() )
output->append( "firstExtent", "null" );
else
output->append( "firstExtent",
str::stream() << _details->firstExtent(txn).toString()
<< " ns:"
<< _getExtent( txn, _details->firstExtent(txn) )->nsDiagnostic.toString());
if ( _details->lastExtent(txn).isNull() )
output->append( "lastExtent", "null" );
else
output->append( "lastExtent", str::stream() << _details->lastExtent(txn).toString()
<< " ns:"
<< _getExtent( txn, _details->lastExtent(txn) )->nsDiagnostic.toString());
// 22222222222222222222222222
{ // validate extent basics
BSONArrayBuilder extentData;
int extentCount = 0;
DiskLoc extentDiskLoc;
try {
if ( !_details->firstExtent(txn).isNull() ) {
_getExtent( txn, _details->firstExtent(txn) )->assertOk();
_getExtent( txn, _details->lastExtent(txn) )->assertOk();
}
extentDiskLoc = _details->firstExtent(txn);
while (!extentDiskLoc.isNull()) {
Extent* thisExtent = _getExtent( txn, extentDiskLoc );
if (full) {
extentData << thisExtent->dump();
}
if (!thisExtent->validates(extentDiskLoc, &results->errors)) {
results->valid = false;
}
DiskLoc nextDiskLoc = thisExtent->xnext;
if (extentCount > 0 && !nextDiskLoc.isNull()
&& _getExtent( txn, nextDiskLoc )->xprev != extentDiskLoc) {
StringBuilder sb;
sb << "'xprev' pointer " << _getExtent( txn, nextDiskLoc )->xprev.toString()
<< " in extent " << nextDiskLoc.toString()
<< " does not point to extent " << extentDiskLoc.toString();
results->errors.push_back( sb.str() );
results->valid = false;
}
if (nextDiskLoc.isNull() && extentDiskLoc != _details->lastExtent(txn)) {
StringBuilder sb;
sb << "'lastExtent' pointer " << _details->lastExtent(txn).toString()
<< " does not point to last extent in list " << extentDiskLoc.toString();
results->errors.push_back( sb.str() );
results->valid = false;
}
extentDiskLoc = nextDiskLoc;
extentCount++;
txn->checkForInterrupt();
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception validating extent " << extentCount
<< ": " << e.what();
results->errors.push_back( sb.str() );
results->valid = false;
return Status::OK();
}
output->append("extentCount", extentCount);
if ( full )
output->appendArray( "extents" , extentData.arr() );
}
try {
// 333333333333333333333333333
bool testingLastExtent = false;
try {
DiskLoc firstExtentLoc = _details->firstExtent(txn);
if (firstExtentLoc.isNull()) {
// this is ok
}
else {
output->append("firstExtentDetails", _getExtent(txn, firstExtentLoc)->dump());
if (!_getExtent(txn, firstExtentLoc)->xprev.isNull()) {
StringBuilder sb;
sb << "'xprev' pointer in 'firstExtent' " << _details->firstExtent(txn).toString()
<< " is " << _getExtent(txn, firstExtentLoc)->xprev.toString()
<< ", should be null";
results->errors.push_back( sb.str() );
results->valid = false;
}
}
testingLastExtent = true;
DiskLoc lastExtentLoc = _details->lastExtent(txn);
if (lastExtentLoc.isNull()) {
// this is ok
}
else {
if (firstExtentLoc != lastExtentLoc) {
output->append("lastExtentDetails", _getExtent(txn, lastExtentLoc)->dump());
if (!_getExtent(txn, lastExtentLoc)->xnext.isNull()) {
StringBuilder sb;
sb << "'xnext' pointer in 'lastExtent' " << lastExtentLoc.toString()
<< " is " << _getExtent(txn, lastExtentLoc)->xnext.toString()
<< ", should be null";
results->errors.push_back( sb.str() );
results->valid = false;
}
}
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception processing '"
<< (testingLastExtent ? "lastExtent" : "firstExtent")
<< "': " << e.what();
results->errors.push_back( sb.str() );
results->valid = false;
}
// 4444444444444444444444444
set<DiskLoc> recs;
if( scanData ) {
int n = 0;
int nInvalid = 0;
long long nQuantizedSize = 0;
long long nPowerOf2QuantizedSize = 0;
long long len = 0;
long long nlen = 0;
long long bsonLen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
scoped_ptr<RecordIterator> iterator( getIterator( txn,
DiskLoc(),
false,
CollectionScanParams::FORWARD ) );
DiskLoc cl;
while ( !( cl = iterator->getNext() ).isNull() ) {
n++;
if ( n < 1000000 )
recs.insert(cl);
if ( isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = recordFor(cl);
len += r->lengthWithHeaders();
nlen += r->netLength();
if ( r->lengthWithHeaders() ==
quantizeAllocationSpace( r->lengthWithHeaders() ) ) {
// Count the number of records having a size consistent with
// the quantizeAllocationSpace quantization implementation.
++nQuantizedSize;
}
if ( r->lengthWithHeaders() ==
quantizePowerOf2AllocationSpace( r->lengthWithHeaders() ) ) {
// Count the number of records having a size consistent with the
// quantizePowerOf2AllocationSpace quantization implementation.
++nPowerOf2QuantizedSize;
}
if (full){
size_t dataSize = 0;
const Status status = adaptor->validate( r->toRecordData(), &dataSize );
if (!status.isOK()) {
results->valid = false;
if (nInvalid == 0) // only log once;
results->errors.push_back( "invalid object detected (see logs)" );
nInvalid++;
log() << "Invalid object detected in " << _ns
<< ": " << status.reason();
}
else {
bsonLen += dataSize;
}
}
}
if ( isCapped() && !_details->capLooped() ) {
output->append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
results->valid = false;
results->errors.push_back( "too many out of order records" );
}
}
output->append("objectsFound", n);
if (full) {
output->append("invalidObjects", nInvalid);
}
output->appendNumber("nQuantizedSize", nQuantizedSize);
output->appendNumber("nPowerOf2QuantizedSize", nPowerOf2QuantizedSize);
output->appendNumber("bytesWithHeaders", len);
output->appendNumber("bytesWithoutHeaders", nlen);
if (full) {
output->appendNumber("bytesBson", bsonLen);
}
} // end scanData
// 55555555555555555555555555
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << _details->deletedListEntry(i).isNull();
}
int ndel = 0;
long long delSize = 0;
BSONArrayBuilder delBucketSizes;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = _details->deletedListEntry(i);
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
string err( str::stream() << "bad pointer in deleted record list: "
<< loc.toString()
<< " bucket: " << i
<< " k: " << k );
results->errors.push_back( err );
results->valid = false;
break;
}
const DeletedRecord* d = deletedRecordFor(loc);
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
txn->checkForInterrupt();
}
delBucketSizes << k;
}
catch (...) {
results->errors.push_back( (string)"exception in deleted chain for bucket " +
BSONObjBuilder::numStr(i) );
results->valid = false;
}
}
output->appendNumber("deletedCount", ndel);
output->appendNumber("deletedSize", delSize);
if ( full ) {
output->append( "delBucketSizes", delBucketSizes.arr() );
}
if ( incorrect ) {
results->errors.push_back( BSONObjBuilder::numStr(incorrect) +
" records from datafile are in deleted list" );
results->valid = false;
}
}
catch (AssertionException) {
results->errors.push_back( "exception during validate" );
results->valid = false;
}
return Status::OK();
}
void validateNS(const char *ns, NamespaceDetails *d, const BSONObj& cmdObj, BSONObjBuilder& result) {
const bool full = cmdObj["full"].trueValue();
const bool scanData = full || cmdObj["scandata"].trueValue();
bool valid = true;
BSONArrayBuilder errors; // explanation(s) for why valid = false
if ( d->isCapped() ){
result.append("capped", d->isCapped());
result.appendNumber("max", d->maxCappedDocs());
}
result.append("firstExtent", str::stream() << d->firstExtent.toString() << " ns:" << d->firstExtent.ext()->nsDiagnostic.toString());
result.append( "lastExtent", str::stream() << d->lastExtent.toString() << " ns:" << d->lastExtent.ext()->nsDiagnostic.toString());
BSONArrayBuilder extentData;
try {
d->firstExtent.ext()->assertOk();
d->lastExtent.ext()->assertOk();
DiskLoc el = d->firstExtent;
int ne = 0;
while( !el.isNull() ) {
Extent *e = el.ext();
e->assertOk();
el = e->xnext;
ne++;
if ( full )
extentData << e->dump();
killCurrentOp.checkForInterrupt();
}
result.append("extentCount", ne);
}
catch (...) {
valid=false;
errors << "extent asserted";
}
if ( full )
result.appendArray( "extents" , extentData.arr() );
result.appendNumber("datasize", d->stats.datasize);
result.appendNumber("nrecords", d->stats.nrecords);
result.appendNumber("lastExtentSize", d->lastExtentSize);
result.appendNumber("padding", d->paddingFactor());
try {
try {
result.append("firstExtentDetails", d->firstExtent.ext()->dump());
valid = valid && d->firstExtent.ext()->validates() &&
d->firstExtent.ext()->xprev.isNull();
}
catch (...) {
errors << "exception firstextent";
valid = false;
}
set<DiskLoc> recs;
if( scanData ) {
shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
int n = 0;
int nInvalid = 0;
long long len = 0;
long long nlen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
while ( c->ok() ) {
n++;
DiskLoc cl = c->currLoc();
if ( n < 1000000 )
recs.insert(cl);
if ( d->isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = c->_current();
len += r->lengthWithHeaders();
nlen += r->netLength();
if (full){
BSONObj obj = BSONObj::make(r);
if (!obj.isValid() || !obj.valid()){ // both fast and deep checks
valid = false;
if (nInvalid == 0) // only log once;
errors << "invalid bson object detected (see logs for more info)";
nInvalid++;
if (strcmp("_id", obj.firstElementFieldName()) == 0){
try {
obj.firstElement().validate(); // throws on error
log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl;
}
catch(...){
log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl;
}
}
else {
log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl;
}
}
}
c->advance();
}
if ( d->isCapped() && !d->capLooped() ) {
result.append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
valid = false;
errors << "too many out of order records";
}
}
result.append("objectsFound", n);
if (full) {
result.append("invalidObjects", nInvalid);
}
result.appendNumber("bytesWithHeaders", len);
result.appendNumber("bytesWithoutHeaders", nlen);
}
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << d->deletedList[i].isNull();
}
int ndel = 0;
long long delSize = 0;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = d->deletedList[i];
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( d->isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
if ( loc.a() <= 0 || strstr(ns, "hudsonSmall") == 0 ) {
string err (str::stream() << "bad deleted loc: " << loc.toString() << " bucket:" << i << " k:" << k);
errors << err;
valid = false;
break;
}
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
killCurrentOp.checkForInterrupt();
}
}
catch (...) {
errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i));
valid = false;
}
}
result.appendNumber("deletedCount", ndel);
result.appendNumber("deletedSize", delSize);
if ( incorrect ) {
errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list");
valid = false;
}
int idxn = 0;
try {
result.append("nIndexes", d->nIndexes);
BSONObjBuilder indexes; // not using subObjStart to be exception safe
NamespaceDetails::IndexIterator i = d->ii();
while( i.more() ) {
IndexDetails& id = i.next();
log() << "validating index " << idxn << ": " << id.indexNamespace() << endl;
long long keys = id.idxInterface().fullValidate(id.head, id.keyPattern());
indexes.appendNumber(id.indexNamespace(), keys);
idxn++;
}
result.append("keysPerIndex", indexes.done());
}
catch (...) {
errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn));
valid=false;
}
}
catch (AssertionException) {
errors << "exception during validate";
valid = false;
}
result.appendBool("valid", valid);
result.append("errors", errors.arr());
if ( !full ){
result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
}
if ( !valid ) {
result.append("advice", "ns corrupt, requires repair");
}
}