void SimpleRecordStoreV1::addDeletedRec(OperationContext* txn, const DiskLoc& dloc) {
DeletedRecord* d = drec(dloc);
int b = bucket(d->lengthWithHeaders());
*txn->recoveryUnit()->writing(&d->nextDeleted()) = _details->deletedListEntry(b);
_details->setDeletedListEntry(txn, b, dloc);
}
void SimpleRecordStoreV1::addDeletedRec( OperationContext* txn, const DiskLoc& dloc ) {
DeletedRecord* d = drec( dloc );
DEBUGGING log() << "TEMP: add deleted rec " << dloc.toString() << ' ' << hex << d->extentOfs() << endl;
int b = bucket(d->lengthWithHeaders());
*txn->recoveryUnit()->writing(&d->nextDeleted()) = _details->deletedListEntry(b);
_details->setDeletedListEntry(txn, b, dloc);
}
/* assumes already zeroed -- insufficient for block 'reuse' perhaps */
DiskLoc Extent::init(const char *nsname, int _length, int _fileNo, int _offset, bool capped) {
magic = extentSignature;
myLoc.set(_fileNo, _offset);
xnext.Null();
xprev.Null();
nsDiagnostic = nsname;
length = _length;
firstRecord.Null();
lastRecord.Null();
DiskLoc emptyLoc;
int delRecLength;
extent_getEmptyLoc(nsname, myLoc, _length, capped, emptyLoc, delRecLength);
DeletedRecord* empty = getDur().writing(DataFileMgr::getDeletedRecord(emptyLoc));
empty->lengthWithHeaders() = delRecLength;
empty->extentOfs() = myLoc.getOfs();
empty->nextDeleted().Null();
return emptyLoc;
}
DiskLoc RecordStoreV1Base::_findFirstSpot( OperationContext* txn,
const DiskLoc& extDiskLoc, Extent* e ) {
DiskLoc emptyLoc = extDiskLoc;
emptyLoc.inc( Extent::HeaderSize() );
int delRecLength = e->length - Extent::HeaderSize();
if ( delRecLength >= 32*1024 && _ns.find('$') != string::npos && !isCapped() ) {
// probably an index. so skip forward to keep its records page aligned
int& ofs = emptyLoc.GETOFS();
int newOfs = (ofs + 0xfff) & ~0xfff;
delRecLength -= (newOfs-ofs);
dassert( delRecLength > 0 );
ofs = newOfs;
}
DeletedRecord* empty = txn->recoveryUnit()->writing(drec(emptyLoc));
empty->lengthWithHeaders() = delRecLength;
empty->extentOfs() = e->myLoc.getOfs();
empty->nextDeleted().Null();
return emptyLoc;
}
DiskLoc Extent::_reuse(const char *nsname, bool capped) {
LOG(3) << "_reuse extent was:" << nsDiagnostic.toString() << " now:" << nsname << endl;
if (magic != extentSignature) {
StringBuilder sb;
sb << "bad extent signature " << integerToHex(magic)
<< " for namespace '" << nsDiagnostic.toString()
<< "' found in Extent::_reuse";
msgasserted(10360, sb.str());
}
nsDiagnostic = nsname;
markEmpty();
DiskLoc emptyLoc;
int delRecLength;
extent_getEmptyLoc(nsname, myLoc, length, capped, emptyLoc, delRecLength);
// todo: some dup code here and below in Extent::init
DeletedRecord* empty = getDur().writing(DataFileMgr::getDeletedRecord(emptyLoc));
empty->lengthWithHeaders() = delRecLength;
empty->extentOfs() = myLoc.getOfs();
empty->nextDeleted().Null();
return emptyLoc;
}
void initializeV1RS(OperationContext* txn,
const LocAndSize* records,
const LocAndSize* drecs,
DummyExtentManager* em,
DummyRecordStoreV1MetaData* md) {
invariant(records || drecs); // if both are NULL nothing is being created...
// Need to start with a blank slate
invariant(em->numFiles() == 0);
invariant(md->firstExtent().isNull());
// pre-allocate extents (even extents that aren't part of this RS)
{
typedef std::map<int, size_t> ExtentSizes;
ExtentSizes extentSizes;
accumulateExtentSizeRequirements(records, &extentSizes);
accumulateExtentSizeRequirements(drecs, &extentSizes);
invariant(!extentSizes.empty());
const int maxExtent = extentSizes.rbegin()->first;
for (int i = 0; i <= maxExtent; i++) {
const size_t size = extentSizes.count(i) ? extentSizes[i] : 0;
const DiskLoc loc = em->allocateExtent(txn, md->isCapped(), size, 0);
// This function and assertState depend on these details of DummyExtentManager
invariant(loc.a() == i);
invariant(loc.getOfs() == 0);
}
// link together extents that should be part of this RS
md->setFirstExtent(txn, DiskLoc(extentSizes.begin()->first, 0));
md->setLastExtent(txn, DiskLoc(extentSizes.rbegin()->first, 0));
for (ExtentSizes::iterator it = extentSizes.begin();
boost::next(it) != extentSizes.end(); /* ++it */ ) {
const int a = it->first;
++it;
const int b = it->first;
em->getExtent(DiskLoc(a, 0))->xnext = DiskLoc(b, 0);
em->getExtent(DiskLoc(b, 0))->xprev = DiskLoc(a, 0);
}
// This signals "done allocating new extents".
if (md->isCapped())
md->setDeletedListEntry(txn, 1, DiskLoc());
}
if (records && !records[0].loc.isNull()) {
int recIdx = 0;
DiskLoc extLoc = md->firstExtent();
while (!extLoc.isNull()) {
Extent* ext = em->getExtent(extLoc);
int prevOfs = DiskLoc::NullOfs;
while (extLoc.a() == records[recIdx].loc.a()) { // for all records in this extent
const DiskLoc loc = records[recIdx].loc;
const int size = records[recIdx].size;;
invariant(size >= Record::HeaderSize);
md->incrementStats(txn, size - Record::HeaderSize, 1);
if (ext->firstRecord.isNull())
ext->firstRecord = loc;
Record* rec = em->recordForV1(loc);
rec->lengthWithHeaders() = size;
rec->extentOfs() = 0;
rec->prevOfs() = prevOfs;
prevOfs = loc.getOfs();
const DiskLoc nextLoc = records[recIdx + 1].loc;
if (nextLoc.a() == loc.a()) { // if next is in same extent
rec->nextOfs() = nextLoc.getOfs();
}
else {
rec->nextOfs() = DiskLoc::NullOfs;
ext->lastRecord = loc;
}
recIdx++;
}
extLoc = ext->xnext;
}
invariant(records[recIdx].loc.isNull());
}
if (drecs && !drecs[0].loc.isNull()) {
int drecIdx = 0;
DiskLoc* prevNextPtr = NULL;
int lastBucket = -1;
while (!drecs[drecIdx].loc.isNull()) {
const DiskLoc loc = drecs[drecIdx].loc;
const int size = drecs[drecIdx].size;
invariant(size >= Record::HeaderSize);
const int bucket = RecordStoreV1Base::bucket(size);
if (md->isCapped()) {
// All drecs form a single list in bucket 0
if (prevNextPtr == NULL) {
md->setDeletedListEntry(txn, 0, loc);
}
else {
*prevNextPtr = loc;
}
if (loc.a() < md->capExtent().a()
&& drecs[drecIdx + 1].loc.a() == md->capExtent().a()) {
// Bucket 1 is known as cappedLastDelRecLastExtent
md->setDeletedListEntry(txn, 1, loc);
}
}
else if (bucket != lastBucket) {
invariant(bucket > lastBucket); // if this fails, drecs weren't sorted by bucket
md->setDeletedListEntry(txn, bucket, loc);
lastBucket = bucket;
}
else {
*prevNextPtr = loc;
}
DeletedRecord* drec = &em->recordForV1(loc)->asDeleted();
drec->lengthWithHeaders() = size;
drec->extentOfs() = 0;
drec->nextDeleted() = DiskLoc();
prevNextPtr = &drec->nextDeleted();
drecIdx++;
}
}
// Make sure we set everything up as requested.
assertStateV1RS(records, drecs, em, md);
}
DiskLoc SimpleRecordStoreV1::_allocFromExistingExtents(OperationContext* txn, int lenToAllocRaw) {
// Slowly drain the deletedListLegacyGrabBag by popping one record off and putting it in the
// correct deleted list each time we try to allocate a new record. This ensures we won't
// orphan any data when upgrading from old versions, without needing a long upgrade phase.
// This is done before we try to allocate the new record so we can take advantage of the new
// space immediately.
{
const DiskLoc head = _details->deletedListLegacyGrabBag();
if (!head.isNull()) {
_details->setDeletedListLegacyGrabBag(txn, drec(head)->nextDeleted());
addDeletedRec(txn, head);
}
}
// align size up to a multiple of 4
const int lenToAlloc = (lenToAllocRaw + (4 - 1)) & ~(4 - 1);
freelistAllocs.increment();
DiskLoc loc;
DeletedRecord* dr = NULL;
{
int myBucket;
for (myBucket = bucket(lenToAlloc); myBucket < Buckets; myBucket++) {
// Only look at the first entry in each bucket. This works because we are either
// quantizing or allocating fixed-size blocks.
const DiskLoc head = _details->deletedListEntry(myBucket);
if (head.isNull())
continue;
DeletedRecord* const candidate = drec(head);
if (candidate->lengthWithHeaders() >= lenToAlloc) {
loc = head;
dr = candidate;
break;
}
}
if (!dr)
return DiskLoc(); // no space
// Unlink ourself from the deleted list
_details->setDeletedListEntry(txn, myBucket, dr->nextDeleted());
*txn->recoveryUnit()->writing(&dr->nextDeleted()) = DiskLoc().setInvalid(); // defensive
}
invariant(dr->extentOfs() < loc.getOfs());
// Split the deleted record if it has at least as much left over space as our smallest
// allocation size. Otherwise, just take the whole DeletedRecord.
const int remainingLength = dr->lengthWithHeaders() - lenToAlloc;
if (remainingLength >= bucketSizes[0]) {
txn->recoveryUnit()->writingInt(dr->lengthWithHeaders()) = lenToAlloc;
const DiskLoc newDelLoc = DiskLoc(loc.a(), loc.getOfs() + lenToAlloc);
DeletedRecord* newDel = txn->recoveryUnit()->writing(drec(newDelLoc));
newDel->extentOfs() = dr->extentOfs();
newDel->lengthWithHeaders() = remainingLength;
newDel->nextDeleted().Null();
addDeletedRec(txn, newDelLoc);
}
return loc;
}
DiskLoc SimpleRecordStoreV1::_allocFromExistingExtents( OperationContext* txn,
int lenToAlloc ) {
// align size up to a multiple of 4
lenToAlloc = (lenToAlloc + (4-1)) & ~(4-1);
freelistAllocs.increment();
DiskLoc loc;
{
DiskLoc *prev = 0;
DiskLoc *bestprev = 0;
DiskLoc bestmatch;
int bestmatchlen = INT_MAX; // sentinel meaning we haven't found a record big enough
int b = bucket(lenToAlloc);
DiskLoc cur = _details->deletedListEntry(b);
int extra = 5; // look for a better fit, a little.
int chain = 0;
while ( 1 ) {
{ // defensive check
int fileNumber = cur.a();
int fileOffset = cur.getOfs();
if (fileNumber < -1 || fileNumber >= 100000 || fileOffset < 0) {
StringBuilder sb;
sb << "Deleted record list corrupted in collection " << _ns
<< ", bucket " << b
<< ", link number " << chain
<< ", invalid link is " << cur.toString()
<< ", throwing Fatal Assertion";
log() << sb.str() << endl;
fassertFailed(16469);
}
}
if ( cur.isNull() ) {
// move to next bucket. if we were doing "extra", just break
if ( bestmatchlen < INT_MAX )
break;
if ( chain > 0 ) {
// if we looked at things in the right bucket, but they were not suitable
freelistBucketExhausted.increment();
}
b++;
if ( b > MaxBucket ) {
// out of space. alloc a new extent.
freelistIterations.increment( 1 + chain );
return DiskLoc();
}
cur = _details->deletedListEntry(b);
prev = 0;
continue;
}
DeletedRecord *r = drec(cur);
if ( r->lengthWithHeaders() >= lenToAlloc &&
r->lengthWithHeaders() < bestmatchlen ) {
bestmatchlen = r->lengthWithHeaders();
bestmatch = cur;
bestprev = prev;
if (r->lengthWithHeaders() == lenToAlloc)
// exact match, stop searching
break;
}
if ( bestmatchlen < INT_MAX && --extra <= 0 )
break;
if ( ++chain > 30 && b <= MaxBucket ) {
// too slow, force move to next bucket to grab a big chunk
//b++;
freelistIterations.increment( chain );
chain = 0;
cur.Null();
}
else {
cur = r->nextDeleted();
prev = &r->nextDeleted();
}
}
// unlink ourself from the deleted list
DeletedRecord *bmr = drec(bestmatch);
if ( bestprev ) {
*txn->recoveryUnit()->writing(bestprev) = bmr->nextDeleted();
}
else {
// should be the front of a free-list
int myBucket = bucket(bmr->lengthWithHeaders());
invariant( _details->deletedListEntry(myBucket) == bestmatch );
_details->setDeletedListEntry(txn, myBucket, bmr->nextDeleted());
}
*txn->recoveryUnit()->writing(&bmr->nextDeleted()) = DiskLoc().setInvalid(); // defensive.
invariant(bmr->extentOfs() < bestmatch.getOfs());
freelistIterations.increment( 1 + chain );
loc = bestmatch;
}
if ( loc.isNull() )
return loc;
// determine if we should chop up
DeletedRecord *r = drec(loc);
/* note we want to grab from the front so our next pointers on disk tend
to go in a forward direction which is important for performance. */
int regionlen = r->lengthWithHeaders();
invariant( r->extentOfs() < loc.getOfs() );
int left = regionlen - lenToAlloc;
if ( left < 24 || left < (lenToAlloc / 8) ) {
// you get the whole thing.
return loc;
}
// don't quantize:
// - $ collections (indexes) as we already have those aligned the way we want SERVER-8425
if ( _normalCollection ) {
// we quantize here so that it only impacts newly sized records
// this prevents oddities with older records and space re-use SERVER-8435
lenToAlloc = std::min( r->lengthWithHeaders(),
quantizeAllocationSpace( lenToAlloc ) );
left = regionlen - lenToAlloc;
if ( left < 24 ) {
// you get the whole thing.
return loc;
}
}
/* split off some for further use. */
txn->recoveryUnit()->writingInt(r->lengthWithHeaders()) = lenToAlloc;
DiskLoc newDelLoc = loc;
newDelLoc.inc(lenToAlloc);
DeletedRecord* newDel = drec(newDelLoc);
DeletedRecord* newDelW = txn->recoveryUnit()->writing(newDel);
newDelW->extentOfs() = r->extentOfs();
newDelW->lengthWithHeaders() = left;
newDelW->nextDeleted().Null();
addDeletedRec( txn, newDelLoc );
return loc;
}
void validateNS(const string& ns,
Collection* collection,
const BSONObj& cmdObj,
BSONObjBuilder& result) {
const bool full = cmdObj["full"].trueValue();
const bool scanData = full || cmdObj["scandata"].trueValue();
NamespaceDetails* nsd = collection->details();
bool valid = true;
BSONArrayBuilder errors; // explanation(s) for why valid = false
if ( collection->isCapped() ){
result.append("capped", nsd->isCapped());
result.appendNumber("max", nsd->maxCappedDocs());
}
if ( nsd->firstExtent().isNull() )
result.append( "firstExtent", "null" );
else
result.append( "firstExtent", str::stream() << nsd->firstExtent().toString()
<< " ns:" << nsd->firstExtent().ext()->nsDiagnostic.toString());
if ( nsd->lastExtent().isNull() )
result.append( "lastExtent", "null" );
else
result.append( "lastExtent", str::stream() << nsd->lastExtent().toString()
<< " ns:" << nsd->lastExtent().ext()->nsDiagnostic.toString());
BSONArrayBuilder extentData;
int extentCount = 0;
try {
if ( !nsd->firstExtent().isNull() ) {
nsd->firstExtent().ext()->assertOk();
nsd->lastExtent().ext()->assertOk();
}
DiskLoc extentDiskLoc = nsd->firstExtent();
while (!extentDiskLoc.isNull()) {
Extent* thisExtent = extentDiskLoc.ext();
if (full) {
extentData << thisExtent->dump();
}
if (!thisExtent->validates(extentDiskLoc, &errors)) {
valid = false;
}
DiskLoc nextDiskLoc = thisExtent->xnext;
if (extentCount > 0 && !nextDiskLoc.isNull()
&& nextDiskLoc.ext()->xprev != extentDiskLoc) {
StringBuilder sb;
sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString()
<< " in extent " << nextDiskLoc.toString()
<< " does not point to extent " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
if (nextDiskLoc.isNull() && extentDiskLoc != nsd->lastExtent()) {
StringBuilder sb;
sb << "'lastExtent' pointer " << nsd->lastExtent().toString()
<< " does not point to last extent in list " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
extentDiskLoc = nextDiskLoc;
extentCount++;
killCurrentOp.checkForInterrupt();
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception validating extent " << extentCount
<< ": " << e.what();
errors << sb.str();
valid = false;
}
result.append("extentCount", extentCount);
if ( full )
result.appendArray( "extents" , extentData.arr() );
result.appendNumber("datasize", nsd->dataSize());
result.appendNumber("nrecords", nsd->numRecords());
result.appendNumber("lastExtentSize", nsd->lastExtentSize());
result.appendNumber("padding", nsd->paddingFactor());
try {
bool testingLastExtent = false;
try {
if (nsd->firstExtent().isNull()) {
// this is ok
}
else {
result.append("firstExtentDetails", nsd->firstExtent().ext()->dump());
if (!nsd->firstExtent().ext()->xprev.isNull()) {
StringBuilder sb;
sb << "'xprev' pointer in 'firstExtent' " << nsd->firstExtent().toString()
<< " is " << nsd->firstExtent().ext()->xprev.toString()
<< ", should be null";
errors << sb.str();
valid=false;
}
}
testingLastExtent = true;
if (nsd->lastExtent().isNull()) {
// this is ok
}
else {
if (nsd->firstExtent() != nsd->lastExtent()) {
result.append("lastExtentDetails", nsd->lastExtent().ext()->dump());
if (!nsd->lastExtent().ext()->xnext.isNull()) {
StringBuilder sb;
sb << "'xnext' pointer in 'lastExtent' " << nsd->lastExtent().toString()
<< " is " << nsd->lastExtent().ext()->xnext.toString()
<< ", should be null";
errors << sb.str();
valid = false;
}
}
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception processing '"
<< (testingLastExtent ? "lastExtent" : "firstExtent")
<< "': " << e.what();
errors << sb.str();
valid = false;
}
set<DiskLoc> recs;
if( scanData ) {
int n = 0;
int nInvalid = 0;
long long nQuantizedSize = 0;
long long nPowerOf2QuantizedSize = 0;
long long len = 0;
long long nlen = 0;
long long bsonLen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
DiskLoc cl;
Runner::RunnerState state;
auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns));
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &cl))) {
n++;
if ( n < 1000000 )
recs.insert(cl);
if ( nsd->isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = cl.rec();
len += r->lengthWithHeaders();
nlen += r->netLength();
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizeAllocationSpace
( r->lengthWithHeaders() ) ) {
// Count the number of records having a size consistent with
// the quantizeAllocationSpace quantization implementation.
++nQuantizedSize;
}
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizePowerOf2AllocationSpace
( r->lengthWithHeaders() - 1 ) ) {
// Count the number of records having a size consistent with the
// quantizePowerOf2AllocationSpace quantization implementation.
// Because of SERVER-8311, power of 2 quantization is not idempotent and
// r->lengthWithHeaders() - 1 must be checked instead of the record
// length itself.
++nPowerOf2QuantizedSize;
}
if (full){
BSONObj obj = BSONObj::make(r);
if (!obj.isValid() || !obj.valid()){ // both fast and deep checks
valid = false;
if (nInvalid == 0) // only log once;
errors << "invalid bson object detected (see logs for more info)";
nInvalid++;
if (strcmp("_id", obj.firstElementFieldName()) == 0){
try {
obj.firstElement().validate(); // throws on error
log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl;
}
catch(...){
log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl;
}
}
else {
log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl;
}
}
else {
bsonLen += obj.objsize();
}
}
}
if (Runner::RUNNER_EOF != state) {
// TODO: more descriptive logging.
warning() << "Internal error while reading collection " << ns << endl;
}
if ( nsd->isCapped() && !nsd->capLooped() ) {
result.append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
valid = false;
errors << "too many out of order records";
}
}
result.append("objectsFound", n);
if (full) {
result.append("invalidObjects", nInvalid);
}
result.appendNumber("nQuantizedSize", nQuantizedSize);
result.appendNumber("nPowerOf2QuantizedSize", nPowerOf2QuantizedSize);
result.appendNumber("bytesWithHeaders", len);
result.appendNumber("bytesWithoutHeaders", nlen);
if (full) {
result.appendNumber("bytesBson", bsonLen);
}
}
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << nsd->deletedListEntry(i).isNull();
}
int ndel = 0;
long long delSize = 0;
BSONArrayBuilder delBucketSizes;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = nsd->deletedListEntry(i);
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( nsd->isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
string err( str::stream() << "bad pointer in deleted record list: "
<< loc.toString()
<< " bucket: " << i
<< " k: " << k );
errors << err;
valid = false;
break;
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
killCurrentOp.checkForInterrupt();
}
delBucketSizes << k;
}
catch (...) {
errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i));
valid = false;
}
}
result.appendNumber("deletedCount", ndel);
result.appendNumber("deletedSize", delSize);
if ( full ) {
result << "delBucketSizes" << delBucketSizes.arr();
}
if ( incorrect ) {
errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list");
valid = false;
}
int idxn = 0;
try {
IndexCatalog* indexCatalog = collection->getIndexCatalog();
result.append("nIndexes", nsd->getCompletedIndexCount());
BSONObjBuilder indexes; // not using subObjStart to be exception safe
NamespaceDetails::IndexIterator i = nsd->ii();
while( i.more() ) {
IndexDetails& id = i.next();
log() << "validating index " << idxn << ": " << id.indexNamespace() << endl;
IndexDescriptor* descriptor = indexCatalog->getDescriptor( idxn );
verify( descriptor );
IndexAccessMethod* iam = indexCatalog->getIndex( descriptor );
verify( iam );
int64_t keys;
iam->validate(&keys);
indexes.appendNumber(id.indexNamespace(), static_cast<long long>(keys));
idxn++;
}
result.append("keysPerIndex", indexes.done());
}
catch (...) {
errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn));
valid=false;
}
}
catch (AssertionException) {
errors << "exception during validate";
valid = false;
}
result.appendBool("valid", valid);
result.append("errors", errors.arr());
if ( !full ){
result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
}
if ( !valid ) {
result.append("advice", "ns corrupt, requires repair");
}
}
void validateNS(const char *ns, NamespaceDetails *d, const BSONObj& cmdObj, BSONObjBuilder& result) {
const bool full = cmdObj["full"].trueValue();
const bool scanData = full || cmdObj["scandata"].trueValue();
bool valid = true;
BSONArrayBuilder errors; // explanation(s) for why valid = false
if ( d->isCapped() ){
result.append("capped", d->isCapped());
result.appendNumber("max", d->maxCappedDocs());
}
result.append("firstExtent", str::stream() << d->firstExtent.toString() << " ns:" << d->firstExtent.ext()->nsDiagnostic.toString());
result.append( "lastExtent", str::stream() << d->lastExtent.toString() << " ns:" << d->lastExtent.ext()->nsDiagnostic.toString());
BSONArrayBuilder extentData;
try {
d->firstExtent.ext()->assertOk();
d->lastExtent.ext()->assertOk();
DiskLoc el = d->firstExtent;
int ne = 0;
while( !el.isNull() ) {
Extent *e = el.ext();
e->assertOk();
el = e->xnext;
ne++;
if ( full )
extentData << e->dump();
killCurrentOp.checkForInterrupt();
}
result.append("extentCount", ne);
}
catch (...) {
valid=false;
errors << "extent asserted";
}
if ( full )
result.appendArray( "extents" , extentData.arr() );
result.appendNumber("datasize", d->stats.datasize);
result.appendNumber("nrecords", d->stats.nrecords);
result.appendNumber("lastExtentSize", d->lastExtentSize);
result.appendNumber("padding", d->paddingFactor());
try {
try {
result.append("firstExtentDetails", d->firstExtent.ext()->dump());
valid = valid && d->firstExtent.ext()->validates() &&
d->firstExtent.ext()->xprev.isNull();
}
catch (...) {
errors << "exception firstextent";
valid = false;
}
set<DiskLoc> recs;
if( scanData ) {
shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
int n = 0;
int nInvalid = 0;
long long len = 0;
long long nlen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
while ( c->ok() ) {
n++;
DiskLoc cl = c->currLoc();
if ( n < 1000000 )
recs.insert(cl);
if ( d->isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = c->_current();
len += r->lengthWithHeaders();
nlen += r->netLength();
if (full){
BSONObj obj = BSONObj::make(r);
if (!obj.isValid() || !obj.valid()){ // both fast and deep checks
valid = false;
if (nInvalid == 0) // only log once;
errors << "invalid bson object detected (see logs for more info)";
nInvalid++;
if (strcmp("_id", obj.firstElementFieldName()) == 0){
try {
obj.firstElement().validate(); // throws on error
log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl;
}
catch(...){
log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl;
}
}
else {
log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl;
}
}
}
c->advance();
}
if ( d->isCapped() && !d->capLooped() ) {
result.append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
valid = false;
errors << "too many out of order records";
}
}
result.append("objectsFound", n);
if (full) {
result.append("invalidObjects", nInvalid);
}
result.appendNumber("bytesWithHeaders", len);
result.appendNumber("bytesWithoutHeaders", nlen);
}
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << d->deletedList[i].isNull();
}
int ndel = 0;
long long delSize = 0;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = d->deletedList[i];
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( d->isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
if ( loc.a() <= 0 || strstr(ns, "hudsonSmall") == 0 ) {
string err (str::stream() << "bad deleted loc: " << loc.toString() << " bucket:" << i << " k:" << k);
errors << err;
valid = false;
break;
}
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
killCurrentOp.checkForInterrupt();
}
}
catch (...) {
errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i));
valid = false;
}
}
result.appendNumber("deletedCount", ndel);
result.appendNumber("deletedSize", delSize);
if ( incorrect ) {
errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list");
valid = false;
}
int idxn = 0;
try {
result.append("nIndexes", d->nIndexes);
BSONObjBuilder indexes; // not using subObjStart to be exception safe
NamespaceDetails::IndexIterator i = d->ii();
while( i.more() ) {
IndexDetails& id = i.next();
log() << "validating index " << idxn << ": " << id.indexNamespace() << endl;
long long keys = id.idxInterface().fullValidate(id.head, id.keyPattern());
indexes.appendNumber(id.indexNamespace(), keys);
idxn++;
}
result.append("keysPerIndex", indexes.done());
}
catch (...) {
errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn));
valid=false;
}
}
catch (AssertionException) {
errors << "exception during validate";
valid = false;
}
result.appendBool("valid", valid);
result.append("errors", errors.arr());
if ( !full ){
result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
}
if ( !valid ) {
result.append("advice", "ns corrupt, requires repair");
}
}
