bool Helpers::findById(Database* database, const char *ns, BSONObj query, BSONObj& result ,
bool* nsFound , bool* indexFound ) {
Lock::assertAtLeastReadLocked(ns);
invariant( database );
Collection* collection = database->getCollection( ns );
if ( !collection ) {
return false;
}
if ( nsFound )
*nsFound = true;
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex();
if ( !desc )
return false;
if ( indexFound )
*indexFound = 1;
// See SERVER-12397. This may not always be true.
BtreeBasedAccessMethod* accessMethod =
static_cast<BtreeBasedAccessMethod*>(catalog->getIndex( desc ));
DiskLoc loc = accessMethod->findSingle( query["_id"].wrap() );
if ( loc.isNull() )
return false;
result = collection->docFor( loc );
return true;
}
bool Helpers::findById(OperationContext* opCtx,
Database* database,
StringData ns,
BSONObj query,
BSONObj& result,
bool* nsFound,
bool* indexFound) {
invariant(database);
Collection* collection = database->getCollection(opCtx, ns);
if (!collection) {
return false;
}
if (nsFound)
*nsFound = true;
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex(opCtx);
if (!desc)
return false;
if (indexFound)
*indexFound = 1;
RecordId loc = catalog->getIndex(desc)->findSingle(opCtx, query["_id"].wrap());
if (loc.isNull())
return false;
result = collection->docFor(opCtx, loc).value();
return true;
}
RecordId Helpers::findById(OperationContext* txn, Collection* collection, const BSONObj& idquery) {
verify(collection);
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex(txn);
uassert(13430, "no _id index", desc);
return catalog->getIndex(desc)->findSingle(txn, idquery["_id"].wrap());
}
DiskLoc Helpers::findById(Collection* collection, const BSONObj& idquery) {
verify(collection);
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex();
uassert(13430, "no _id index", desc);
// See SERVER-12397. This may not always be true.
BtreeBasedAccessMethod* accessMethod =
static_cast<BtreeBasedAccessMethod*>(catalog->getIndex( desc ));
return accessMethod->findSingle( idquery["_id"].wrap() );
}
uint64_t Collection::getIndexSize(OperationContext* opCtx, BSONObjBuilder* details, int scale) {
IndexCatalog* idxCatalog = getIndexCatalog();
IndexCatalog::IndexIterator ii = idxCatalog->getIndexIterator(opCtx, true);
uint64_t totalSize = 0;
while (ii.more()) {
IndexDescriptor* d = ii.next();
IndexAccessMethod* iam = idxCatalog->getIndex(d);
long long ds = iam->getSpaceUsedBytes(opCtx);
totalSize += ds;
if (details) {
details->appendNumber(d->indexName(), ds / scale);
}
}
return totalSize;
}
bool Helpers::findById(OperationContext* txn,
Database* database,
const char *ns,
BSONObj query,
BSONObj& result,
bool* nsFound,
bool* indexFound) {
invariant(database);
Collection* collection = database->getCollection( ns );
if ( !collection ) {
return false;
}
if ( nsFound )
*nsFound = true;
IndexCatalog* catalog = collection->getIndexCatalog();
const IndexDescriptor* desc = catalog->findIdIndex( txn );
if ( !desc )
return false;
if ( indexFound )
*indexFound = 1;
// See SERVER-12397. This may not always be true.
BtreeBasedAccessMethod* accessMethod =
static_cast<BtreeBasedAccessMethod*>(catalog->getIndex( desc ));
RecordId loc = accessMethod->findSingle( txn, query["_id"].wrap() );
if ( loc.isNull() )
return false;
result = collection->docFor(txn, loc).value();
return true;
}
PlanStage::StageState TwoDNear::work(WorkingSetID* out) {
++_commonStats.works;
// Adds the amount of time taken by work() to executionTimeMillis.
ScopedTimer timer(&_commonStats.executionTimeMillis);
if (!_initted) {
_initted = true;
if ( !_params.collection )
return PlanStage::IS_EOF;
IndexCatalog* indexCatalog = _params.collection->getIndexCatalog();
IndexDescriptor* desc = indexCatalog->findIndexByKeyPattern(_params.indexKeyPattern);
if ( desc == NULL )
return PlanStage::IS_EOF;
TwoDAccessMethod* am = static_cast<TwoDAccessMethod*>( indexCatalog->getIndex( desc ) );
auto_ptr<twod_exec::GeoSearch> search;
search.reset(new twod_exec::GeoSearch(_params.collection,
am,
_params.nearQuery.centroid.oldPoint,
_params.numWanted,
_params.filter,
_params.nearQuery.maxDistance,
_params.nearQuery.isNearSphere ? twod_exec::GEO_SPHERE
: twod_exec::GEO_PLANE));
// This is where all the work is done. :(
search->exec();
_specificStats.objectsLoaded = search->_objectsLoaded;
_specificStats.nscanned = search->_lookedAt;
for (twod_exec::GeoHopper::Holder::iterator it = search->_points.begin();
it != search->_points.end(); it++) {
WorkingSetID id = _workingSet->allocate();
WorkingSetMember* member = _workingSet->get(id);
member->loc = it->_loc;
member->obj = _params.collection->docFor(member->loc);
member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ;
if (_params.addDistMeta) {
member->addComputed(new GeoDistanceComputedData(it->_distance));
}
if (_params.addPointMeta) {
member->addComputed(new GeoNearPointComputedData(it->_pt));
}
_results.push(Result(id, it->_distance));
_invalidationMap.insert(pair<DiskLoc, WorkingSetID>(it->_loc, id));
}
}
if (isEOF()) { return PlanStage::IS_EOF; }
Result result = _results.top();
_results.pop();
*out = result.id;
// Remove from invalidation map.
WorkingSetMember* member = _workingSet->get(*out);
// The WSM may have been mutated or deleted so it may not have a loc.
if (member->hasLoc()) {
typedef multimap<DiskLoc, WorkingSetID>::iterator MMIT;
pair<MMIT, MMIT> range = _invalidationMap.equal_range(member->loc);
for (MMIT it = range.first; it != range.second; ++it) {
if (it->second == *out) {
_invalidationMap.erase(it);
break;
}
}
}
++_commonStats.advanced;
return PlanStage::ADVANCED;
}
Status appendCollectionStorageStats(OperationContext* opCtx,
const NamespaceString& nss,
const BSONObj& param,
BSONObjBuilder* result) {
int scale = 1;
if (param["scale"].isNumber()) {
scale = param["scale"].numberInt();
if (scale < 1) {
return {ErrorCodes::BadValue, "scale has to be >= 1"};
}
} else if (param["scale"].trueValue()) {
return {ErrorCodes::BadValue, "scale has to be a number >= 1"};
}
bool verbose = param["verbose"].trueValue();
AutoGetCollectionForReadCommand ctx(opCtx, nss);
Collection* collection = ctx.getCollection(); // Will be set if present
if (!ctx.getDb() || !collection) {
result->appendNumber("size", 0);
result->appendNumber("count", 0);
result->appendNumber("storageSize", 0);
result->append("nindexes", 0);
result->appendNumber("totalIndexSize", 0);
result->append("indexDetails", BSONObj());
result->append("indexSizes", BSONObj());
std::string errmsg = !(ctx.getDb()) ? "Database [" + nss.db().toString() + "] not found."
: "Collection [" + nss.toString() + "] not found.";
return {ErrorCodes::NamespaceNotFound, errmsg};
}
long long size = collection->dataSize(opCtx) / scale;
result->appendNumber("size", size);
long long numRecords = collection->numRecords(opCtx);
result->appendNumber("count", numRecords);
if (numRecords)
result->append("avgObjSize", collection->averageObjectSize(opCtx));
RecordStore* recordStore = collection->getRecordStore();
result->appendNumber(
"storageSize",
static_cast<long long>(recordStore->storageSize(opCtx, result, verbose ? 1 : 0)) / scale);
recordStore->appendCustomStats(opCtx, result, scale);
IndexCatalog* indexCatalog = collection->getIndexCatalog();
result->append("nindexes", indexCatalog->numIndexesReady(opCtx));
BSONObjBuilder indexDetails;
IndexCatalog::IndexIterator i = indexCatalog->getIndexIterator(opCtx, false);
while (i.more()) {
const IndexDescriptor* descriptor = i.next();
IndexAccessMethod* iam = indexCatalog->getIndex(descriptor);
invariant(iam);
BSONObjBuilder bob;
if (iam->appendCustomStats(opCtx, &bob, scale)) {
indexDetails.append(descriptor->indexName(), bob.obj());
}
}
result->append("indexDetails", indexDetails.obj());
BSONObjBuilder indexSizes;
long long indexSize = collection->getIndexSize(opCtx, &indexSizes, scale);
result->appendNumber("totalIndexSize", indexSize / scale);
result->append("indexSizes", indexSizes.obj());
return Status::OK();
}
void validateNS(const string& ns,
Collection* collection,
const BSONObj& cmdObj,
BSONObjBuilder& result) {
const bool full = cmdObj["full"].trueValue();
const bool scanData = full || cmdObj["scandata"].trueValue();
NamespaceDetails* nsd = collection->details();
bool valid = true;
BSONArrayBuilder errors; // explanation(s) for why valid = false
if ( collection->isCapped() ){
result.append("capped", nsd->isCapped());
result.appendNumber("max", nsd->maxCappedDocs());
}
if ( nsd->firstExtent().isNull() )
result.append( "firstExtent", "null" );
else
result.append( "firstExtent", str::stream() << nsd->firstExtent().toString()
<< " ns:" << nsd->firstExtent().ext()->nsDiagnostic.toString());
if ( nsd->lastExtent().isNull() )
result.append( "lastExtent", "null" );
else
result.append( "lastExtent", str::stream() << nsd->lastExtent().toString()
<< " ns:" << nsd->lastExtent().ext()->nsDiagnostic.toString());
BSONArrayBuilder extentData;
int extentCount = 0;
try {
if ( !nsd->firstExtent().isNull() ) {
nsd->firstExtent().ext()->assertOk();
nsd->lastExtent().ext()->assertOk();
}
DiskLoc extentDiskLoc = nsd->firstExtent();
while (!extentDiskLoc.isNull()) {
Extent* thisExtent = extentDiskLoc.ext();
if (full) {
extentData << thisExtent->dump();
}
if (!thisExtent->validates(extentDiskLoc, &errors)) {
valid = false;
}
DiskLoc nextDiskLoc = thisExtent->xnext;
if (extentCount > 0 && !nextDiskLoc.isNull()
&& nextDiskLoc.ext()->xprev != extentDiskLoc) {
StringBuilder sb;
sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString()
<< " in extent " << nextDiskLoc.toString()
<< " does not point to extent " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
if (nextDiskLoc.isNull() && extentDiskLoc != nsd->lastExtent()) {
StringBuilder sb;
sb << "'lastExtent' pointer " << nsd->lastExtent().toString()
<< " does not point to last extent in list " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
extentDiskLoc = nextDiskLoc;
extentCount++;
killCurrentOp.checkForInterrupt();
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception validating extent " << extentCount
<< ": " << e.what();
errors << sb.str();
valid = false;
}
result.append("extentCount", extentCount);
if ( full )
result.appendArray( "extents" , extentData.arr() );
result.appendNumber("datasize", nsd->dataSize());
result.appendNumber("nrecords", nsd->numRecords());
result.appendNumber("lastExtentSize", nsd->lastExtentSize());
result.appendNumber("padding", nsd->paddingFactor());
try {
bool testingLastExtent = false;
try {
if (nsd->firstExtent().isNull()) {
// this is ok
}
else {
result.append("firstExtentDetails", nsd->firstExtent().ext()->dump());
if (!nsd->firstExtent().ext()->xprev.isNull()) {
StringBuilder sb;
sb << "'xprev' pointer in 'firstExtent' " << nsd->firstExtent().toString()
<< " is " << nsd->firstExtent().ext()->xprev.toString()
<< ", should be null";
errors << sb.str();
valid=false;
}
}
testingLastExtent = true;
if (nsd->lastExtent().isNull()) {
// this is ok
}
else {
if (nsd->firstExtent() != nsd->lastExtent()) {
result.append("lastExtentDetails", nsd->lastExtent().ext()->dump());
if (!nsd->lastExtent().ext()->xnext.isNull()) {
StringBuilder sb;
sb << "'xnext' pointer in 'lastExtent' " << nsd->lastExtent().toString()
<< " is " << nsd->lastExtent().ext()->xnext.toString()
<< ", should be null";
errors << sb.str();
valid = false;
}
}
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception processing '"
<< (testingLastExtent ? "lastExtent" : "firstExtent")
<< "': " << e.what();
errors << sb.str();
valid = false;
}
set<DiskLoc> recs;
if( scanData ) {
int n = 0;
int nInvalid = 0;
long long nQuantizedSize = 0;
long long nPowerOf2QuantizedSize = 0;
long long len = 0;
long long nlen = 0;
long long bsonLen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
DiskLoc cl;
Runner::RunnerState state;
auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns));
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &cl))) {
n++;
if ( n < 1000000 )
recs.insert(cl);
if ( nsd->isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = cl.rec();
len += r->lengthWithHeaders();
nlen += r->netLength();
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizeAllocationSpace
( r->lengthWithHeaders() ) ) {
// Count the number of records having a size consistent with
// the quantizeAllocationSpace quantization implementation.
++nQuantizedSize;
}
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizePowerOf2AllocationSpace
( r->lengthWithHeaders() - 1 ) ) {
// Count the number of records having a size consistent with the
// quantizePowerOf2AllocationSpace quantization implementation.
// Because of SERVER-8311, power of 2 quantization is not idempotent and
// r->lengthWithHeaders() - 1 must be checked instead of the record
// length itself.
++nPowerOf2QuantizedSize;
}
if (full){
BSONObj obj = BSONObj::make(r);
if (!obj.isValid() || !obj.valid()){ // both fast and deep checks
valid = false;
if (nInvalid == 0) // only log once;
errors << "invalid bson object detected (see logs for more info)";
nInvalid++;
if (strcmp("_id", obj.firstElementFieldName()) == 0){
try {
obj.firstElement().validate(); // throws on error
log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl;
}
catch(...){
log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl;
}
}
else {
log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl;
}
}
else {
bsonLen += obj.objsize();
}
}
}
if (Runner::RUNNER_EOF != state) {
// TODO: more descriptive logging.
warning() << "Internal error while reading collection " << ns << endl;
}
if ( nsd->isCapped() && !nsd->capLooped() ) {
result.append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
valid = false;
errors << "too many out of order records";
}
}
result.append("objectsFound", n);
if (full) {
result.append("invalidObjects", nInvalid);
}
result.appendNumber("nQuantizedSize", nQuantizedSize);
result.appendNumber("nPowerOf2QuantizedSize", nPowerOf2QuantizedSize);
result.appendNumber("bytesWithHeaders", len);
result.appendNumber("bytesWithoutHeaders", nlen);
if (full) {
result.appendNumber("bytesBson", bsonLen);
}
}
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << nsd->deletedListEntry(i).isNull();
}
int ndel = 0;
long long delSize = 0;
BSONArrayBuilder delBucketSizes;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = nsd->deletedListEntry(i);
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( nsd->isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
string err( str::stream() << "bad pointer in deleted record list: "
<< loc.toString()
<< " bucket: " << i
<< " k: " << k );
errors << err;
valid = false;
break;
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
killCurrentOp.checkForInterrupt();
}
delBucketSizes << k;
}
catch (...) {
errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i));
valid = false;
}
}
result.appendNumber("deletedCount", ndel);
result.appendNumber("deletedSize", delSize);
if ( full ) {
result << "delBucketSizes" << delBucketSizes.arr();
}
if ( incorrect ) {
errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list");
valid = false;
}
int idxn = 0;
try {
IndexCatalog* indexCatalog = collection->getIndexCatalog();
result.append("nIndexes", nsd->getCompletedIndexCount());
BSONObjBuilder indexes; // not using subObjStart to be exception safe
NamespaceDetails::IndexIterator i = nsd->ii();
while( i.more() ) {
IndexDetails& id = i.next();
log() << "validating index " << idxn << ": " << id.indexNamespace() << endl;
IndexDescriptor* descriptor = indexCatalog->getDescriptor( idxn );
verify( descriptor );
IndexAccessMethod* iam = indexCatalog->getIndex( descriptor );
verify( iam );
int64_t keys;
iam->validate(&keys);
indexes.appendNumber(id.indexNamespace(), static_cast<long long>(keys));
idxn++;
}
result.append("keysPerIndex", indexes.done());
}
catch (...) {
errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn));
valid=false;
}
}
catch (AssertionException) {
errors << "exception during validate";
valid = false;
}
result.appendBool("valid", valid);
result.append("errors", errors.arr());
if ( !full ){
result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
}
if ( !valid ) {
result.append("advice", "ns corrupt, requires repair");
}
}
PlanStage::StageState TwoD::work(WorkingSetID* out) {
if (isEOF()) { return PlanStage::IS_EOF; }
if (!_initted) {
_initted = true;
if ( !_params.collection )
return PlanStage::IS_EOF;
IndexCatalog* indexCatalog = _params.collection->getIndexCatalog();
_descriptor = indexCatalog->findIndexByKeyPattern(_params.indexKeyPattern);
if ( _descriptor == NULL )
return PlanStage::IS_EOF;
_am = static_cast<TwoDAccessMethod*>( indexCatalog->getIndex( _descriptor ) );
verify( _am );
if (NULL != _params.gq.getGeometry()._cap.get()) {
_browse.reset(new twod_exec::GeoCircleBrowse(_params, _am));
}
else if (NULL != _params.gq.getGeometry()._polygon.get()) {
_browse.reset(new twod_exec::GeoPolygonBrowse(_params, _am));
}
else {
verify(NULL != _params.gq.getGeometry()._box.get());
_browse.reset(new twod_exec::GeoBoxBrowse(_params, _am));
}
// Fill out static portion of plan stats.
// We will retrieve the geo hashes used by the geo browser
// when the search is complete.
_specificStats.type = _browse->_type;
_specificStats.field = _params.gq.getField();
_specificStats.converterParams = _browse->_converter->getParams();
return PlanStage::NEED_TIME;
}
verify(NULL != _browse.get());
if (!_browse->ok()) {
// Grab geo hashes before disposing geo browser.
_specificStats.expPrefixes.swap(_browse->_expPrefixes);
_browse.reset();
return PlanStage::IS_EOF;
}
WorkingSetID id = _workingSet->allocate();
WorkingSetMember* member = _workingSet->get(id);
member->loc = _browse->currLoc();
member->obj = _params.collection->docFor(member->loc);
member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ;
_browse->advance();
*out = id;
_commonStats.advanced++;
_commonStats.works++;
return PlanStage::ADVANCED;
}