int64_t Collection::storageSize( int* numExtents, BSONArrayBuilder* extentInfo ) const { if ( _details->firstExtent().isNull() ) { if ( numExtents ) *numExtents = 0; return 0; } Extent* e = getExtentManager()->getExtent( _details->firstExtent() ); long long total = 0; int n = 0; while ( e ) { total += e->length; n++; if ( extentInfo ) { extentInfo->append( BSON( "len" << e->length << "loc: " << e->myLoc.toBSONObj() ) ); } e = getExtentManager()->getNextExtent( e ); } if ( numExtents ) *numExtents = n; return total; }
void Collection::deleteDocument( const DiskLoc& loc, bool cappedOK, bool noWarn, BSONObj* deletedId ) { if ( _details->isCapped() && !cappedOK ) { log() << "failing remove on a capped ns " << _ns << endl; uasserted( 17115, "cannot remove from a capped collection" ); // XXX 10089 return; } BSONObj doc = docFor( loc ); if ( deletedId ) { BSONElement e = doc["_id"]; if ( e.type() ) { *deletedId = e.wrap(); } } /* check if any cursors point to us. if so, advance them. */ ClientCursor::aboutToDelete(_ns.ns(), _details, loc); Record* rec = getExtentManager()->recordFor( loc ); _indexCatalog.unindexRecord( doc, loc, noWarn); _recordStore.deallocRecord( loc, rec ); _infoCache.notifyOfWriteOp(); }
void Collection::_compactExtent(const DiskLoc diskloc, int extentNumber, MultiIndexBlock& indexesToInsertTo, const CompactOptions* compactOptions, CompactStats* stats ) { log() << "compact begin extent #" << extentNumber << " for namespace " << _ns << " " << diskloc; unsigned oldObjSize = 0; // we'll report what the old padding was unsigned oldObjSizeWithPadding = 0; Extent *e = diskloc.ext(); e->assertOk(); verify( e->validates(diskloc) ); { // the next/prev pointers within the extent might not be in order so we first // page the whole thing in sequentially log() << "compact paging in len=" << e->length/1000000.0 << "MB" << endl; Timer t; size_t length = e->length; touch_pages( reinterpret_cast<const char*>(e), length ); int ms = t.millis(); if( ms > 1000 ) log() << "compact end paging in " << ms << "ms " << e->length/1000000.0/t.seconds() << "MB/sec" << endl; } { log() << "compact copying records" << endl; long long datasize = 0; long long nrecords = 0; DiskLoc L = e->firstRecord; if( !L.isNull() ) { while( 1 ) { Record *recOld = L.rec(); L = getExtentManager()->getNextRecordInExtent(L); BSONObj objOld = BSONObj::make(recOld); if ( compactOptions->validateDocuments && !objOld.valid() ) { // object is corrupt! log() << "compact skipping corrupt document!"; stats->corruptDocuments++; } else { unsigned docSize = objOld.objsize(); nrecords++; oldObjSize += docSize; oldObjSizeWithPadding += recOld->netLength(); unsigned lenWHdr = docSize + Record::HeaderSize; unsigned lenWPadding = lenWHdr; switch( compactOptions->paddingMode ) { case CompactOptions::NONE: if ( details()->isUserFlagSet(NamespaceDetails::Flag_UsePowerOf2Sizes) ) lenWPadding = details()->quantizePowerOf2AllocationSpace(lenWPadding); break; case CompactOptions::PRESERVE: // if we are preserving the padding, the record should not change size lenWPadding = recOld->lengthWithHeaders(); break; case CompactOptions::MANUAL: lenWPadding = compactOptions->computeRecordSize(lenWPadding); if (lenWPadding < lenWHdr || lenWPadding > BSONObjMaxUserSize / 2 ) { lenWPadding = lenWHdr; } break; } CompactDocWriter writer( objOld, lenWPadding ); StatusWith<DiskLoc> status = _recordStore->insertRecord( &writer, 0 ); uassertStatusOK( status.getStatus() ); datasize += _recordStore->recordFor( status.getValue() )->netLength(); InsertDeleteOptions options; options.logIfError = false; options.dupsAllowed = true; // in compact we should be doing no checking indexesToInsertTo.insert( objOld, status.getValue(), options ); } if( L.isNull() ) { // we just did the very last record from the old extent. it's still pointed to // by the old extent ext, but that will be fixed below after this loop break; } // remove the old records (orphan them) periodically so our commit block doesn't get too large bool stopping = false; RARELY stopping = *killCurrentOp.checkForInterruptNoAssert() != 0; if( stopping || getDur().aCommitIsNeeded() ) { e->firstRecord.writing() = L; Record *r = L.rec(); getDur().writingInt(r->prevOfs()) = DiskLoc::NullOfs; getDur().commitIfNeeded(); killCurrentOp.checkForInterrupt(); } } } // if !L.isNull() verify( details()->firstExtent() == diskloc ); verify( details()->lastExtent() != diskloc ); DiskLoc newFirst = e->xnext; details()->firstExtent().writing() = newFirst; newFirst.ext()->xprev.writing().Null(); getDur().writing(e)->markEmpty(); getExtentManager()->freeExtents( diskloc, diskloc ); getDur().commitIfNeeded(); { double op = 1.0; if( oldObjSize ) op = static_cast<double>(oldObjSizeWithPadding)/oldObjSize; log() << "compact finished extent #" << extentNumber << " containing " << nrecords << " documents (" << datasize/1000000.0 << "MB)" << " oldPadding: " << op << ' ' << static_cast<unsigned>(op*100.0)/100; } } }
void Database::getStats( OperationContext* opCtx, BSONObjBuilder* output, double scale ) { bool empty = isEmpty() || getExtentManager()->numFiles() == 0; list<string> collections; if ( !empty ) _dbEntry->getCollectionNamespaces( &collections ); long long ncollections = 0; long long objects = 0; long long size = 0; long long storageSize = 0; long long numExtents = 0; long long indexes = 0; long long indexSize = 0; for (list<string>::const_iterator it = collections.begin(); it != collections.end(); ++it) { const string ns = *it; Collection* collection = getCollection( opCtx, ns ); if ( !collection ) continue; ncollections += 1; objects += collection->numRecords(); size += collection->dataSize(); BSONObjBuilder temp; storageSize += collection->getRecordStore()->storageSize( &temp ); numExtents += temp.obj()["numExtents"].numberInt(); // XXX indexes += collection->getIndexCatalog()->numIndexesTotal(); indexSize += getIndexSizeForCollection(opCtx, collection); } output->append ( "db" , _name ); output->appendNumber( "collections" , ncollections ); output->appendNumber( "objects" , objects ); output->append ( "avgObjSize" , objects == 0 ? 0 : double(size) / double(objects) ); output->appendNumber( "dataSize" , size / scale ); output->appendNumber( "storageSize" , storageSize / scale); output->appendNumber( "numExtents" , numExtents ); output->appendNumber( "indexes" , indexes ); output->appendNumber( "indexSize" , indexSize / scale ); if ( !empty ) { output->appendNumber( "fileSize" , fileSize() / scale ); output->appendNumber( "nsSizeMB", (int)_dbEntry->namespaceIndex().fileLength() / 1024 / 1024 ); } else { output->appendNumber( "fileSize" , 0 ); } BSONObjBuilder dataFileVersion( output->subobjStart( "dataFileVersion" ) ); if ( !empty ) { int major, minor; getFileFormat( opCtx, &major, &minor ); dataFileVersion.append( "major", major ); dataFileVersion.append( "minor", minor ); } dataFileVersion.done(); if ( !empty ){ int freeListSize = 0; int64_t freeListSpace = 0; getExtentManager()->freeListStats( &freeListSize, &freeListSpace ); BSONObjBuilder extentFreeList( output->subobjStart( "extentFreeList" ) ); extentFreeList.append( "num", freeListSize ); extentFreeList.appendNumber( "totalSize", static_cast<long long>( freeListSpace / scale ) ); extentFreeList.done(); } }
void Database::flushFiles( bool sync ) { return getExtentManager()->flushFiles( sync ); }
int Database::numFiles() const { return getExtentManager()->numFiles(); }
long long Database::fileSize() const { return getExtentManager()->fileSize(); }
StatusWith<CompactStats> Collection::compact( const CompactOptions* compactOptions ) { if ( isCapped() ) return StatusWith<CompactStats>( ErrorCodes::BadValue, "cannot compact capped collection" ); if ( _indexCatalog.numIndexesInProgress() ) return StatusWith<CompactStats>( ErrorCodes::BadValue, "cannot compact when indexes in progress" ); // this is a big job, so might as well make things tidy before we start just to be nice. getDur().commitIfNeeded(); list<DiskLoc> extents; for( DiskLoc extLocation = _details->firstExtent(); !extLocation.isNull(); extLocation = getExtentManager()->getExtent( extLocation )->xnext ) { extents.push_back( extLocation ); } log() << "compact " << extents.size() << " extents"; // same data, but might perform a little different after compact? _infoCache.reset(); vector<BSONObj> indexSpecs; { IndexCatalog::IndexIterator ii( _indexCatalog.getIndexIterator( false ) ); while ( ii.more() ) { IndexDescriptor* descriptor = ii.next(); const BSONObj spec = _compactAdjustIndexSpec(descriptor->infoObj()); const BSONObj key = spec.getObjectField("key"); const Status keyStatus = validateKeyPattern(key); if (!keyStatus.isOK()) { return StatusWith<CompactStats>( ErrorCodes::CannotCreateIndex, str::stream() << "Cannot compact collection due to invalid index " << spec << ": " << keyStatus.reason() << " For more info see" << " http://dochub.mongodb.org/core/index-validation"); } indexSpecs.push_back(spec); } } log() << "compact orphan deleted lists" << endl; _details->orphanDeletedList(); // Start over from scratch with our extent sizing and growth _details->setLastExtentSize( 0 ); // before dropping indexes, at least make sure we can allocate one extent! // this will allocate an extent and add to free list // if it cannot, it will throw an exception increaseStorageSize( _details->lastExtentSize(), true ); // note that the drop indexes call also invalidates all clientcursors for the namespace, // which is important and wanted here log() << "compact dropping indexes" << endl; Status status = _indexCatalog.dropAllIndexes( true ); if ( !status.isOK() ) { return StatusWith<CompactStats>( status ); } getDur().commitIfNeeded(); killCurrentOp.checkForInterrupt(); CompactStats stats; MultiIndexBlock multiIndexBlock( this ); status = multiIndexBlock.init( indexSpecs ); if ( !status.isOK() ) return StatusWith<CompactStats>( status ); // reset data size and record counts to 0 for this namespace // as we're about to tally them up again for each new extent _details->setStats( 0, 0 ); ProgressMeterHolder pm(cc().curop()->setMessage("compact extent", "Extent Compacting Progress", extents.size())); int extentNumber = 0; for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) { _compactExtent(*i, extentNumber++, multiIndexBlock, compactOptions, &stats ); pm.hit(); } invariant( getExtentManager()->getExtent( _details->firstExtent() )->xprev.isNull() ); // indexes will do their own progress meter pm.finished(); log() << "starting index commits"; status = multiIndexBlock.commit(); if ( !status.isOK() ) return StatusWith<CompactStats>( status ); return StatusWith<CompactStats>( stats ); }
Extent* Collection::increaseStorageSize( int size, bool enforceQuota ) { return getExtentManager()->increaseStorageSize( _ns, _details, size, enforceQuota ? largestFileNumberInQuota() : 0 ); }
StatusWith<DiskLoc> Collection::updateDocument( const DiskLoc& oldLocation, const BSONObj& objNew, bool enforceQuota, OpDebug* debug ) { Record* oldRecord = getExtentManager()->recordFor( oldLocation ); BSONObj objOld = BSONObj::make( oldRecord ); if ( objOld.hasElement( "_id" ) ) { BSONElement oldId = objOld["_id"]; BSONElement newId = objNew["_id"]; if ( oldId != newId ) return StatusWith<DiskLoc>( ErrorCodes::InternalError, "in Collection::updateDocument _id mismatch", 13596 ); } if ( ns().coll() == "system.users" ) { // XXX - andy and spencer think this should go away now V2UserDocumentParser parser; Status s = parser.checkValidUserDocument(objNew); if ( !s.isOK() ) return StatusWith<DiskLoc>( s ); } /* duplicate key check. we descend the btree twice - once for this check, and once for the actual inserts, further below. that is suboptimal, but it's pretty complicated to do it the other way without rollbacks... */ OwnedPointerVector<UpdateTicket> updateTickets; updateTickets.mutableVector().resize(_indexCatalog.numIndexesTotal()); for (int i = 0; i < _indexCatalog.numIndexesTotal(); ++i) { IndexDescriptor* descriptor = _indexCatalog.getDescriptor( i ); IndexAccessMethod* iam = _indexCatalog.getIndex( descriptor ); InsertDeleteOptions options; options.logIfError = false; options.dupsAllowed = !(KeyPattern::isIdKeyPattern(descriptor->keyPattern()) || descriptor->unique()) || ignoreUniqueIndex(descriptor); updateTickets.mutableVector()[i] = new UpdateTicket(); Status ret = iam->validateUpdate(objOld, objNew, oldLocation, options, updateTickets.mutableVector()[i]); if ( !ret.isOK() ) { return StatusWith<DiskLoc>( ret ); } } if ( oldRecord->netLength() < objNew.objsize() ) { // doesn't fit, have to move to new location if ( _details->isCapped() ) return StatusWith<DiskLoc>( ErrorCodes::InternalError, "failing update: objects in a capped ns cannot grow", 10003 ); moveCounter.increment(); _details->paddingTooSmall(); // unindex old record, don't delete // this way, if inserting new doc fails, we can re-index this one ClientCursor::aboutToDelete(_ns.ns(), _details, oldLocation); _indexCatalog.unindexRecord( objOld, oldLocation, true ); if ( debug ) { if (debug->nmoved == -1) // default of -1 rather than 0 debug->nmoved = 1; else debug->nmoved += 1; } StatusWith<DiskLoc> loc = insertDocument( objNew, enforceQuota ); if ( loc.isOK() ) { // insert successful, now lets deallocate the old location // remember its already unindexed _recordStore.deallocRecord( oldLocation, oldRecord ); } else { // new doc insert failed, so lets re-index the old document and location _indexCatalog.indexRecord( objOld, oldLocation ); } return loc; } _infoCache.notifyOfWriteOp(); _details->paddingFits(); if ( debug ) debug->keyUpdates = 0; for (int i = 0; i < _indexCatalog.numIndexesTotal(); ++i) { IndexDescriptor* descriptor = _indexCatalog.getDescriptor( i ); IndexAccessMethod* iam = _indexCatalog.getIndex( descriptor ); int64_t updatedKeys; Status ret = iam->update(*updateTickets.vector()[i], &updatedKeys); if ( !ret.isOK() ) return StatusWith<DiskLoc>( ret ); if ( debug ) debug->keyUpdates += updatedKeys; } // update in place int sz = objNew.objsize(); memcpy(getDur().writingPtr(oldRecord->data(), sz), objNew.objdata(), sz); return StatusWith<DiskLoc>( oldLocation ); }
BSONObj Collection::docFor( const DiskLoc& loc ) { Record* rec = getExtentManager()->recordFor( loc ); return BSONObj::make( rec->accessed() ); }