Status SimpleRecordStoreV1::compact( OperationContext* txn, RecordStoreCompactAdaptor* adaptor, const CompactOptions* options, CompactStats* stats ) { std::vector<DiskLoc> extents; for( DiskLoc extLocation = _details->firstExtent(txn); !extLocation.isNull(); extLocation = _extentManager->getExtent( extLocation )->xnext ) { extents.push_back( extLocation ); } log() << "compact " << extents.size() << " extents"; { WriteUnitOfWork wunit(txn); // Orphaning the deleted lists ensures that all inserts go to new extents rather than // the ones that existed before starting the compact. If we abort the operation before // completion, any free space in the old extents will be leaked and never reused unless // the collection is compacted again or dropped. This is considered an acceptable // failure mode as no data will be lost. log() << "compact orphan deleted lists" << endl; _details->orphanDeletedList(txn); // Start over from scratch with our extent sizing and growth _details->setLastExtentSize( txn, 0 ); // create a new extent so new records go there increaseStorageSize( txn, _details->lastExtentSize(txn), true ); wunit.commit(); } ProgressMeterHolder pm(*txn->setMessage("compact extent", "Extent Compacting Progress", extents.size())); // Go through all old extents and move each record to a new set of extents. int extentNumber = 0; for( std::vector<DiskLoc>::iterator it = extents.begin(); it != extents.end(); it++ ) { txn->checkForInterrupt(); invariant(_details->firstExtent(txn) == *it); // empties and removes the first extent _compactExtent(txn, *it, extentNumber++, adaptor, options, stats ); invariant(_details->firstExtent(txn) != *it); pm.hit(); } invariant( _extentManager->getExtent( _details->firstExtent(txn) )->xprev.isNull() ); invariant( _extentManager->getExtent( _details->lastExtent(txn) )->xnext.isNull() ); // indexes will do their own progress meter pm.finished(); return Status::OK(); }
StatusWith<DiskLoc> SimpleRecordStoreV1::allocRecord( OperationContext* txn, int lengthWithHeaders, bool enforceQuota ) { DiskLoc loc = _allocFromExistingExtents( txn, lengthWithHeaders ); if ( !loc.isNull() ) return StatusWith<DiskLoc>( loc ); LOG(1) << "allocating new extent"; increaseStorageSize( txn, _extentManager->followupSize( lengthWithHeaders, _details->lastExtentSize(txn)), enforceQuota ); loc = _allocFromExistingExtents( txn, lengthWithHeaders ); if ( !loc.isNull() ) { // got on first try return StatusWith<DiskLoc>( loc ); } log() << "warning: alloc() failed after allocating new extent. " << "lengthWithHeaders: " << lengthWithHeaders << " last extent size:" << _details->lastExtentSize(txn) << "; trying again"; for ( int z = 0; z < 10 && lengthWithHeaders > _details->lastExtentSize(txn); z++ ) { log() << "try #" << z << endl; increaseStorageSize( txn, _extentManager->followupSize( lengthWithHeaders, _details->lastExtentSize(txn)), enforceQuota ); loc = _allocFromExistingExtents( txn, lengthWithHeaders ); if ( ! loc.isNull() ) return StatusWith<DiskLoc>( loc ); } return StatusWith<DiskLoc>( ErrorCodes::InternalError, "cannot allocate space" ); }
Status SimpleRecordStoreV1::compact( OperationContext* txn, RecordStoreCompactAdaptor* adaptor, const CompactOptions* options, CompactStats* stats ) { // this is a big job, so might as well make things tidy before we start just to be nice. txn->recoveryUnit()->commitIfNeeded(); list<DiskLoc> extents; for( DiskLoc extLocation = _details->firstExtent(); !extLocation.isNull(); extLocation = _extentManager->getExtent( extLocation )->xnext ) { extents.push_back( extLocation ); } log() << "compact " << extents.size() << " extents"; log() << "compact orphan deleted lists" << endl; _details->orphanDeletedList(txn); // Start over from scratch with our extent sizing and growth _details->setLastExtentSize( txn, 0 ); // create a new extent so new records go there increaseStorageSize( txn, _details->lastExtentSize(), true ); // reset data size and record counts to 0 for this namespace // as we're about to tally them up again for each new extent _details->setStats( txn, 0, 0 ); ProgressMeterHolder pm(*txn->setMessage("compact extent", "Extent Compacting Progress", extents.size())); int extentNumber = 0; for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) { _compactExtent(txn, *i, extentNumber++, adaptor, options, stats ); pm.hit(); } invariant( _extentManager->getExtent( _details->firstExtent() )->xprev.isNull() ); invariant( _extentManager->getExtent( _details->lastExtent() )->xnext.isNull() ); // indexes will do their own progress meter pm.finished(); return Status::OK(); }
StatusWith<CompactStats> Collection::compact( const CompactOptions* compactOptions ) { if ( isCapped() ) return StatusWith<CompactStats>( ErrorCodes::BadValue, "cannot compact capped collection" ); if ( _indexCatalog.numIndexesInProgress() ) return StatusWith<CompactStats>( ErrorCodes::BadValue, "cannot compact when indexes in progress" ); NamespaceDetails* d = details(); // this is a big job, so might as well make things tidy before we start just to be nice. getDur().commitIfNeeded(); list<DiskLoc> extents; for( DiskLoc L = d->firstExtent(); !L.isNull(); L = L.ext()->xnext ) extents.push_back(L); log() << "compact " << extents.size() << " extents" << endl; // same data, but might perform a little different after compact? _infoCache.reset(); vector<BSONObj> indexSpecs; { IndexCatalog::IndexIterator ii( _indexCatalog.getIndexIterator( false ) ); while ( ii.more() ) { IndexDescriptor* descriptor = ii.next(); const BSONObj spec = _compactAdjustIndexSpec(descriptor->infoObj()); const BSONObj key = spec.getObjectField("key"); const Status keyStatus = validateKeyPattern(key); if (!keyStatus.isOK()) { return StatusWith<CompactStats>( ErrorCodes::CannotCreateIndex, str::stream() << "Cannot rebuild index " << spec << ": " << keyStatus.reason() << " For more info see" << " http://dochub.mongodb.org/core/index-validation"); } indexSpecs.push_back(spec); } } log() << "compact orphan deleted lists" << endl; d->orphanDeletedList(); // Start over from scratch with our extent sizing and growth d->setLastExtentSize( 0 ); // before dropping indexes, at least make sure we can allocate one extent! // this will allocate an extent and add to free list // if it cannot, it will throw an exception increaseStorageSize( _details->lastExtentSize(), true ); // note that the drop indexes call also invalidates all clientcursors for the namespace, // which is important and wanted here log() << "compact dropping indexes" << endl; Status status = _indexCatalog.dropAllIndexes( true ); if ( !status.isOK() ) { return StatusWith<CompactStats>( status ); } getDur().commitIfNeeded(); killCurrentOp.checkForInterrupt(); CompactStats stats; MultiIndexBlock multiIndexBlock( this ); status = multiIndexBlock.init( indexSpecs ); if ( !status.isOK() ) return StatusWith<CompactStats>( status ); // reset data size and record counts to 0 for this namespace // as we're about to tally them up again for each new extent d->setStats( 0, 0 ); ProgressMeterHolder pm(cc().curop()->setMessage("compact extent", "Extent Compacting Progress", extents.size())); int extentNumber = 0; for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) { _compactExtent(*i, extentNumber++, multiIndexBlock, compactOptions, &stats ); pm.hit(); } verify( d->firstExtent().ext()->xprev.isNull() ); // indexes will do their own progress meter? pm.finished(); log() << "starting index commits"; status = multiIndexBlock.commit(); if ( !status.isOK() ) return StatusWith<CompactStats>( status ); return StatusWith<CompactStats>( stats ); }
StatusWith<CompactStats> Collection::compact( const CompactOptions* compactOptions ) { if ( isCapped() ) return StatusWith<CompactStats>( ErrorCodes::BadValue, "cannot compact capped collection" ); if ( _indexCatalog.numIndexesInProgress() ) return StatusWith<CompactStats>( ErrorCodes::BadValue, "cannot compact when indexes in progress" ); _database->_initForWrites(); NamespaceDetails* d = details(); // this is a big job, so might as well make things tidy before we start just to be nice. getDur().commitIfNeeded(); list<DiskLoc> extents; for( DiskLoc L = d->firstExtent(); !L.isNull(); L = L.ext()->xnext ) extents.push_back(L); log() << "compact " << extents.size() << " extents" << endl; // same data, but might perform a little different after compact? _infoCache.reset(); vector<BSONObj> indexSpecs; { IndexCatalog::IndexIterator ii( _indexCatalog.getIndexIterator( false ) ); while ( ii.more() ) { IndexDescriptor* descriptor = ii.next(); indexSpecs.push_back( _compactAdjustIndexSpec( descriptor->infoObj() ) ); } } log() << "compact orphan deleted lists" << endl; d->orphanDeletedList(); // Start over from scratch with our extent sizing and growth d->setLastExtentSize( 0 ); // before dropping indexes, at least make sure we can allocate one extent! // this will allocate an extent and add to free list // if it cannot, it will throw an exception increaseStorageSize( _details->lastExtentSize(), true ); // note that the drop indexes call also invalidates all clientcursors for the namespace, // which is important and wanted here log() << "compact dropping indexes" << endl; Status status = _indexCatalog.dropAllIndexes( true ); if ( !status.isOK() ) { return StatusWith<CompactStats>( status ); } getDur().commitIfNeeded(); CompactStats stats; OwnedPointerVector<IndexCatalog::IndexBuildBlock> indexBuildBlocks; vector<IndexAccessMethod*> indexesToInsertTo; vector< std::pair<IndexAccessMethod*,IndexAccessMethod*> > bulkToCommit; for ( size_t i = 0; i < indexSpecs.size(); i++ ) { killCurrentOp.checkForInterrupt(false); BSONObj info = indexSpecs[i]; info = _compactAdjustIndexSpec( info ); info = _indexCatalog.fixIndexSpec( info ); auto_ptr<IndexCatalog::IndexBuildBlock> block( new IndexCatalog::IndexBuildBlock( this,info ) ); Status status = block->init(); if ( !status.isOK() ) return StatusWith<CompactStats>(status); IndexAccessMethod* accessMethod = block->getEntry()->accessMethod(); status = accessMethod->initializeAsEmpty(); if ( !status.isOK() ) return StatusWith<CompactStats>(status); IndexAccessMethod* bulk = accessMethod->initiateBulk(); if ( bulk ) { indexesToInsertTo.push_back( bulk ); bulkToCommit.push_back( std::pair<IndexAccessMethod*,IndexAccessMethod*>( accessMethod, bulk ) ); } else { indexesToInsertTo.push_back( accessMethod ); } indexBuildBlocks.mutableVector().push_back( block.release() ); } // reset data size and record counts to 0 for this namespace // as we're about to tally them up again for each new extent d->setStats( 0, 0 ); ProgressMeterHolder pm(cc().curop()->setMessage("compact extent", "Extent Compacting Progress", extents.size())); int extentNumber = 0; for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) { _compactExtent(*i, extentNumber++, indexesToInsertTo, compactOptions, &stats ); pm.hit(); } verify( d->firstExtent().ext()->xprev.isNull() ); // indexes will do their own progress meter? pm.finished(); log() << "starting index commits"; for ( size_t i = 0; i < bulkToCommit.size(); i++ ) { bulkToCommit[i].first->commitBulk( bulkToCommit[i].second, false, NULL ); } for ( size_t i = 0; i < indexBuildBlocks.size(); i++ ) { IndexCatalog::IndexBuildBlock* block = indexBuildBlocks.mutableVector()[i]; block->success(); } return StatusWith<CompactStats>( stats ); }