Status SimpleRecordStoreV1::compact( OperationContext* txn,
RecordStoreCompactAdaptor* adaptor,
const CompactOptions* options,
CompactStats* stats ) {
std::vector<DiskLoc> extents;
for( DiskLoc extLocation = _details->firstExtent(txn);
!extLocation.isNull();
extLocation = _extentManager->getExtent( extLocation )->xnext ) {
extents.push_back( extLocation );
}
log() << "compact " << extents.size() << " extents";
{
WriteUnitOfWork wunit(txn);
// Orphaning the deleted lists ensures that all inserts go to new extents rather than
// the ones that existed before starting the compact. If we abort the operation before
// completion, any free space in the old extents will be leaked and never reused unless
// the collection is compacted again or dropped. This is considered an acceptable
// failure mode as no data will be lost.
log() << "compact orphan deleted lists" << endl;
_details->orphanDeletedList(txn);
// Start over from scratch with our extent sizing and growth
_details->setLastExtentSize( txn, 0 );
// create a new extent so new records go there
increaseStorageSize( txn, _details->lastExtentSize(txn), true );
wunit.commit();
}
ProgressMeterHolder pm(*txn->setMessage("compact extent",
"Extent Compacting Progress",
extents.size()));
// Go through all old extents and move each record to a new set of extents.
int extentNumber = 0;
for( std::vector<DiskLoc>::iterator it = extents.begin(); it != extents.end(); it++ ) {
txn->checkForInterrupt();
invariant(_details->firstExtent(txn) == *it);
// empties and removes the first extent
_compactExtent(txn, *it, extentNumber++, adaptor, options, stats );
invariant(_details->firstExtent(txn) != *it);
pm.hit();
}
invariant( _extentManager->getExtent( _details->firstExtent(txn) )->xprev.isNull() );
invariant( _extentManager->getExtent( _details->lastExtent(txn) )->xnext.isNull() );
// indexes will do their own progress meter
pm.finished();
return Status::OK();
}
StatusWith<DiskLoc> SimpleRecordStoreV1::allocRecord( OperationContext* txn,
int lengthWithHeaders,
bool enforceQuota ) {
DiskLoc loc = _allocFromExistingExtents( txn, lengthWithHeaders );
if ( !loc.isNull() )
return StatusWith<DiskLoc>( loc );
LOG(1) << "allocating new extent";
increaseStorageSize( txn,
_extentManager->followupSize( lengthWithHeaders,
_details->lastExtentSize(txn)),
enforceQuota );
loc = _allocFromExistingExtents( txn, lengthWithHeaders );
if ( !loc.isNull() ) {
// got on first try
return StatusWith<DiskLoc>( loc );
}
log() << "warning: alloc() failed after allocating new extent. "
<< "lengthWithHeaders: " << lengthWithHeaders << " last extent size:"
<< _details->lastExtentSize(txn) << "; trying again";
for ( int z = 0; z < 10 && lengthWithHeaders > _details->lastExtentSize(txn); z++ ) {
log() << "try #" << z << endl;
increaseStorageSize( txn,
_extentManager->followupSize( lengthWithHeaders,
_details->lastExtentSize(txn)),
enforceQuota );
loc = _allocFromExistingExtents( txn, lengthWithHeaders );
if ( ! loc.isNull() )
return StatusWith<DiskLoc>( loc );
}
return StatusWith<DiskLoc>( ErrorCodes::InternalError, "cannot allocate space" );
}
Status SimpleRecordStoreV1::compact( OperationContext* txn,
RecordStoreCompactAdaptor* adaptor,
const CompactOptions* options,
CompactStats* stats ) {
// this is a big job, so might as well make things tidy before we start just to be nice.
txn->recoveryUnit()->commitIfNeeded();
list<DiskLoc> extents;
for( DiskLoc extLocation = _details->firstExtent();
!extLocation.isNull();
extLocation = _extentManager->getExtent( extLocation )->xnext ) {
extents.push_back( extLocation );
}
log() << "compact " << extents.size() << " extents";
log() << "compact orphan deleted lists" << endl;
_details->orphanDeletedList(txn);
// Start over from scratch with our extent sizing and growth
_details->setLastExtentSize( txn, 0 );
// create a new extent so new records go there
increaseStorageSize( txn, _details->lastExtentSize(), true );
// reset data size and record counts to 0 for this namespace
// as we're about to tally them up again for each new extent
_details->setStats( txn, 0, 0 );
ProgressMeterHolder pm(*txn->setMessage("compact extent",
"Extent Compacting Progress",
extents.size()));
int extentNumber = 0;
for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) {
_compactExtent(txn, *i, extentNumber++, adaptor, options, stats );
pm.hit();
}
invariant( _extentManager->getExtent( _details->firstExtent() )->xprev.isNull() );
invariant( _extentManager->getExtent( _details->lastExtent() )->xnext.isNull() );
// indexes will do their own progress meter
pm.finished();
return Status::OK();
}
StatusWith<CompactStats> Collection::compact( const CompactOptions* compactOptions ) {
if ( isCapped() )
return StatusWith<CompactStats>( ErrorCodes::BadValue,
"cannot compact capped collection" );
if ( _indexCatalog.numIndexesInProgress() )
return StatusWith<CompactStats>( ErrorCodes::BadValue,
"cannot compact when indexes in progress" );
NamespaceDetails* d = details();
// this is a big job, so might as well make things tidy before we start just to be nice.
getDur().commitIfNeeded();
list<DiskLoc> extents;
for( DiskLoc L = d->firstExtent(); !L.isNull(); L = L.ext()->xnext )
extents.push_back(L);
log() << "compact " << extents.size() << " extents" << endl;
// same data, but might perform a little different after compact?
_infoCache.reset();
vector<BSONObj> indexSpecs;
{
IndexCatalog::IndexIterator ii( _indexCatalog.getIndexIterator( false ) );
while ( ii.more() ) {
IndexDescriptor* descriptor = ii.next();
const BSONObj spec = _compactAdjustIndexSpec(descriptor->infoObj());
const BSONObj key = spec.getObjectField("key");
const Status keyStatus = validateKeyPattern(key);
if (!keyStatus.isOK()) {
return StatusWith<CompactStats>(
ErrorCodes::CannotCreateIndex,
str::stream() << "Cannot rebuild index " << spec << ": "
<< keyStatus.reason()
<< " For more info see"
<< " http://dochub.mongodb.org/core/index-validation");
}
indexSpecs.push_back(spec);
}
}
log() << "compact orphan deleted lists" << endl;
d->orphanDeletedList();
// Start over from scratch with our extent sizing and growth
d->setLastExtentSize( 0 );
// before dropping indexes, at least make sure we can allocate one extent!
// this will allocate an extent and add to free list
// if it cannot, it will throw an exception
increaseStorageSize( _details->lastExtentSize(), true );
// note that the drop indexes call also invalidates all clientcursors for the namespace,
// which is important and wanted here
log() << "compact dropping indexes" << endl;
Status status = _indexCatalog.dropAllIndexes( true );
if ( !status.isOK() ) {
return StatusWith<CompactStats>( status );
}
getDur().commitIfNeeded();
killCurrentOp.checkForInterrupt();
CompactStats stats;
MultiIndexBlock multiIndexBlock( this );
status = multiIndexBlock.init( indexSpecs );
if ( !status.isOK() )
return StatusWith<CompactStats>( status );
// reset data size and record counts to 0 for this namespace
// as we're about to tally them up again for each new extent
d->setStats( 0, 0 );
ProgressMeterHolder pm(cc().curop()->setMessage("compact extent",
"Extent Compacting Progress",
extents.size()));
int extentNumber = 0;
for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) {
_compactExtent(*i, extentNumber++, multiIndexBlock, compactOptions, &stats );
pm.hit();
}
verify( d->firstExtent().ext()->xprev.isNull() );
// indexes will do their own progress meter?
pm.finished();
log() << "starting index commits";
status = multiIndexBlock.commit();
if ( !status.isOK() )
return StatusWith<CompactStats>( status );
return StatusWith<CompactStats>( stats );
}
StatusWith<CompactStats> Collection::compact( const CompactOptions* compactOptions ) {
if ( isCapped() )
return StatusWith<CompactStats>( ErrorCodes::BadValue,
"cannot compact capped collection" );
if ( _indexCatalog.numIndexesInProgress() )
return StatusWith<CompactStats>( ErrorCodes::BadValue,
"cannot compact when indexes in progress" );
_database->_initForWrites();
NamespaceDetails* d = details();
// this is a big job, so might as well make things tidy before we start just to be nice.
getDur().commitIfNeeded();
list<DiskLoc> extents;
for( DiskLoc L = d->firstExtent(); !L.isNull(); L = L.ext()->xnext )
extents.push_back(L);
log() << "compact " << extents.size() << " extents" << endl;
// same data, but might perform a little different after compact?
_infoCache.reset();
vector<BSONObj> indexSpecs;
{
IndexCatalog::IndexIterator ii( _indexCatalog.getIndexIterator( false ) );
while ( ii.more() ) {
IndexDescriptor* descriptor = ii.next();
indexSpecs.push_back( _compactAdjustIndexSpec( descriptor->infoObj() ) );
}
}
log() << "compact orphan deleted lists" << endl;
d->orphanDeletedList();
// Start over from scratch with our extent sizing and growth
d->setLastExtentSize( 0 );
// before dropping indexes, at least make sure we can allocate one extent!
// this will allocate an extent and add to free list
// if it cannot, it will throw an exception
increaseStorageSize( _details->lastExtentSize(), true );
// note that the drop indexes call also invalidates all clientcursors for the namespace,
// which is important and wanted here
log() << "compact dropping indexes" << endl;
Status status = _indexCatalog.dropAllIndexes( true );
if ( !status.isOK() ) {
return StatusWith<CompactStats>( status );
}
getDur().commitIfNeeded();
CompactStats stats;
OwnedPointerVector<IndexCatalog::IndexBuildBlock> indexBuildBlocks;
vector<IndexAccessMethod*> indexesToInsertTo;
vector< std::pair<IndexAccessMethod*,IndexAccessMethod*> > bulkToCommit;
for ( size_t i = 0; i < indexSpecs.size(); i++ ) {
killCurrentOp.checkForInterrupt(false);
BSONObj info = indexSpecs[i];
info = _compactAdjustIndexSpec( info );
info = _indexCatalog.fixIndexSpec( info );
auto_ptr<IndexCatalog::IndexBuildBlock> block( new IndexCatalog::IndexBuildBlock( this,info ) );
Status status = block->init();
if ( !status.isOK() )
return StatusWith<CompactStats>(status);
IndexAccessMethod* accessMethod = block->getEntry()->accessMethod();
status = accessMethod->initializeAsEmpty();
if ( !status.isOK() )
return StatusWith<CompactStats>(status);
IndexAccessMethod* bulk = accessMethod->initiateBulk();
if ( bulk ) {
indexesToInsertTo.push_back( bulk );
bulkToCommit.push_back( std::pair<IndexAccessMethod*,IndexAccessMethod*>( accessMethod, bulk ) );
}
else {
indexesToInsertTo.push_back( accessMethod );
}
indexBuildBlocks.mutableVector().push_back( block.release() );
}
// reset data size and record counts to 0 for this namespace
// as we're about to tally them up again for each new extent
d->setStats( 0, 0 );
ProgressMeterHolder pm(cc().curop()->setMessage("compact extent",
"Extent Compacting Progress",
extents.size()));
int extentNumber = 0;
for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) {
_compactExtent(*i, extentNumber++, indexesToInsertTo, compactOptions, &stats );
pm.hit();
}
verify( d->firstExtent().ext()->xprev.isNull() );
// indexes will do their own progress meter?
pm.finished();
log() << "starting index commits";
for ( size_t i = 0; i < bulkToCommit.size(); i++ ) {
bulkToCommit[i].first->commitBulk( bulkToCommit[i].second, false, NULL );
}
for ( size_t i = 0; i < indexBuildBlocks.size(); i++ ) {
IndexCatalog::IndexBuildBlock* block = indexBuildBlocks.mutableVector()[i];
block->success();
}
return StatusWith<CompactStats>( stats );
}
