void NamespaceDetails::cappedTruncateLastDelUpdate() {
if ( _capExtent == _firstExtent ) {
// Only one extent of the collection is in use, so there
// is no deleted record in a previous extent, so nullify
// cappedLastDelRecLastExtent().
cappedLastDelRecLastExtent().writing() = DiskLoc();
}
else {
// Scan through all deleted records in the collection
// until the last deleted record for the extent prior
// to the new capExtent is found. Then set
// cappedLastDelRecLastExtent() to that deleted record.
DiskLoc i = cappedListOfAllDeletedRecords();
for( ;
!i.drec()->nextDeleted().isNull() &&
!inCapExtent( i.drec()->nextDeleted() );
i = i.drec()->nextDeleted() );
// In our capped storage model, every extent must have at least one
// deleted record. Here we check that 'i' is not the last deleted
// record. (We expect that there will be deleted records in the new
// capExtent as well.)
verify( !i.drec()->nextDeleted().isNull() );
cappedLastDelRecLastExtent().writing() = i;
}
}
DiskLoc NamespaceDetails::__capAlloc( int len ) {
DiskLoc prev = cappedLastDelRecLastExtent();
DiskLoc i = cappedFirstDeletedInCurExtent();
DiskLoc ret;
for (; !i.isNull() && inCapExtent( i ); prev = i, i = i.drec()->nextDeleted() ) {
// We need to keep at least one DR per extent in cappedListOfAllDeletedRecords(),
// so make sure there's space to create a DR at the end.
if ( i.drec()->lengthWithHeaders() >= len + 24 ) {
ret = i;
break;
}
}
/* unlink ourself from the deleted list */
if ( !ret.isNull() ) {
if ( prev.isNull() )
cappedListOfAllDeletedRecords().writing() = ret.drec()->nextDeleted();
else
prev.drec()->nextDeleted().writing() = ret.drec()->nextDeleted();
ret.drec()->nextDeleted().writing().setInvalid(); // defensive.
verify( ret.drec()->extentOfs() < ret.getOfs() );
}
return ret;
}
void NamespaceDetails::advanceCapExtent( const char *ns ) {
// We want cappedLastDelRecLastExtent() to be the last DeletedRecord of the prev cap extent
// (or DiskLoc() if new capExtent == firstExtent)
if ( capExtent == lastExtent )
getDur().writingDiskLoc( cappedLastDelRecLastExtent() ) = DiskLoc();
else {
DiskLoc i = cappedFirstDeletedInCurExtent();
for (; !i.isNull() && nextIsInCapExtent( i ); i = i.drec()->nextDeleted );
getDur().writingDiskLoc( cappedLastDelRecLastExtent() ) = i;
}
getDur().writingDiskLoc( capExtent ) = theCapExtent()->xnext.isNull() ? firstExtent : theCapExtent()->xnext;
/* this isn't true if a collection has been renamed...that is ok just used for diagnostics */
//dassert( theCapExtent()->ns == ns );
theCapExtent()->assertOk();
getDur().writingDiskLoc( capFirstNewRecord ) = DiskLoc();
}
void NamespaceDetails::emptyCappedCollection( const char *ns ) {
DEV assert( this == nsdetails(ns) );
massert( 13424, "collection must be capped", capped );
massert( 13425, "background index build in progress", !backgroundIndexBuildInProgress );
massert( 13426, "indexes present", nIndexes == 0 );
ClientCursor::invalidate( ns );
NamespaceDetailsTransient::clearForPrefix( ns );
cappedLastDelRecLastExtent() = DiskLoc();
cappedListOfAllDeletedRecords() = DiskLoc();
// preserve firstExtent/lastExtent
capExtent = firstExtent;
stats.datasize = stats.nrecords = 0;
// lastExtentSize preserve
// nIndexes preserve 0
// capped preserve true
// max preserve
paddingFactor = 1.0;
flags = 0;
capFirstNewRecord = DiskLoc();
capFirstNewRecord.setInvalid();
cappedLastDelRecLastExtent().setInvalid();
// dataFileVersion preserve
// indexFileVersion preserve
multiKeyIndexBits = 0;
reservedA = 0;
extraOffset = 0;
// backgroundIndexBuildInProgress preserve 0
memset(reserved, 0, sizeof(reserved));
for( DiskLoc ext = firstExtent; !ext.isNull(); ext = ext.ext()->xnext ) {
DiskLoc prev = ext.ext()->xprev;
DiskLoc next = ext.ext()->xnext;
DiskLoc empty = ext.ext()->reuse( ns );
ext.ext()->xprev = prev;
ext.ext()->xnext = next;
addDeletedRec( empty.drec(), empty );
}
}
void NamespaceDetails::cappedTruncateAfter(const char *ns, DiskLoc end, bool inclusive) {
DEV verify( this == nsdetails(ns) );
verify( cappedLastDelRecLastExtent().isValid() );
// We iteratively remove the newest document until the newest document
// is 'end', then we remove 'end' if requested.
bool foundLast = false;
while( 1 ) {
if ( foundLast ) {
// 'end' has been found and removed, so break.
break;
}
getDur().commitIfNeeded();
// 'curr' will point to the newest document in the collection.
DiskLoc curr = theCapExtent()->lastRecord;
verify( !curr.isNull() );
if ( curr == end ) {
if ( inclusive ) {
// 'end' has been found, so break next iteration.
foundLast = true;
}
else {
// 'end' has been found, so break.
break;
}
}
// TODO The algorithm used in this function cannot generate an
// empty collection, but we could call emptyCappedCollection() in
// this case instead of asserting.
uassert( 13415, "emptying the collection is not allowed", _stats.nrecords > 1 );
// Delete the newest record, and coalesce the new deleted
// record with existing deleted records.
theDataFileMgr.deleteRecord(this, ns, curr.rec(), curr, true);
compact();
// This is the case where we have not yet had to remove any
// documents to make room for other documents, and we are allocating
// documents from free space in fresh extents instead of reusing
// space from familiar extents.
if ( !capLooped() ) {
// We just removed the last record from the 'capExtent', and
// the 'capExtent' can't be empty, so we set 'capExtent' to
// capExtent's prev extent.
if ( theCapExtent()->lastRecord.isNull() ) {
verify( !theCapExtent()->xprev.isNull() );
// NOTE Because we didn't delete the last document, and
// capLooped() is false, capExtent is not the first extent
// so xprev will be nonnull.
_capExtent.writing() = theCapExtent()->xprev;
theCapExtent()->assertOk();
// update cappedLastDelRecLastExtent()
cappedTruncateLastDelUpdate();
}
continue;
}
// This is the case where capLooped() is true, and we just deleted
// from capExtent, and we just deleted capFirstNewRecord, which was
// the last record on the fresh side of capExtent.
// NOTE In this comparison, curr and potentially capFirstNewRecord
// may point to invalid data, but we can still compare the
// references themselves.
if ( curr == _capFirstNewRecord ) {
// Set 'capExtent' to the first nonempty extent prior to the
// initial capExtent. There must be such an extent because we
// have not deleted the last document in the collection. It is
// possible that all extents other than the capExtent are empty.
// In this case we will keep the initial capExtent and specify
// that all records contained within are on the fresh rather than
// stale side of the extent.
DiskLoc newCapExtent = _capExtent;
do {
// Find the previous extent, looping if necessary.
newCapExtent = ( newCapExtent == _firstExtent ) ? _lastExtent : newCapExtent.ext()->xprev;
newCapExtent.ext()->assertOk();
}
while ( newCapExtent.ext()->firstRecord.isNull() );
_capExtent.writing() = newCapExtent;
// Place all documents in the new capExtent on the fresh side
// of the capExtent by setting capFirstNewRecord to the first
// document in the new capExtent.
_capFirstNewRecord.writing() = theCapExtent()->firstRecord;
// update cappedLastDelRecLastExtent()
cappedTruncateLastDelUpdate();
}
}
}
DiskLoc NamespaceDetails::cappedAlloc(const char *ns, int len) {
if ( len > theCapExtent()->length ) {
// the extent check is a way to try and improve performance
uassert( 16328 , str::stream() << "document is larger than capped size "
<< len << " > " << storageSize() , len <= storageSize() );
}
// signal done allocating new extents.
if ( !cappedLastDelRecLastExtent().isValid() )
getDur().writingDiskLoc( cappedLastDelRecLastExtent() ) = DiskLoc();
verify( len < 400000000 );
int passes = 0;
int maxPasses = ( len / 30 ) + 2; // 30 is about the smallest entry that could go in the oplog
if ( maxPasses < 5000 ) {
// this is for bacwards safety since 5000 was the old value
maxPasses = 5000;
}
DiskLoc loc;
// delete records until we have room and the max # objects limit achieved.
/* this fails on a rename -- that is ok but must keep commented out */
//verify( theCapExtent()->ns == ns );
theCapExtent()->assertOk();
DiskLoc firstEmptyExtent;
while ( 1 ) {
if ( _stats.nrecords < maxCappedDocs() ) {
loc = __capAlloc( len );
if ( !loc.isNull() )
break;
}
// If on first iteration through extents, don't delete anything.
if ( !_capFirstNewRecord.isValid() ) {
advanceCapExtent( ns );
if ( _capExtent != _firstExtent )
_capFirstNewRecord.writing().setInvalid();
// else signal done with first iteration through extents.
continue;
}
if ( !_capFirstNewRecord.isNull() &&
theCapExtent()->firstRecord == _capFirstNewRecord ) {
// We've deleted all records that were allocated on the previous
// iteration through this extent.
advanceCapExtent( ns );
continue;
}
if ( theCapExtent()->firstRecord.isNull() ) {
if ( firstEmptyExtent.isNull() )
firstEmptyExtent = _capExtent;
advanceCapExtent( ns );
if ( firstEmptyExtent == _capExtent ) {
maybeComplain( ns, len );
return DiskLoc();
}
continue;
}
DiskLoc fr = theCapExtent()->firstRecord;
theDataFileMgr.deleteRecord(this, ns, fr.rec(), fr, true); // ZZZZZZZZZZZZ
compact();
if( ++passes > maxPasses ) {
StringBuilder sb;
sb << "passes >= maxPasses in NamespaceDetails::cappedAlloc: ns: " << ns
<< ", len: " << len
<< ", maxPasses: " << maxPasses
<< ", _maxDocsInCapped: " << _maxDocsInCapped
<< ", nrecords: " << _stats.nrecords
<< ", datasize: " << _stats.datasize;
msgasserted(10345, sb.str());
}
}
// Remember first record allocated on this iteration through capExtent.
if ( _capFirstNewRecord.isValid() && _capFirstNewRecord.isNull() )
getDur().writingDiskLoc(_capFirstNewRecord) = loc;
return loc;
}
DiskLoc &NamespaceDetails::cappedFirstDeletedInCurExtent() {
if ( cappedLastDelRecLastExtent().isNull() )
return cappedListOfAllDeletedRecords();
else
return cappedLastDelRecLastExtent().drec()->nextDeleted();
}
DiskLoc NamespaceDetails::cappedAlloc(const char *ns, int len) {
// signal done allocating new extents.
if ( !cappedLastDelRecLastExtent().isValid() )
getDur().writingDiskLoc( cappedLastDelRecLastExtent() ) = DiskLoc();
verify( len < 400000000 );
int passes = 0;
int maxPasses = ( len / 30 ) + 2; // 30 is about the smallest entry that could go in the oplog
if ( maxPasses < 5000 ) {
// this is for bacwards safety since 5000 was the old value
maxPasses = 5000;
}
DiskLoc loc;
// delete records until we have room and the max # objects limit achieved.
/* this fails on a rename -- that is ok but must keep commented out */
//verify( theCapExtent()->ns == ns );
theCapExtent()->assertOk();
DiskLoc firstEmptyExtent;
while ( 1 ) {
if ( stats.nrecords < max ) {
loc = __capAlloc( len );
if ( !loc.isNull() )
break;
}
// If on first iteration through extents, don't delete anything.
if ( !capFirstNewRecord.isValid() ) {
advanceCapExtent( ns );
if ( capExtent != firstExtent )
capFirstNewRecord.writing().setInvalid();
// else signal done with first iteration through extents.
continue;
}
if ( !capFirstNewRecord.isNull() &&
theCapExtent()->firstRecord == capFirstNewRecord ) {
// We've deleted all records that were allocated on the previous
// iteration through this extent.
advanceCapExtent( ns );
continue;
}
if ( theCapExtent()->firstRecord.isNull() ) {
if ( firstEmptyExtent.isNull() )
firstEmptyExtent = capExtent;
advanceCapExtent( ns );
if ( firstEmptyExtent == capExtent ) {
maybeComplain( ns, len );
return DiskLoc();
}
continue;
}
DiskLoc fr = theCapExtent()->firstRecord;
theDataFileMgr.deleteRecord(ns, fr.rec(), fr, true); // ZZZZZZZZZZZZ
compact();
if( ++passes > maxPasses ) {
log() << "passes ns:" << ns << " len:" << len << " maxPasses: " << maxPasses << '\n';
log() << "passes max:" << max << " nrecords:" << stats.nrecords << " datasize: " << stats.datasize << endl;
massert( 10345 , "passes >= maxPasses in capped collection alloc", false );
}
}
// Remember first record allocated on this iteration through capExtent.
if ( capFirstNewRecord.isValid() && capFirstNewRecord.isNull() )
getDur().writingDiskLoc(capFirstNewRecord) = loc;
return loc;
}
/* everything from end on, eliminate from the capped collection.
@param inclusive if true, deletes end (i.e. closed or open range)
*/
void NamespaceDetails::cappedTruncateAfter(const char *ns, DiskLoc end, bool inclusive) {
DEV assert( this == nsdetails(ns) );
assert( cappedLastDelRecLastExtent().isValid() );
bool foundLast = false;
while( 1 ) {
if ( foundLast ) {
break;
}
DiskLoc curr = theCapExtent()->lastRecord;
assert( !curr.isNull() );
if ( curr == end ) {
if ( inclusive ) {
foundLast = true;
} else {
break;
}
}
uassert( 13415, "emptying the collection is not allowed", stats.nrecords > 1 );
if ( !capLooped() ) {
theDataFileMgr.deleteRecord(ns, curr.rec(), curr, true);
compact();
if ( theCapExtent()->lastRecord.isNull() ) {
assert( !theCapExtent()->xprev.isNull() );
capExtent = theCapExtent()->xprev;
theCapExtent()->assertOk();
if ( capExtent == firstExtent ) {
cappedLastDelRecLastExtent() = DiskLoc();
} else {
// slow - there's no prev ptr for deleted rec
DiskLoc i = cappedListOfAllDeletedRecords();
for( ;
!i.drec()->nextDeleted.isNull() &&
!inCapExtent( i.drec()->nextDeleted );
i = i.drec()->nextDeleted );
assert( !i.drec()->nextDeleted.isNull() ); // I believe there is always at least one drec per extent
cappedLastDelRecLastExtent() = i;
}
}
continue;
}
theDataFileMgr.deleteRecord(ns, curr.rec(), curr, true);
compact();
if ( curr == capFirstNewRecord ) { // invalid, but can compare locations
capExtent = ( capExtent == firstExtent ) ? lastExtent : theCapExtent()->xprev;
theCapExtent()->assertOk();
assert( !theCapExtent()->firstRecord.isNull() );
capFirstNewRecord = theCapExtent()->firstRecord;
if ( capExtent == firstExtent ) {
cappedLastDelRecLastExtent() = DiskLoc();
} else {
// slow - there's no prev ptr for deleted rec
DiskLoc i = cappedListOfAllDeletedRecords();
for( ;
!i.drec()->nextDeleted.isNull() &&
!inCapExtent( i.drec()->nextDeleted );
i = i.drec()->nextDeleted );
assert( !i.drec()->nextDeleted.isNull() ); // I believe there is always at least one drec per extent
cappedLastDelRecLastExtent() = i;
}
}
}
}