void RecordStoreV1Base::deleteRecord( TransactionExperiment* txn, const DiskLoc& dl ) {
Record* todelete = recordFor( dl );
/* remove ourself from the record next/prev chain */
{
if ( todelete->prevOfs() != DiskLoc::NullOfs ) {
DiskLoc prev = getPrevRecordInExtent( dl );
Record* prevRecord = recordFor( prev );
txn->writingInt( prevRecord->nextOfs() ) = todelete->nextOfs();
}
if ( todelete->nextOfs() != DiskLoc::NullOfs ) {
DiskLoc next = getNextRecord( dl );
Record* nextRecord = recordFor( next );
txn->writingInt( nextRecord->prevOfs() ) = todelete->prevOfs();
}
}
/* remove ourself from extent pointers */
{
Extent *e = txn->writing( _getExtent( _getExtentLocForRecord( dl ) ) );
if ( e->firstRecord == dl ) {
if ( todelete->nextOfs() == DiskLoc::NullOfs )
e->firstRecord.Null();
else
e->firstRecord.set(dl.a(), todelete->nextOfs() );
}
if ( e->lastRecord == dl ) {
if ( todelete->prevOfs() == DiskLoc::NullOfs )
e->lastRecord.Null();
else
e->lastRecord.set(dl.a(), todelete->prevOfs() );
}
}
/* add to the free list */
{
_details->incrementStats( txn, -1 * todelete->netLength(), -1 );
if ( _isSystemIndexes ) {
/* temp: if in system.indexes, don't reuse, and zero out: we want to be
careful until validated more, as IndexDetails has pointers
to this disk location. so an incorrectly done remove would cause
a lot of problems.
*/
memset( txn->writingPtr(todelete, todelete->lengthWithHeaders() ),
0, todelete->lengthWithHeaders() );
}
else {
DEV {
unsigned long long *p = reinterpret_cast<unsigned long long *>( todelete->data() );
*txn->writing(p) = 0;
}
addDeletedRec(txn, dl);
}
}
}
DiskLoc RecordStoreV1Base::getNextRecord( const DiskLoc& loc ) const {
DiskLoc next = getNextRecordInExtent( loc );
if ( !next.isNull() )
return next;
// now traverse extents
Extent* e = _getExtent( _getExtentLocForRecord(loc) );
while ( 1 ) {
if ( e->xnext.isNull() )
return DiskLoc(); // end of collection
e = _getExtent( e->xnext );
if ( !e->firstRecord.isNull() )
break;
// entire extent could be empty, keep looking
}
return e->firstRecord;
}
DiskLoc RecordStoreV1Base::getPrevRecord(OperationContext* txn, const DiskLoc& loc) const {
DiskLoc prev = getPrevRecordInExtent(txn, loc);
if (!prev.isNull()) {
return prev;
}
// now traverse extents
Extent* e = _getExtent(txn, _getExtentLocForRecord(txn, loc));
while (1) {
if (e->xprev.isNull())
return DiskLoc(); // end of collection
e = _getExtent(txn, e->xprev);
if (!e->firstRecord.isNull())
break;
// entire extent could be empty, keep looking
}
return e->lastRecord;
}
//
// Capped collection traversal
//
CappedRecordStoreV1Iterator::CappedRecordStoreV1Iterator( OperationContext* txn,
const CappedRecordStoreV1* collection,
const DiskLoc& start, bool tailable,
const CollectionScanParams::Direction& dir)
: _txn(txn), _recordStore(collection), _curr(start), _tailable(tailable),
_direction(dir), _killedByInvalidate(false) {
if (_curr.isNull()) {
const RecordStoreV1MetaData* nsd = _recordStore->details();
// If a start position isn't specified, we fill one out from the start of the
// collection.
if (CollectionScanParams::FORWARD == _direction) {
// Going forwards.
if (!nsd->capLooped()) {
// If our capped collection doesn't loop around, the first record is easy.
_curr = collection->firstRecord(_txn);
}
else {
// Our capped collection has "looped' around.
// Copied verbatim from ForwardCappedCursor::init.
// TODO ELABORATE
_curr = _getExtent( nsd->capExtent() )->firstRecord;
if (!_curr.isNull() && _curr == nsd->capFirstNewRecord()) {
_curr = _getExtent( nsd->capExtent() )->lastRecord;
_curr = nextLoop(_curr);
}
}
}
else {
// Going backwards
if (!nsd->capLooped()) {
// Start at the end.
_curr = collection->lastRecord(_txn);
}
else {
_curr = _getExtent( nsd->capExtent() )->lastRecord;
}
}
}
}
Status RecordStoreV1Base::touch( OperationContext* txn, BSONObjBuilder* output ) const {
Timer t;
// Note: when this class has document level locking, we'll need a lock to get extents
// and then ideally only hold the collection lock from above while doing actual touching.
std::vector<touch_location> ranges;
{
Extent* ext = _getExtent( _details->firstExtent() );
while ( ext ) {
touch_location tl;
tl.root = reinterpret_cast<const char*>(ext);
tl.length = ext->length;
ranges.push_back(tl);
if ( ext->xnext.isNull() )
ext = NULL;
else
ext = _getExtent( ext->xnext );
}
}
/* TODO(ERH)
std::string progress_msg = "touch " + ns + " extents";
ProgressMeterHolder pm(cc().curop()->setMessage(progress_msg.c_str(),
"Touch Progress",
ranges.size()));
*/
for ( std::vector<touch_location>::iterator it = ranges.begin(); it != ranges.end(); ++it ) {
touch_pages( it->root, it->length );
//pm.hit();
txn->checkForInterrupt();
}
//pm.finished();
if ( output ) {
output->append( "numRanges", static_cast<int>( ranges.size() ) );
output->append( "millis", t.millis() );
}
return Status::OK();
}
Status RecordStoreV1Base::touch(OperationContext* txn, BSONObjBuilder* output) const {
Timer t;
std::vector<touch_location> ranges;
{
DiskLoc nextLoc = _details->firstExtent(txn);
Extent* ext = nextLoc.isNull() ? NULL : _getExtent(txn, nextLoc);
while (ext) {
touch_location tl;
tl.root = reinterpret_cast<const char*>(ext);
tl.length = ext->length;
ranges.push_back(tl);
nextLoc = ext->xnext;
if (nextLoc.isNull())
ext = NULL;
else
ext = _getExtent(txn, nextLoc);
}
}
std::string progress_msg = "touch " + std::string(txn->getNS()) + " extents";
stdx::unique_lock<Client> lk(*txn->getClient());
ProgressMeterHolder pm(
*txn->setMessage_inlock(progress_msg.c_str(), "Touch Progress", ranges.size()));
lk.unlock();
for (std::vector<touch_location>::iterator it = ranges.begin(); it != ranges.end(); ++it) {
touch_pages(it->root, it->length);
pm.hit();
txn->checkForInterrupt();
}
pm.finished();
if (output) {
output->append("numRanges", static_cast<int>(ranges.size()));
output->append("millis", t.millis());
}
return Status::OK();
}
vector<RecordIterator*> SimpleRecordStoreV1::getManyIterators( OperationContext* txn ) const {
OwnedPointerVector<RecordIterator> iterators;
const Extent* ext;
for (DiskLoc extLoc = details()->firstExtent(txn); !extLoc.isNull(); extLoc = ext->xnext) {
ext = _getExtent(txn, extLoc);
if (ext->firstRecord.isNull())
continue;
iterators.push_back(
new RecordStoreV1Base::IntraExtentIterator(txn, ext->firstRecord, this));
}
return iterators.release();
}
void RecordStoreV1Base::_addRecordToRecListInExtent(OperationContext* txn,
Record *r,
DiskLoc loc) {
dassert( recordFor(loc) == r );
Extent *e = _getExtent( _getExtentLocForRecord( loc ) );
if ( e->lastRecord.isNull() ) {
*txn->recoveryUnit()->writing(&e->firstRecord) = loc;
*txn->recoveryUnit()->writing(&e->lastRecord) = loc;
r->prevOfs() = r->nextOfs() = DiskLoc::NullOfs;
}
else {
Record *oldlast = recordFor(e->lastRecord);
r->prevOfs() = e->lastRecord.getOfs();
r->nextOfs() = DiskLoc::NullOfs;
txn->recoveryUnit()->writingInt(oldlast->nextOfs()) = loc.getOfs();
*txn->recoveryUnit()->writing(&e->lastRecord) = loc;
}
}
DiskLoc CappedRecordStoreV1Iterator::getNextCapped(const DiskLoc& dl) {
invariant(!dl.isNull());
const RecordStoreV1MetaData* details = _recordStore->details();
if (CollectionScanParams::FORWARD == _direction) {
// If it's not looped, it's easy.
if (!_recordStore->details()->capLooped()) {
return _getNextRecord( dl );
}
// TODO ELABORATE
// EOF.
if (dl == _getExtent( details->capExtent() )->lastRecord) {
return DiskLoc();
}
DiskLoc ret = nextLoop(dl);
// If we become capFirstNewRecord from same extent, advance to next extent.
if (ret == details->capFirstNewRecord() && ret != _getExtent( details->capExtent() )->firstRecord) {
ret = nextLoop(_getExtent( details->capExtent() )->lastRecord);
}
// If we have just gotten to beginning of capExtent, skip to capFirstNewRecord
if (ret == _getExtent( details->capExtent() )->firstRecord) { ret = details->capFirstNewRecord(); }
return ret;
}
else {
if (!details->capLooped()) { return _getPrevRecord( dl ); }
// TODO ELABORATE
// Last record
if (details->capFirstNewRecord() == _getExtent( details->capExtent() )->firstRecord) {
if (dl == nextLoop(_getExtent( details->capExtent() )->lastRecord)) {
return DiskLoc();
}
}
else {
if (dl == _getExtent( details->capExtent() )->firstRecord) { return DiskLoc(); }
}
DiskLoc ret;
// If we are capFirstNewRecord, advance to prev extent, otherwise just get prev.
if (dl == details->capFirstNewRecord()) {
ret = prevLoop(_getExtent( details->capExtent() )->firstRecord);
}
else {
ret = prevLoop(dl);
}
// If we just became last in cap extent, advance past capFirstNewRecord
// (We know ext(capExtent)->firstRecord != capFirstNewRecord, since would
// have returned DiskLoc() earlier otherwise.)
if (ret == _getExtent( details->capExtent() )->lastRecord) {
ret = _getPrevRecord( details->capFirstNewRecord() );
}
return ret;
}
}
Box2d MgBaseShape::getExtent() const { return _getExtent(); }
Status RecordStoreV1Base::validate( OperationContext* txn,
bool full, bool scanData,
ValidateAdaptor* adaptor,
ValidateResults* results, BSONObjBuilder* output ) const {
// 1) basic status that require no iteration
// 2) extent level info
// 3) check extent start and end
// 4) check each non-deleted record
// 5) check deleted list
// -------------
// 1111111111111111111
if ( isCapped() ){
output->appendBool("capped", true);
output->appendNumber("max", _details->maxCappedDocs());
}
output->appendNumber("datasize", _details->dataSize());
output->appendNumber("nrecords", _details->numRecords());
output->appendNumber("lastExtentSize", _details->lastExtentSize(txn));
output->appendNumber("padding", _details->paddingFactor());
if ( _details->firstExtent(txn).isNull() )
output->append( "firstExtent", "null" );
else
output->append( "firstExtent",
str::stream() << _details->firstExtent(txn).toString()
<< " ns:"
<< _getExtent( txn, _details->firstExtent(txn) )->nsDiagnostic.toString());
if ( _details->lastExtent(txn).isNull() )
output->append( "lastExtent", "null" );
else
output->append( "lastExtent", str::stream() << _details->lastExtent(txn).toString()
<< " ns:"
<< _getExtent( txn, _details->lastExtent(txn) )->nsDiagnostic.toString());
// 22222222222222222222222222
{ // validate extent basics
BSONArrayBuilder extentData;
int extentCount = 0;
DiskLoc extentDiskLoc;
try {
if ( !_details->firstExtent(txn).isNull() ) {
_getExtent( txn, _details->firstExtent(txn) )->assertOk();
_getExtent( txn, _details->lastExtent(txn) )->assertOk();
}
extentDiskLoc = _details->firstExtent(txn);
while (!extentDiskLoc.isNull()) {
Extent* thisExtent = _getExtent( txn, extentDiskLoc );
if (full) {
extentData << thisExtent->dump();
}
if (!thisExtent->validates(extentDiskLoc, &results->errors)) {
results->valid = false;
}
DiskLoc nextDiskLoc = thisExtent->xnext;
if (extentCount > 0 && !nextDiskLoc.isNull()
&& _getExtent( txn, nextDiskLoc )->xprev != extentDiskLoc) {
StringBuilder sb;
sb << "'xprev' pointer " << _getExtent( txn, nextDiskLoc )->xprev.toString()
<< " in extent " << nextDiskLoc.toString()
<< " does not point to extent " << extentDiskLoc.toString();
results->errors.push_back( sb.str() );
results->valid = false;
}
if (nextDiskLoc.isNull() && extentDiskLoc != _details->lastExtent(txn)) {
StringBuilder sb;
sb << "'lastExtent' pointer " << _details->lastExtent(txn).toString()
<< " does not point to last extent in list " << extentDiskLoc.toString();
results->errors.push_back( sb.str() );
results->valid = false;
}
extentDiskLoc = nextDiskLoc;
extentCount++;
txn->checkForInterrupt();
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception validating extent " << extentCount
<< ": " << e.what();
results->errors.push_back( sb.str() );
results->valid = false;
return Status::OK();
}
output->append("extentCount", extentCount);
if ( full )
output->appendArray( "extents" , extentData.arr() );
}
try {
// 333333333333333333333333333
bool testingLastExtent = false;
try {
DiskLoc firstExtentLoc = _details->firstExtent(txn);
if (firstExtentLoc.isNull()) {
// this is ok
}
else {
output->append("firstExtentDetails", _getExtent(txn, firstExtentLoc)->dump());
if (!_getExtent(txn, firstExtentLoc)->xprev.isNull()) {
StringBuilder sb;
sb << "'xprev' pointer in 'firstExtent' " << _details->firstExtent(txn).toString()
<< " is " << _getExtent(txn, firstExtentLoc)->xprev.toString()
<< ", should be null";
results->errors.push_back( sb.str() );
results->valid = false;
}
}
testingLastExtent = true;
DiskLoc lastExtentLoc = _details->lastExtent(txn);
if (lastExtentLoc.isNull()) {
// this is ok
}
else {
if (firstExtentLoc != lastExtentLoc) {
output->append("lastExtentDetails", _getExtent(txn, lastExtentLoc)->dump());
if (!_getExtent(txn, lastExtentLoc)->xnext.isNull()) {
StringBuilder sb;
sb << "'xnext' pointer in 'lastExtent' " << lastExtentLoc.toString()
<< " is " << _getExtent(txn, lastExtentLoc)->xnext.toString()
<< ", should be null";
results->errors.push_back( sb.str() );
results->valid = false;
}
}
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception processing '"
<< (testingLastExtent ? "lastExtent" : "firstExtent")
<< "': " << e.what();
results->errors.push_back( sb.str() );
results->valid = false;
}
// 4444444444444444444444444
set<DiskLoc> recs;
if( scanData ) {
int n = 0;
int nInvalid = 0;
long long nQuantizedSize = 0;
long long nPowerOf2QuantizedSize = 0;
long long len = 0;
long long nlen = 0;
long long bsonLen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
scoped_ptr<RecordIterator> iterator( getIterator( txn,
DiskLoc(),
false,
CollectionScanParams::FORWARD ) );
DiskLoc cl;
while ( !( cl = iterator->getNext() ).isNull() ) {
n++;
if ( n < 1000000 )
recs.insert(cl);
if ( isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = recordFor(cl);
len += r->lengthWithHeaders();
nlen += r->netLength();
if ( r->lengthWithHeaders() ==
quantizeAllocationSpace( r->lengthWithHeaders() ) ) {
// Count the number of records having a size consistent with
// the quantizeAllocationSpace quantization implementation.
++nQuantizedSize;
}
if ( r->lengthWithHeaders() ==
quantizePowerOf2AllocationSpace( r->lengthWithHeaders() ) ) {
// Count the number of records having a size consistent with the
// quantizePowerOf2AllocationSpace quantization implementation.
++nPowerOf2QuantizedSize;
}
if (full){
size_t dataSize = 0;
const Status status = adaptor->validate( r->toRecordData(), &dataSize );
if (!status.isOK()) {
results->valid = false;
if (nInvalid == 0) // only log once;
results->errors.push_back( "invalid object detected (see logs)" );
nInvalid++;
log() << "Invalid object detected in " << _ns
<< ": " << status.reason();
}
else {
bsonLen += dataSize;
}
}
}
if ( isCapped() && !_details->capLooped() ) {
output->append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
results->valid = false;
results->errors.push_back( "too many out of order records" );
}
}
output->append("objectsFound", n);
if (full) {
output->append("invalidObjects", nInvalid);
}
output->appendNumber("nQuantizedSize", nQuantizedSize);
output->appendNumber("nPowerOf2QuantizedSize", nPowerOf2QuantizedSize);
output->appendNumber("bytesWithHeaders", len);
output->appendNumber("bytesWithoutHeaders", nlen);
if (full) {
output->appendNumber("bytesBson", bsonLen);
}
} // end scanData
// 55555555555555555555555555
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << _details->deletedListEntry(i).isNull();
}
int ndel = 0;
long long delSize = 0;
BSONArrayBuilder delBucketSizes;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = _details->deletedListEntry(i);
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
string err( str::stream() << "bad pointer in deleted record list: "
<< loc.toString()
<< " bucket: " << i
<< " k: " << k );
results->errors.push_back( err );
results->valid = false;
break;
}
const DeletedRecord* d = deletedRecordFor(loc);
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
txn->checkForInterrupt();
}
delBucketSizes << k;
}
catch (...) {
results->errors.push_back( (string)"exception in deleted chain for bucket " +
BSONObjBuilder::numStr(i) );
results->valid = false;
}
}
output->appendNumber("deletedCount", ndel);
output->appendNumber("deletedSize", delSize);
if ( full ) {
output->append( "delBucketSizes", delBucketSizes.arr() );
}
if ( incorrect ) {
results->errors.push_back( BSONObjBuilder::numStr(incorrect) +
" records from datafile are in deleted list" );
results->valid = false;
}
}
catch (AssertionException) {
results->errors.push_back( "exception during validate" );
results->valid = false;
}
return Status::OK();
}
void RecordStoreV1Base::deleteRecord( OperationContext* txn, const DiskLoc& dl ) {
Record* todelete = recordFor( dl );
invariant( todelete->netLength() >= 4 ); // this is required for defensive code
/* remove ourself from the record next/prev chain */
{
if ( todelete->prevOfs() != DiskLoc::NullOfs ) {
DiskLoc prev = getPrevRecordInExtent( txn, dl );
Record* prevRecord = recordFor( prev );
txn->recoveryUnit()->writingInt( prevRecord->nextOfs() ) = todelete->nextOfs();
}
if ( todelete->nextOfs() != DiskLoc::NullOfs ) {
DiskLoc next = getNextRecord( txn, dl );
Record* nextRecord = recordFor( next );
txn->recoveryUnit()->writingInt( nextRecord->prevOfs() ) = todelete->prevOfs();
}
}
/* remove ourself from extent pointers */
{
DiskLoc extentLoc = todelete->myExtentLoc(dl);
Extent *e = _getExtent( txn, extentLoc );
if ( e->firstRecord == dl ) {
txn->recoveryUnit()->writing(&e->firstRecord);
if ( todelete->nextOfs() == DiskLoc::NullOfs )
e->firstRecord.Null();
else
e->firstRecord.set(dl.a(), todelete->nextOfs() );
}
if ( e->lastRecord == dl ) {
txn->recoveryUnit()->writing(&e->lastRecord);
if ( todelete->prevOfs() == DiskLoc::NullOfs )
e->lastRecord.Null();
else
e->lastRecord.set(dl.a(), todelete->prevOfs() );
}
}
/* add to the free list */
{
_details->incrementStats( txn, -1 * todelete->netLength(), -1 );
if ( _isSystemIndexes ) {
/* temp: if in system.indexes, don't reuse, and zero out: we want to be
careful until validated more, as IndexDetails has pointers
to this disk location. so an incorrectly done remove would cause
a lot of problems.
*/
memset( txn->recoveryUnit()->writingPtr(todelete, todelete->lengthWithHeaders() ),
0, todelete->lengthWithHeaders() );
}
else {
// this is defensive so we can detect if we are still using a location
// that was deleted
memset(txn->recoveryUnit()->writingPtr(todelete->data(), 4), 0xee, 4);
addDeletedRec(txn, dl);
}
}
}
