Status RocksRecordStore::validate( OperationContext* txn,
bool full,
bool scanData,
ValidateAdaptor* adaptor,
ValidateResults* results,
BSONObjBuilder* output ) const {
// TODO validate that _numRecords and _dataSize are correct in scanData mode
if ( scanData ) {
bool invalidObject = false;
boost::scoped_ptr<RecordIterator> iter( getIterator( txn ) );
while( !iter->isEOF() ) {
RecordData data = dataFor( txn, iter->curr() );
size_t dataSize;
const Status status = adaptor->validate( data, &dataSize );
if (!status.isOK()) {
results->valid = false;
if ( invalidObject ) {
results->errors.push_back("invalid object detected (see logs)");
}
invalidObject = true;
log() << "Invalid object detected in " << _ns << ": " << status.reason();
}
iter->getNext();
}
}
return Status::OK();
}
boost::optional<Record> KVRecordStore::KVRecordCursor::next() {
if (isEOF()) {
return boost::none;
}
_saveLocAndVal();
_cursor->advance(_txn);
Record r;
r.id = _savedLoc;
r.data = dataFor(_savedLoc);
return r;
}
Status KVRecordStore::validate( OperationContext* txn,
bool full,
bool scanData,
ValidateAdaptor* adaptor,
ValidateResults* results,
BSONObjBuilder* output ) {
bool invalidObject = false;
long long numRecords = 0;
long long dataSizeTotal = 0;
const bool forward = true;
for (boost::scoped_ptr<KVRecordCursor> iter( getKVCursor(txn, forward) );
!iter->isEOF(); ) {
numRecords++;
if (scanData) {
RecordData data = dataFor( txn, iter->curr() );
size_t dataSize;
if (full) {
const Status status = adaptor->validate( data, &dataSize );
if (!status.isOK()) {
results->valid = false;
if ( invalidObject ) {
results->errors.push_back("invalid object detected (see logs)");
}
invalidObject = true;
log() << "Invalid object detected in " << _ns << ": " << status.reason();
}
dataSizeTotal += static_cast<long long>(dataSize);
}
}
iter->getNext();
}
if (_sizeStorer && full && scanData && results->valid) {
if (numRecords != _numRecords.load() || dataSizeTotal != _dataSize.load()) {
warning() << ns() << ": Existing record and data size counters ("
<< _numRecords.load() << " records " << _dataSize.load() << " bytes) "
<< "are inconsistent with full validation results ("
<< numRecords << " records " << dataSizeTotal << " bytes). "
<< "Updating counters with new values.";
}
_numRecords.store(numRecords);
_dataSize.store(dataSizeTotal);
long long oldNumRecords;
long long oldDataSize;
_sizeStorer->load(_ident, &oldNumRecords, &oldDataSize);
if (numRecords != oldNumRecords || dataSizeTotal != oldDataSize) {
warning() << ns() << ": Existing data in size storer ("
<< oldNumRecords << " records " << oldDataSize << " bytes) "
<< "is inconsistent with full validation results ("
<< numRecords << " records " << dataSizeTotal << " bytes). "
<< "Updating size storer with new values.";
}
_sizeStorer->store(this, _ident, numRecords, dataSizeTotal);
}
output->appendNumber("nrecords", numRecords);
return Status::OK();
}
void SimpleRecordStoreV1::_compactExtent(OperationContext* txn,
const DiskLoc diskloc,
int extentNumber,
RecordStoreCompactAdaptor* adaptor,
const CompactOptions* compactOptions,
CompactStats* stats ) {
log() << "compact begin extent #" << extentNumber
<< " for namespace " << _ns << " " << diskloc;
unsigned oldObjSize = 0; // we'll report what the old padding was
unsigned oldObjSizeWithPadding = 0;
Extent *e = _extentManager->getExtent( diskloc );
e->assertOk();
fassert( 17437, e->validates(diskloc) );
{
// the next/prev pointers within the extent might not be in order so we first
// page the whole thing in sequentially
log() << "compact paging in len=" << e->length/1000000.0 << "MB" << endl;
Timer t;
size_t length = e->length;
touch_pages( reinterpret_cast<const char*>(e), length );
int ms = t.millis();
if( ms > 1000 )
log() << "compact end paging in " << ms << "ms "
<< e->length/1000000.0/t.seconds() << "MB/sec" << endl;
}
{
log() << "compact copying records" << endl;
long long datasize = 0;
long long nrecords = 0;
DiskLoc L = e->firstRecord;
if( !L.isNull() ) {
while( 1 ) {
Record *recOld = recordFor(L);
RecordData oldData = recOld->toRecordData();
L = getNextRecordInExtent(L);
if ( compactOptions->validateDocuments && !adaptor->isDataValid( oldData ) ) {
// object is corrupt!
log() << "compact skipping corrupt document!";
stats->corruptDocuments++;
}
else {
unsigned dataSize = adaptor->dataSize( oldData );
unsigned docSize = dataSize;
nrecords++;
oldObjSize += docSize;
oldObjSizeWithPadding += recOld->netLength();
unsigned lenWHdr = docSize + Record::HeaderSize;
unsigned lenWPadding = lenWHdr;
switch( compactOptions->paddingMode ) {
case CompactOptions::NONE:
if ( _details->isUserFlagSet(Flag_UsePowerOf2Sizes) )
lenWPadding = quantizePowerOf2AllocationSpace(lenWPadding);
break;
case CompactOptions::PRESERVE:
// if we are preserving the padding, the record should not change size
lenWPadding = recOld->lengthWithHeaders();
break;
case CompactOptions::MANUAL:
lenWPadding = compactOptions->computeRecordSize(lenWPadding);
if (lenWPadding < lenWHdr || lenWPadding > BSONObjMaxUserSize / 2 ) {
lenWPadding = lenWHdr;
}
break;
}
CompactDocWriter writer( recOld, dataSize, lenWPadding );
StatusWith<DiskLoc> status = insertRecord( txn, &writer, 0 );
uassertStatusOK( status.getStatus() );
datasize += recordFor( status.getValue() )->netLength();
adaptor->inserted( dataFor( status.getValue() ), status.getValue() );
}
if( L.isNull() ) {
// we just did the very last record from the old extent. it's still pointed to
// by the old extent ext, but that will be fixed below after this loop
break;
}
// remove the old records (orphan them) periodically so our commit block doesn't get too large
bool stopping = false;
RARELY stopping = !txn->checkForInterruptNoAssert().isOK();
if( stopping || txn->recoveryUnit()->isCommitNeeded() ) {
*txn->recoveryUnit()->writing(&e->firstRecord) = L;
Record *r = recordFor(L);
txn->recoveryUnit()->writingInt(r->prevOfs()) = DiskLoc::NullOfs;
txn->recoveryUnit()->commitIfNeeded();
txn->checkForInterrupt();
}
}
} // if !L.isNull()
invariant( _details->firstExtent() == diskloc );
invariant( _details->lastExtent() != diskloc );
DiskLoc newFirst = e->xnext;
_details->setFirstExtent( txn, newFirst );
*txn->recoveryUnit()->writing(&_extentManager->getExtent( newFirst )->xprev) = DiskLoc();
_extentManager->freeExtent( txn, diskloc );
txn->recoveryUnit()->commitIfNeeded();
{
double op = 1.0;
if( oldObjSize )
op = static_cast<double>(oldObjSizeWithPadding)/oldObjSize;
log() << "compact finished extent #" << extentNumber << " containing " << nrecords
<< " documents (" << datasize/1000000.0 << "MB)"
<< " oldPadding: " << op << ' ' << static_cast<unsigned>(op*100.0)/100;
}
}
}