DiskLoc RocksRecordStore::Iterator::getNext() {
if ( !_iterator->Valid() ) {
return DiskLoc();
}
DiskLoc toReturn = curr();
if ( _forward() )
_iterator->Next();
else
_iterator->Prev();
return toReturn;
}
/* fetch a single object from collection ns that matches query
set your db SavedContext first
*/
DiskLoc Helpers::findOne(const StringData& ns, const BSONObj &query, bool requireIndex) {
shared_ptr<Cursor> c =
NamespaceDetailsTransient::getCursor( ns.data() , query, BSONObj(),
requireIndex ?
QueryPlanSelectionPolicy::indexOnly() :
QueryPlanSelectionPolicy::any() );
while( c->ok() ) {
if ( c->currentMatches() && !c->getsetdup( c->currLoc() ) ) {
return c->currLoc();
}
c->advance();
}
return DiskLoc();
}
/* fetch a single object from collection ns that matches query
set your db SavedContext first
*/
DiskLoc Helpers::findOne(Collection* collection, const BSONObj &query, bool requireIndex) {
if ( !collection )
return DiskLoc();
CanonicalQuery* cq;
const WhereCallbackReal whereCallback(collection->ns().db());
massert(17244, "Could not canonicalize " + query.toString(),
CanonicalQuery::canonicalize(collection->ns(), query, &cq, whereCallback).isOK());
Runner* rawRunner;
size_t options = requireIndex ? QueryPlannerParams::NO_TABLE_SCAN : QueryPlannerParams::DEFAULT;
massert(17245, "Could not get runner for query " + query.toString(),
getRunner(collection, cq, &rawRunner, options).isOK());
auto_ptr<Runner> runner(rawRunner);
Runner::RunnerState state;
DiskLoc loc;
if (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &loc))) {
return loc;
}
return DiskLoc();
}
int64_t Collection::countTableScan( const MatchExpression* expression ) {
scoped_ptr<RecordIterator> iterator( getIterator( DiskLoc(),
false,
CollectionScanParams::FORWARD ) );
int64_t count = 0;
while ( !iterator->isEOF() ) {
DiskLoc loc = iterator->getNext();
BSONObj obj = docFor( loc );
if ( expression->matchesBSON( obj ) )
count++;
}
return count;
}
DummyRecordStoreV1MetaData::DummyRecordStoreV1MetaData( bool capped, int userFlags ) {
_dataSize = 0;
_numRecords = 0;
_capped = capped;
_userFlags = userFlags;
_lastExtentSize = 0;
_paddingFactor = 1;
_maxCappedDocs = numeric_limits<long long>::max();
_capFirstNewRecord.setInvalid();
if ( _capped ) {
// copied from NamespaceDetails::NamespaceDetails()
setDeletedListEntry( NULL, 1, DiskLoc().setInvalid() );
}
}
void CappedIterator::invalidate(const DiskLoc& dl) {
if ((_tailable && _curr.isNull() && dl == _prev) || (dl == _curr)) {
// In the _tailable case, we're about to kill the DiskLoc that we're tailing. Nothing
// that we can possibly do to survive that.
//
// In the _curr case, we *could* move to the next thing, since there is actually a next
// thing, but according to clientcursor.cpp:
// "note we cannot advance here. if this condition occurs, writes to the oplog
// have "caught" the reader. skipping ahead, the reader would miss postentially
// important data."
_curr = _prev = DiskLoc();
_killedByInvalidate = true;
}
}
virtual void keyAt(DiskLoc thisLoc, int pos, BSONObj& key, DiskLoc& recordLoc) {
recordLoc = DiskLoc();
const BtreeBucket<V>* bucket = thisLoc.btree<V>();
int n = bucket->nKeys();
if( pos < 0 || pos >= n || n == 0xffff /* bucket deleted */ || ! bucket->isUsed( pos ) ){
// log() << "Pos: " << pos << " n " << n << endl;
return;
}
typename BtreeBucket<V>::KeyNode kn = bucket->keyNode(pos);
key = kn.key.toBson();
recordLoc = kn.recordLoc;
}
virtual RecordStore* newCappedRecordStore( int64_t cappedMaxSize,
int64_t cappedMaxDocs ) {
OperationContextNoop txn;
DummyRecordStoreV1MetaData* md = new DummyRecordStoreV1MetaData( true, 0 );
CappedRecordStoreV1* rs = new CappedRecordStoreV1( &txn,
NULL,
"a.b",
md,
&_em,
false );
LocAndSize records[] = {
{}
};
LocAndSize drecs[] = {
{DiskLoc(0, 1000), 1000},
{}
};
md->setCapExtent(&txn, DiskLoc(0, 0));
md->setCapFirstNewRecord(&txn, DiskLoc().setInvalid());
initializeV1RS(&txn, records, drecs, NULL, &_em, md);
return rs;
}
BSONObjExternalSorter::Iterator::Iterator( BSONObjExternalSorter * sorter ) :
_cmp( sorter->_order ) , _in( 0 ){
for ( list<string>::iterator i=sorter->_files.begin(); i!=sorter->_files.end(); i++ ){
_files.push_back( new FileIterator( *i ) );
_stash.push_back( pair<Data,bool>( Data( BSONObj() , DiskLoc() ) , false ) );
}
if ( _files.size() == 0 && sorter->_cur ){
_in = sorter->_cur;
_it = sorter->_cur->begin();
}
}
void OplogStart::invalidate(const DiskLoc& dl, InvalidationType type) {
if (_needInit) { return; }
if (INVALIDATION_DELETION != type) { return; }
if (_backwardsScanning) {
_cs->invalidate(dl, type);
}
else {
verify(_extentHopping);
if (dl == _curloc) {
_curloc = DiskLoc();
}
}
}
virtual void customLocate(const BSONObj& keyBegin,
int keyBeginLen,
bool afterKey,
const vector<const BSONElement*>& keyEnd,
const vector<bool>& keyEndInclusive) {
// makeQueryObject handles stripping of fieldnames for us.
_it = lower_bound(IndexEntry(IndexEntryComparison::makeQueryObject(
keyBegin,
keyBeginLen,
afterKey,
keyEnd,
keyEndInclusive,
-1), // reverse
DiskLoc()));
}
// static
Status ProjectionExecutor::applyFindSyntax(const FindProjection* proj, WorkingSetMember* wsm) {
BSONObjBuilder bob;
if (proj->_includeID) {
BSONElement elt;
if (!wsm->getFieldDotted("_id", &elt)) {
return Status(ErrorCodes::BadValue, "Couldn't get _id field in proj");
}
bob.append(elt);
}
if (proj->_includedFields.size() > 0) {
// We only want stuff in _fields.
const vector<string>& fields = proj->_includedFields;
for (size_t i = 0; i < fields.size(); ++i) {
BSONElement elt;
// We can project a field that doesn't exist. We just ignore it.
// UNITTEST 11738048
if (wsm->getFieldDotted(fields[i], &elt) && !elt.eoo()) {
// TODO: This fails utterly for dotted fields. Fix.
bob.appendAs(elt, fields[i]);
}
}
}
else if (proj->_excludedFields.size() > 0) {
// We want stuff NOT in _fields. This can't be covered, so we expect an obj.
if (!wsm->hasObj()) {
return Status(ErrorCodes::BadValue,
"exclusion specified for projection but no obj to iter over");
}
const unordered_set<string>& fields = proj->_excludedFields;
BSONObjIterator it(wsm->obj);
while (it.more()) {
BSONElement elt = it.next();
if (!mongoutils::str::equals("_id", elt.fieldName())) {
if (fields.end() == fields.find(elt.fieldName())) {
bob.append(elt);
}
}
}
}
wsm->state = WorkingSetMember::OWNED_OBJ;
wsm->obj = bob.obj();
wsm->keyData.clear();
wsm->loc = DiskLoc();
return Status::OK();
}
Status ProjectionStage::transform(WorkingSetMember* member) {
// The default no-fast-path case.
if (ProjectionStageParams::NO_FAST_PATH == _projImpl) {
return _exec->transform(member);
}
BSONObjBuilder bob;
// Note that even if our fast path analysis is bug-free something that is
// covered might be invalidated and just be an obj. In this case we just go
// through the SIMPLE_DOC path which is still correct if the covered data
// is not available.
//
// SIMPLE_DOC implies that we expect an object so it's kind of redundant.
if ((ProjectionStageParams::SIMPLE_DOC == _projImpl) || member->hasObj()) {
// If we got here because of SIMPLE_DOC the planner shouldn't have messed up.
invariant(member->hasObj());
// Apply the SIMPLE_DOC projection.
transformSimpleInclusion(member->obj, _includedFields, bob);
}
else {
invariant(ProjectionStageParams::COVERED_ONE_INDEX == _projImpl);
// We're pulling data out of the key.
invariant(1 == member->keyData.size());
size_t keyIndex = 0;
// Look at every key element...
BSONObjIterator keyIterator(member->keyData[0].keyData);
while (keyIterator.more()) {
BSONElement elt = keyIterator.next();
// If we're supposed to include it...
if (_includeKey[keyIndex]) {
// Do so.
bob.appendAs(elt, _keyFieldNames[keyIndex]);
}
++keyIndex;
}
}
member->state = WorkingSetMember::OWNED_OBJ;
member->keyData.clear();
member->loc = DiskLoc();
member->obj = bob.obj();
return Status::OK();
}
bool BtreeCursor::skipOutOfRangeKeysAndCheckEnd() {
if ( !ok() ) {
return false;
}
int ret = _boundsIterator->advance( currKey() );
if ( ret == -2 ) {
bucket = DiskLoc();
return false;
}
else if ( ret == -1 ) {
++_nscanned;
return false;
}
++_nscanned;
advanceTo( currKey(), ret, _boundsIterator->after(), _boundsIterator->cmp(), _boundsIterator->inc() );
return true;
}
ClientCursor::~ClientCursor() {
if( _pos == -2 ) {
// defensive: destructor called twice
wassert(false);
return;
}
{
recursive_scoped_lock lock(ccmutex);
setLastLoc_inlock( DiskLoc() ); // removes us from bylocation multimap
clientCursorsById.erase(_cursorid);
// defensive:
(CursorId&)_cursorid = -1;
_pos = -2;
}
}
virtual void keyAndRecordAt(DiskLoc bucket, int keyOffset, BSONObj* keyOut,
DiskLoc* recordOut) const {
verify(!bucket.isNull());
const BtreeBucket<Version> *b = bucket.btree<Version>();
int n = b->getN();
if (n == b->INVALID_N_SENTINEL) {
throw UserException(deletedBucketCode, "keyAt bucket deleted");
}
dassert( n >= 0 && n < 10000 );
if (keyOffset >= n) {
*keyOut = BSONObj();
*recordOut = DiskLoc();
} else {
*keyOut = b->keyNode(keyOffset).key.toBson();
*recordOut = b->keyNode(keyOffset).recordLoc;
}
}
DiskLoc RecordStoreV1Base::getNextRecord( const DiskLoc& loc ) const {
DiskLoc next = getNextRecordInExtent( loc );
if ( !next.isNull() )
return next;
// now traverse extents
Extent* e = _getExtent( _getExtentLocForRecord(loc) );
while ( 1 ) {
if ( e->xnext.isNull() )
return DiskLoc(); // end of collection
e = _getExtent( e->xnext );
if ( !e->firstRecord.isNull() )
break;
// entire extent could be empty, keep looking
}
return e->firstRecord;
}
int nRecords() const {
int count = 0;
const Extent* ext;
for ( DiskLoc extLoc = nsd()->firstExtent();
!extLoc.isNull();
extLoc = ext->xnext) {
ext = extentManager()->getExtent(extLoc);
int fileNo = ext->firstRecord.a();
if ( fileNo == -1 )
continue;
for ( int recOfs = ext->firstRecord.getOfs(); recOfs != DiskLoc::NullOfs;
recOfs = recordStore()->recordFor(DiskLoc(fileNo, recOfs))->nextOfs() ) {
++count;
}
}
ASSERT_EQUALS( count, nsd()->numRecords() );
return count;
}
DiskLoc FindingStartCursor::prevLoc( const DiskLoc &rec ) {
Extent *e = rec.rec()->myExtent( rec );
if ( _qp.nsd()->capLooped() ) {
if ( e->xprev.isNull() )
e = _qp.nsd()->lastExtent.ext();
else
e = e->xprev.ext();
if ( e->myLoc != _qp.nsd()->capExtent )
return e->firstRecord;
}
else {
if ( !e->xprev.isNull() ) {
e = e->xprev.ext();
return e->firstRecord;
}
}
return DiskLoc(); // reached beginning of collection
}
DiskLoc ExtentManager::getNextRecord( const DiskLoc& loc ) {
DiskLoc next = getNextRecordInExtent( loc );
if ( !next.isNull() )
return next;
// now traverse extents
Extent *e = extentFor(loc);
while ( 1 ) {
if ( e->xnext.isNull() )
return DiskLoc(); // end of collection
e = e->xnext.ext();
if ( !e->firstRecord.isNull() )
break;
// entire extent could be empty, keep looking
}
return e->firstRecord;
}
void RocksCollectionCatalogEntry::MetaData::parse( const BSONObj& obj ) {
ns = obj["ns"].valuestrsafe();
BSONElement e = obj["indexes"];
if ( e.isABSONObj() ) {
std::vector<BSONElement> entries = e.Array();
for ( unsigned i = 0; i < entries.size(); i++ ) {
BSONObj idx = entries[i].Obj();
IndexMetaData imd;
imd.spec = idx["spec"].Obj();
imd.ready = idx["ready"].trueValue();
imd.head = DiskLoc( idx["head_a"].Int(),
idx["head_b"].Int() );
imd.multikey = idx["multikey"].trueValue();
indexes.push_back( imd );
}
}
}
DiskLoc ExtentManager::getPrevRecord( const DiskLoc& loc ) const {
DiskLoc prev = getPrevRecordInExtent( loc );
if ( !prev.isNull() )
return prev;
// now traverse extents
Extent *e = extentFor(loc);
while ( 1 ) {
if ( e->xprev.isNull() )
return DiskLoc(); // end of collection
e = e->xprev.ext();
if ( !e->firstRecord.isNull() )
break;
// entire extent could be empty, keep looking
}
return e->firstRecord;
}
/* fetch a single object from collection ns that matches query
set your db SavedContext first
*/
DiskLoc Helpers::findOne(const StringData& ns, const BSONObj &query, bool requireIndex) {
CanonicalQuery* cq;
massert(17244, "Could not canonicalize " + query.toString(),
CanonicalQuery::canonicalize(ns.toString(), query, &cq).isOK());
Runner* rawRunner;
size_t options = requireIndex ? QueryPlannerParams::NO_TABLE_SCAN : QueryPlannerParams::DEFAULT;
massert(17245, "Could not get runner for query " + query.toString(),
getRunner(cq, &rawRunner, options).isOK());
auto_ptr<Runner> runner(rawRunner);
Runner::RunnerState state;
DiskLoc loc;
if (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &loc))) {
return loc;
}
return DiskLoc();
}
void run() {
create();
nsd()->deletedList[ 2 ] = nsd()->deletedList[ 0 ].drec()->nextDeleted.drec()->nextDeleted;
nsd()->deletedList[ 0 ].drec()->nextDeleted.drec()->nextDeleted = DiskLoc();
nsd()->deletedList[ 1 ].Null();
NamespaceDetails *d = nsd();
zero( &d->capExtent );
zero( &d->capFirstNewRecord );
nsd();
ASSERT( nsd()->firstExtent == nsd()->capExtent );
ASSERT( nsd()->capExtent.getOfs() != 0 );
ASSERT( !nsd()->capFirstNewRecord.isValid() );
int nDeleted = 0;
for ( DiskLoc i = nsd()->deletedList[ 0 ]; !i.isNull(); i = i.drec()->nextDeleted, ++nDeleted );
ASSERT_EQUALS( 10, nDeleted );
ASSERT( nsd()->deletedList[ 1 ].isNull() );
}
DiskLoc RecordStoreV1Base::getPrevRecord(OperationContext* txn, const DiskLoc& loc) const {
DiskLoc prev = getPrevRecordInExtent(txn, loc);
if (!prev.isNull()) {
return prev;
}
// now traverse extents
Extent* e = _getExtent(txn, _getExtentLocForRecord(txn, loc));
while (1) {
if (e->xprev.isNull())
return DiskLoc(); // end of collection
e = _getExtent(txn, e->xprev);
if (!e->firstRecord.isNull())
break;
// entire extent could be empty, keep looking
}
return e->lastRecord;
}
// Call advance() on a reverse cursor until it is exhausted.
// When a cursor positioned at EOF is advanced, it stays at EOF.
TEST( SortedDataInterface, ExhaustCursorReversed ) {
scoped_ptr<HarnessHelper> harnessHelper( newHarnessHelper() );
scoped_ptr<SortedDataInterface> sorted( harnessHelper->newSortedDataInterface( false ) );
{
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
ASSERT( sorted->isEmpty( opCtx.get() ) );
}
int nToInsert = 10;
for ( int i = 0; i < nToInsert; i++ ) {
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
{
WriteUnitOfWork uow( opCtx.get() );
BSONObj key = BSON( "" << i );
DiskLoc loc( 42, i * 2 );
ASSERT_OK( sorted->insert( opCtx.get(), key, loc, true ) );
uow.commit();
}
}
{
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
ASSERT_EQUALS( nToInsert, sorted->numEntries( opCtx.get() ) );
}
{
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
scoped_ptr<SortedDataInterface::Cursor> cursor( sorted->newCursor( opCtx.get(), -1 ) );
ASSERT( !cursor->locate( maxKey, maxDiskLoc ) );
for ( int i = nToInsert - 1; i >= 0; i-- ) {
ASSERT( !cursor->isEOF() );
ASSERT_EQUALS( BSON( "" << i ), cursor->getKey() );
ASSERT_EQUALS( DiskLoc( 42, i * 2 ), cursor->getDiskLoc() );
cursor->advance();
}
ASSERT( cursor->isEOF() );
// Cursor at EOF should remain at EOF when advanced
cursor->advance();
ASSERT( cursor->isEOF() );
}
}
virtual void keyAndRecordAt(DiskLoc bucket, int keyOffset, BSONObj* keyOut,
DiskLoc* recordOut) const {
verify(!bucket.isNull());
const BtreeBucket<Version> *b = bucket.btree<Version>();
int n = b->getN();
// If n is 0xffff the bucket was deleted.
if (keyOffset < 0 || keyOffset >= n || n == 0xffff || !b->isUsed(keyOffset)) {
return;
}
if (keyOffset >= n) {
*keyOut = BSONObj();
*recordOut = DiskLoc();
} else {
*keyOut = b->keyNode(keyOffset).key.toBson();
*recordOut = b->keyNode(keyOffset).recordLoc;
}
}
// Insert multiple keys and try to iterate through all of them
// using a forward cursor while calling savePosition() and
// restorePosition() in succession.
TEST( SortedDataInterface, SaveAndRestorePositionWhileIterateCursor ) {
scoped_ptr<HarnessHelper> harnessHelper( newHarnessHelper() );
scoped_ptr<SortedDataInterface> sorted( harnessHelper->newSortedDataInterface( false ) );
{
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
ASSERT( sorted->isEmpty( opCtx.get() ) );
}
int nToInsert = 10;
for ( int i = 0; i < nToInsert; i++ ) {
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
{
WriteUnitOfWork uow( opCtx.get() );
BSONObj key = BSON( "" << i );
DiskLoc loc( 42, i * 2 );
ASSERT_OK( sorted->insert( opCtx.get(), key, loc, true ) );
uow.commit();
}
}
{
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
ASSERT_EQUALS( nToInsert, sorted->numEntries( opCtx.get() ) );
}
{
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
scoped_ptr<SortedDataInterface::Cursor> cursor( sorted->newCursor( opCtx.get(), 1 ) );
ASSERT( !cursor->locate( minKey, minDiskLoc ) );
for ( int i = 0; i < nToInsert; i++ ) {
ASSERT( !cursor->isEOF() );
ASSERT_EQUALS( BSON( "" << i ), cursor->getKey() );
ASSERT_EQUALS( DiskLoc( 42, i * 2 ), cursor->getDiskLoc() );
cursor->advance();
cursor->savePosition();
cursor->restorePosition( opCtx.get() );
}
ASSERT( cursor->isEOF() );
}
}
// Insert the same key multiple times and try to iterate through each
// occurrence using a reverse cursor while calling savePosition() and
// restorePosition() in succession. Verify that the DiskLoc is saved
// as part of the current position of the cursor.
TEST( SortedDataInterface, SaveAndRestorePositionWhileIterateCursorWithDupKeysReversed ) {
scoped_ptr<HarnessHelper> harnessHelper( newHarnessHelper() );
scoped_ptr<SortedDataInterface> sorted( harnessHelper->newSortedDataInterface( false ) );
{
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
ASSERT( sorted->isEmpty( opCtx.get() ) );
}
int nToInsert = 10;
for ( int i = 0; i < nToInsert; i++ ) {
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
{
WriteUnitOfWork uow( opCtx.get() );
DiskLoc loc( 42, i * 2 );
ASSERT_OK( sorted->insert( opCtx.get(), key1, loc, true /* allow duplicates */ ) );
uow.commit();
}
}
{
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
ASSERT_EQUALS( nToInsert, sorted->numEntries( opCtx.get() ) );
}
{
scoped_ptr<OperationContext> opCtx( harnessHelper->newOperationContext() );
scoped_ptr<SortedDataInterface::Cursor> cursor( sorted->newCursor( opCtx.get(), -1 ) );
ASSERT( !cursor->locate( maxKey, maxDiskLoc ) );
for ( int i = nToInsert - 1; i >= 0; i-- ) {
ASSERT( !cursor->isEOF() );
ASSERT_EQUALS( key1, cursor->getKey() );
ASSERT_EQUALS( DiskLoc( 42, i * 2 ), cursor->getDiskLoc() );
cursor->advance();
cursor->savePosition();
cursor->restorePosition( opCtx.get() );
}
ASSERT( cursor->isEOF() );
}
}
ClientCursor::~ClientCursor() {
if( _pos == -2 ) {
// defensive: destructor called twice
wassert(false);
return;
}
{
recursive_scoped_lock lock(ccmutex);
if (NULL != _c.get()) {
// Removes 'this' from bylocation map
setLastLoc_inlock( DiskLoc() );
}
clientCursorsById.erase(_cursorid);
// defensive:
_cursorid = INVALID_CURSOR_ID;
_pos = -2;
_pinValue = 0;
}
}
