void saveUnpositioned() final {
savePositioned();
_lastReturnedId = RecordId();
}
RecordId WiredTigerRecordStore::_nextId() {
invariant(!_useOplogHack);
RecordId out = RecordId(_nextIdNum.fetchAndAdd(1));
invariant(out.isNormal());
return out;
}
RecordId WiredTigerRecordStore::_fromKey(int64_t key) {
return RecordId(key);
}
virtual RecordId getNext() {
return RecordId();
}
RecordId VisibilityManager::getAllCommittedRecord() {
stdx::lock_guard<stdx::mutex> lock(_stateLock);
return _uncommittedRecords.empty() ? _highestSeen
: RecordId(_uncommittedRecords.begin()->repr() - 1);
}
StatusWith<RollBackLocalOperations::RollbackCommonPoint> RollBackLocalOperations::onRemoteOperation(
const BSONObj& operation) {
if (_scanned == 0) {
auto result = _localOplogIterator->next();
if (!result.isOK()) {
return StatusWith<RollbackCommonPoint>(ErrorCodes::OplogStartMissing,
"no oplog during initsync");
}
_localOplogValue = result.getValue();
long long diff = static_cast<long long>(getTimestamp(_localOplogValue).getSecs()) -
getTimestamp(operation).getSecs();
// diff could be positive, negative, or zero
log() << "rollback our last optime: " << getTimestamp(_localOplogValue).toStringPretty();
log() << "rollback their last optime: " << getTimestamp(operation).toStringPretty();
log() << "rollback diff in end of log times: " << diff << " seconds";
if (diff > 1800) {
severe() << "rollback too long a time period for a rollback.";
return StatusWith<RollbackCommonPoint>(
ErrorCodes::ExceededTimeLimit,
"rollback error: not willing to roll back more than 30 minutes of data");
}
}
while (getTimestamp(_localOplogValue) > getTimestamp(operation)) {
_scanned++;
auto status = _rollbackOperation(_localOplogValue.first);
if (!status.isOK()) {
invariant(ErrorCodes::NoSuchKey != status.code());
return status;
}
auto result = _localOplogIterator->next();
if (!result.isOK()) {
severe() << "rollback error RS101 reached beginning of local oplog";
log() << " scanned: " << _scanned;
log() << " theirTime: " << getTimestamp(operation).toStringLong();
log() << " ourTime: " << getTimestamp(_localOplogValue).toStringLong();
return StatusWith<RollbackCommonPoint>(ErrorCodes::NoMatchingDocument,
"RS101 reached beginning of local oplog [2]");
}
_localOplogValue = result.getValue();
}
if (getTimestamp(_localOplogValue) == getTimestamp(operation)) {
_scanned++;
if (getHash(_localOplogValue) == getHash(operation)) {
return StatusWith<RollbackCommonPoint>(
std::make_pair(getTimestamp(_localOplogValue), _localOplogValue.second));
}
auto status = _rollbackOperation(_localOplogValue.first);
if (!status.isOK()) {
invariant(ErrorCodes::NoSuchKey != status.code());
return status;
}
auto result = _localOplogIterator->next();
if (!result.isOK()) {
severe() << "rollback error RS101 reached beginning of local oplog";
log() << " scanned: " << _scanned;
log() << " theirTime: " << getTimestamp(operation).toStringLong();
log() << " ourTime: " << getTimestamp(_localOplogValue).toStringLong();
return StatusWith<RollbackCommonPoint>(ErrorCodes::NoMatchingDocument,
"RS101 reached beginning of local oplog [1]");
}
_localOplogValue = result.getValue();
return StatusWith<RollbackCommonPoint>(
ErrorCodes::NoSuchKey,
"Unable to determine common point - same timestamp but different hash. "
"Need to process additional remote operations.");
}
if (getTimestamp(_localOplogValue) < getTimestamp(operation)) {
_scanned++;
return StatusWith<RollbackCommonPoint>(ErrorCodes::NoSuchKey,
"Unable to determine common point. "
"Need to process additional remote operations.");
}
return RollbackCommonPoint(Timestamp(Seconds(1), 0), RecordId());
}
RecordId getRecordIdAt(const void* ptr, size_t byteOffset = 0) {
RecordIdRepr repr;
memcpy(&repr, static_cast<const char*>(ptr) + byteOffset, sizeof(repr));
return RecordId(repr);
}
bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result) {
BSONElement first = cmdObj.firstElement();
uassert(28528,
str::stream() << "Argument to listIndexes must be of type String, not "
<< typeName(first.type()),
first.type() == String);
StringData collectionName = first.valueStringData();
uassert(28529,
str::stream() << "Argument to listIndexes must be a collection name, "
<< "not the empty string",
!collectionName.empty());
const NamespaceString ns(dbname, collectionName);
const long long defaultBatchSize = std::numeric_limits<long long>::max();
long long batchSize;
Status parseCursorStatus = parseCommandCursorOptions(cmdObj, defaultBatchSize, &batchSize);
if (!parseCursorStatus.isOK()) {
return appendCommandStatus(result, parseCursorStatus);
}
AutoGetCollectionForRead autoColl(txn, ns);
if (!autoColl.getDb()) {
return appendCommandStatus(result,
Status(ErrorCodes::NamespaceNotFound, "no database"));
}
const Collection* collection = autoColl.getCollection();
if (!collection) {
return appendCommandStatus(result,
Status(ErrorCodes::NamespaceNotFound, "no collection"));
}
const CollectionCatalogEntry* cce = collection->getCatalogEntry();
invariant(cce);
vector<string> indexNames;
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
indexNames.clear();
cce->getAllIndexes(txn, &indexNames);
}
MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "listIndexes", ns.ns());
auto ws = make_unique<WorkingSet>();
auto root = make_unique<QueuedDataStage>(txn, ws.get());
for (size_t i = 0; i < indexNames.size(); i++) {
BSONObj indexSpec;
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
indexSpec = cce->getIndexSpec(txn, indexNames[i]);
}
MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "listIndexes", ns.ns());
WorkingSetID id = ws->allocate();
WorkingSetMember* member = ws->get(id);
member->keyData.clear();
member->loc = RecordId();
member->obj = Snapshotted<BSONObj>(SnapshotId(), indexSpec.getOwned());
member->transitionToOwnedObj();
root->pushBack(id);
}
std::string cursorNamespace = str::stream() << dbname << ".$cmd." << name << "."
<< ns.coll();
dassert(NamespaceString(cursorNamespace).isValid());
dassert(NamespaceString(cursorNamespace).isListIndexesCursorNS());
dassert(ns == NamespaceString(cursorNamespace).getTargetNSForListIndexes());
auto statusWithPlanExecutor = PlanExecutor::make(
txn, std::move(ws), std::move(root), cursorNamespace, PlanExecutor::YIELD_MANUAL);
if (!statusWithPlanExecutor.isOK()) {
return appendCommandStatus(result, statusWithPlanExecutor.getStatus());
}
unique_ptr<PlanExecutor> exec = std::move(statusWithPlanExecutor.getValue());
BSONArrayBuilder firstBatch;
const int byteLimit = FindCommon::kMaxBytesToReturnToClientAtOnce;
for (long long objCount = 0; objCount < batchSize && firstBatch.len() < byteLimit;
objCount++) {
BSONObj next;
PlanExecutor::ExecState state = exec->getNext(&next, NULL);
if (state == PlanExecutor::IS_EOF) {
break;
}
invariant(state == PlanExecutor::ADVANCED);
firstBatch.append(next);
}
CursorId cursorId = 0LL;
if (!exec->isEOF()) {
exec->saveState();
exec->detachFromOperationContext();
ClientCursor* cursor =
new ClientCursor(CursorManager::getGlobalCursorManager(),
exec.release(),
cursorNamespace,
txn->recoveryUnit()->isReadingFromMajorityCommittedSnapshot());
cursorId = cursor->cursorid();
}
appendCursorResponseObject(cursorId, cursorNamespace, firstBatch.arr(), &result);
return true;
}
virtual RecordId read (ESM::ESMReader& reader) { return RecordId(); }
bool keyExists(const IndexSet& data, const BSONObj& key) {
IndexSet::const_iterator it = data.find(IndexKeyEntry(key, RecordId()));
return it != data.end();
}
virtual StatusWith<RecordId> insertRecord(OperationContext* txn,
const DocWriter* doc,
bool enforceQuota) {
_numInserts++;
return StatusWith<RecordId>(RecordId(6, 4));
}
RecordId MobileRecordStore::_nextId() {
RecordId out = RecordId(_nextIdNum.fetchAndAdd(1));
invariant(out.isNormal());
return out;
}
void saveUnpositioned() final {
save();
_savedId = RecordId(_startIdNum);
}
void run() {
create();
nsd()->deletedListEntry( 2 ) = nsd()->cappedListOfAllDeletedRecords().drec()->nextDeleted().drec()->nextDeleted();
nsd()->cappedListOfAllDeletedRecords().drec()->nextDeleted().drec()->nextDeleted().writing() = RecordId();
nsd()->cappedLastDelRecLastExtent().Null();
NamespaceDetails *d = nsd();
zero( &d->capExtent() );
zero( &d->capFirstNewRecord() );
// this has a side effect of called NamespaceDetails::cappedCheckMigrate
db()->namespaceIndex().details( ns() );
ASSERT( nsd()->firstExtent() == nsd()->capExtent() );
ASSERT( nsd()->capExtent().getOfs() != 0 );
ASSERT( !nsd()->capFirstNewRecord().isValid() );
int nDeleted = 0;
for ( RecordId i = nsd()->cappedListOfAllDeletedRecords(); !i.isNull(); i = i.drec()->nextDeleted(), ++nDeleted );
ASSERT_EQUALS( 10, nDeleted );
ASSERT( nsd()->cappedLastDelRecLastExtent().isNull() );
}
RecordId WiredTigerRecordStore::lowestCappedHiddenRecord() const {
stdx::lock_guard<stdx::mutex> lk(_uncommittedDiskLocsMutex);
return _uncommittedDiskLocs.empty() ? RecordId() : _uncommittedDiskLocs.front();
}
TEST(KVEngineTestHarness, AllCommittedTimestamp) {
unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create());
KVEngine* engine = helper->getEngine();
if (!engine->supportsDocLocking())
return;
unique_ptr<RecordStore> rs;
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
CollectionOptions options;
options.capped = true;
options.cappedSize = 10240;
options.cappedMaxDocs = -1;
NamespaceString oplogNss("local.oplog.rs");
ASSERT_OK(engine->createRecordStore(&opCtx, oplogNss.ns(), "ident", options));
rs = engine->getRecordStore(&opCtx, oplogNss.ns(), "ident", options);
ASSERT(rs);
}
{
Timestamp t11(1, 1);
Timestamp t12(1, 2);
Timestamp t21(2, 1);
auto t11Doc = BSON("ts" << t11);
auto t12Doc = BSON("ts" << t12);
auto t21Doc = BSON("ts" << t21);
Timestamp allCommitted = engine->getAllCommittedTimestamp();
MyOperationContext opCtx1(engine);
WriteUnitOfWork uow1(&opCtx1);
ASSERT_EQ(invariant(rs->insertRecord(
&opCtx1, t11Doc.objdata(), t11Doc.objsize(), Timestamp::min())),
RecordId(1, 1));
Timestamp lastAllCommitted = allCommitted;
allCommitted = engine->getAllCommittedTimestamp();
ASSERT_GTE(allCommitted, lastAllCommitted);
ASSERT_LT(allCommitted, t11);
MyOperationContext opCtx2(engine);
WriteUnitOfWork uow2(&opCtx2);
ASSERT_EQ(invariant(rs->insertRecord(
&opCtx2, t21Doc.objdata(), t21Doc.objsize(), Timestamp::min())),
RecordId(2, 1));
uow2.commit();
lastAllCommitted = allCommitted;
allCommitted = engine->getAllCommittedTimestamp();
ASSERT_GTE(allCommitted, lastAllCommitted);
ASSERT_LT(allCommitted, t11);
ASSERT_EQ(invariant(rs->insertRecord(
&opCtx1, t12Doc.objdata(), t12Doc.objsize(), Timestamp::min())),
RecordId(1, 2));
lastAllCommitted = allCommitted;
allCommitted = engine->getAllCommittedTimestamp();
ASSERT_GTE(allCommitted, lastAllCommitted);
ASSERT_LT(allCommitted, t11);
uow1.commit();
lastAllCommitted = allCommitted;
allCommitted = engine->getAllCommittedTimestamp();
ASSERT_GTE(allCommitted, lastAllCommitted);
ASSERT_LTE(allCommitted, t21);
}
}
bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result) {
BSONElement first = cmdObj.firstElement();
uassert(
28528,
str::stream() << "Argument to listIndexes must be of type String, not "
<< typeName(first.type()),
first.type() == String);
const NamespaceString ns(parseNs(dbname, cmdObj));
uassert(
28529,
str::stream() << "Argument to listIndexes must be a collection name, "
<< "not the empty string",
!ns.coll().empty());
const long long defaultBatchSize = std::numeric_limits<long long>::max();
long long batchSize;
Status parseCursorStatus = parseCommandCursorOptions(cmdObj,
defaultBatchSize,
&batchSize);
if (!parseCursorStatus.isOK()) {
return appendCommandStatus(result, parseCursorStatus);
}
AutoGetCollectionForRead autoColl(txn, ns);
if (!autoColl.getDb()) {
return appendCommandStatus( result, Status( ErrorCodes::NamespaceNotFound,
"no database" ) );
}
const Collection* collection = autoColl.getCollection();
if (!collection) {
return appendCommandStatus( result, Status( ErrorCodes::NamespaceNotFound,
"no collection" ) );
}
const CollectionCatalogEntry* cce = collection->getCatalogEntry();
invariant(cce);
vector<string> indexNames;
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
indexNames.clear();
cce->getAllIndexes( txn, &indexNames );
} MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "listIndexes", ns.ns());
std::auto_ptr<WorkingSet> ws(new WorkingSet());
std::auto_ptr<QueuedDataStage> root(new QueuedDataStage(ws.get()));
for ( size_t i = 0; i < indexNames.size(); i++ ) {
BSONObj indexSpec;
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
indexSpec = cce->getIndexSpec( txn, indexNames[i] );
} MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "listIndexes", ns.ns());
WorkingSetMember member;
member.state = WorkingSetMember::OWNED_OBJ;
member.keyData.clear();
member.loc = RecordId();
member.obj = Snapshotted<BSONObj>(SnapshotId(), indexSpec.getOwned());
root->pushBack(member);
}
std::string cursorNamespace = str::stream() << dbname << ".$cmd." << name << "."
<< ns.coll();
dassert(NamespaceString(cursorNamespace).isValid());
dassert(NamespaceString(cursorNamespace).isListIndexesGetMore());
dassert(ns == NamespaceString(cursorNamespace).getTargetNSForListIndexesGetMore());
PlanExecutor* rawExec;
Status makeStatus = PlanExecutor::make(txn,
ws.release(),
root.release(),
cursorNamespace,
PlanExecutor::YIELD_MANUAL,
&rawExec);
std::auto_ptr<PlanExecutor> exec(rawExec);
if (!makeStatus.isOK()) {
return appendCommandStatus( result, makeStatus );
}
BSONArrayBuilder firstBatch;
const int byteLimit = MaxBytesToReturnToClientAtOnce;
for (long long objCount = 0;
objCount < batchSize && firstBatch.len() < byteLimit;
objCount++) {
BSONObj next;
PlanExecutor::ExecState state = exec->getNext(&next, NULL);
if ( state == PlanExecutor::IS_EOF ) {
break;
}
invariant( state == PlanExecutor::ADVANCED );
firstBatch.append(next);
}
CursorId cursorId = 0LL;
if ( !exec->isEOF() ) {
exec->saveState();
ClientCursor* cursor = new ClientCursor(CursorManager::getGlobalCursorManager(),
exec.release(),
cursorNamespace);
cursorId = cursor->cursorid();
}
appendCursorResponseObject( cursorId, cursorNamespace, firstBatch.arr(), &result );
return true;
}
TEST(KVCatalogTest, Idx1) {
unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create());
KVEngine* engine = helper->getEngine();
unique_ptr<RecordStore> rs;
unique_ptr<KVCatalog> catalog;
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
ASSERT_OK(engine->createRecordStore(&opCtx, "catalog", "catalog", CollectionOptions()));
rs = engine->getRecordStore(&opCtx, "catalog", "catalog", CollectionOptions());
catalog.reset(new KVCatalog(rs.get(), false, false));
uow.commit();
}
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
ASSERT_OK(
catalog->newCollection(&opCtx, "a.b", CollectionOptions(), KVPrefix::kNotPrefixed));
ASSERT_NOT_EQUALS("a.b", catalog->getCollectionIdent("a.b"));
ASSERT_TRUE(catalog->isUserDataIdent(catalog->getCollectionIdent("a.b")));
uow.commit();
}
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
BSONCollectionCatalogEntry::MetaData md;
md.ns = "a.b";
BSONCollectionCatalogEntry::IndexMetaData imd;
imd.spec = BSON("name"
<< "foo");
imd.ready = false;
imd.head = RecordId();
imd.multikey = false;
imd.prefix = KVPrefix::kNotPrefixed;
imd.isBackgroundSecondaryBuild = false;
md.indexes.push_back(imd);
catalog->putMetaData(&opCtx, "a.b", md);
uow.commit();
}
string idxIndent;
{
MyOperationContext opCtx(engine);
idxIndent = catalog->getIndexIdent(&opCtx, "a.b", "foo");
}
{
MyOperationContext opCtx(engine);
ASSERT_EQUALS(idxIndent, catalog->getIndexIdent(&opCtx, "a.b", "foo"));
ASSERT_TRUE(catalog->isUserDataIdent(catalog->getIndexIdent(&opCtx, "a.b", "foo")));
}
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
BSONCollectionCatalogEntry::MetaData md;
md.ns = "a.b";
catalog->putMetaData(&opCtx, "a.b", md); // remove index
BSONCollectionCatalogEntry::IndexMetaData imd;
imd.spec = BSON("name"
<< "foo");
imd.ready = false;
imd.head = RecordId();
imd.multikey = false;
imd.prefix = KVPrefix::kNotPrefixed;
imd.isBackgroundSecondaryBuild = false;
md.indexes.push_back(imd);
catalog->putMetaData(&opCtx, "a.b", md);
uow.commit();
}
{
MyOperationContext opCtx(engine);
ASSERT_NOT_EQUALS(idxIndent, catalog->getIndexIdent(&opCtx, "a.b", "foo"));
}
}
// Returns comparison relative to direction of scan. If rhs would be seen later, returns
// a positive value.
int compareKeys(const BSONObj& lhs, const BSONObj& rhs) const {
int cmp = _data.value_comp().compare({lhs, RecordId()}, {rhs, RecordId()});
return _forward ? cmp : -cmp;
}
TEST(KVCatalogTest, RestartForPrefixes) {
storageGlobalParams.groupCollections = true;
ON_BLOCK_EXIT([&] { storageGlobalParams.groupCollections = false; });
KVPrefix abCollPrefix = KVPrefix::getNextPrefix(NamespaceString("a.b"));
KVPrefix fooIndexPrefix = KVPrefix::getNextPrefix(NamespaceString("a.b"));
unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create());
KVEngine* engine = helper->getEngine();
{
unique_ptr<RecordStore> rs;
unique_ptr<KVCatalog> catalog;
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
ASSERT_OK(engine->createRecordStore(&opCtx, "catalog", "catalog", CollectionOptions()));
rs = engine->getRecordStore(&opCtx, "catalog", "catalog", CollectionOptions());
catalog.reset(new KVCatalog(rs.get(), false, false));
uow.commit();
}
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
ASSERT_OK(catalog->newCollection(&opCtx, "a.b", CollectionOptions(), abCollPrefix));
ASSERT_NOT_EQUALS("a.b", catalog->getCollectionIdent("a.b"));
ASSERT_TRUE(catalog->isUserDataIdent(catalog->getCollectionIdent("a.b")));
uow.commit();
}
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
BSONCollectionCatalogEntry::MetaData md;
md.ns = "a.b";
BSONCollectionCatalogEntry::IndexMetaData imd;
imd.spec = BSON("name"
<< "foo");
imd.ready = false;
imd.head = RecordId();
imd.multikey = false;
imd.prefix = fooIndexPrefix;
imd.isBackgroundSecondaryBuild = false;
md.indexes.push_back(imd);
md.prefix = abCollPrefix;
catalog->putMetaData(&opCtx, "a.b", md);
uow.commit();
}
}
engine = helper->restartEngine();
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
unique_ptr<RecordStore> rs =
engine->getRecordStore(&opCtx, "catalog", "catalog", CollectionOptions());
unique_ptr<KVCatalog> catalog = stdx::make_unique<KVCatalog>(rs.get(), false, false);
catalog->init(&opCtx);
const BSONCollectionCatalogEntry::MetaData md = catalog->getMetaData(&opCtx, "a.b");
ASSERT_EQ("a.b", md.ns);
ASSERT_EQ(abCollPrefix, md.prefix);
ASSERT_EQ(fooIndexPrefix, md.indexes[md.findIndexOffset("foo")].prefix);
}
}
RecordId KVRecordStore::_nextId() {
return RecordId(_nextIdNum.fetchAndAdd(1));
}
virtual RecordId curr() {
return RecordId();
}
