Status KVCatalog::newCollection( OperationContext* opCtx,
const StringData& ns,
const CollectionOptions& options ) {
std::stringstream ss;
ss << ns << "-" << _rand << "-" << _next.fetchAndAdd( 1 );
string ident = ss.str();
boost::mutex::scoped_lock lk( _identsLock );
Entry& old = _idents[ns.toString()];
if ( !old.ident.empty() ) {
return Status( ErrorCodes::NamespaceExists, "collection already exists" );
}
BSONObj obj;
{
BSONObjBuilder b;
b.append( "ns", ns );
b.append( "ident", ident );
BSONCollectionCatalogEntry::MetaData md;
md.ns = ns.toString();
md.options = options;
b.append( "md", md.toBSON() );
obj = b.obj();
}
StatusWith<DiskLoc> res = _rs->insertRecord( opCtx, obj.objdata(), obj.objsize(), false );
if ( !res.isOK() )
return res.getStatus();
old = Entry( ident, res.getValue() );
LOG(1) << "stored meta data for " << ns << " @ " << res.getValue();
return Status::OK();
}
const BSONCollectionCatalogEntry::MetaData KVCatalog::getMetaData( OperationContext* opCtx,
const StringData& ns ) {
DiskLoc loc;
BSONObj obj = _findEntry( opCtx, ns, &loc );
LOG(1) << " got: " << obj;
BSONCollectionCatalogEntry::MetaData md;
if ( obj["md"].isABSONObj() )
md.parse( obj["md"].Obj() );
return md;
}
BSONCollectionCatalogEntry::MetaData KVCatalog::getMetaData(OperationContext* opCtx,
StringData ns) const {
BSONObj obj = _findEntry(opCtx, ns);
LOG(3) << " fetched CCE metadata: " << obj;
BSONCollectionCatalogEntry::MetaData md;
const BSONElement mdElement = obj["md"];
if (mdElement.isABSONObj()) {
LOG(3) << "returning metadata: " << mdElement;
md.parse(mdElement.Obj());
}
return md;
}
Status KVCatalog::renameCollection(OperationContext* opCtx,
StringData fromNS,
StringData toNS,
bool stayTemp) {
std::unique_ptr<Lock::ResourceLock> rLk;
if (!_isRsThreadSafe && opCtx->lockState()) {
rLk.reset(new Lock::ResourceLock(opCtx->lockState(), resourceIdCatalogMetadata, MODE_X));
}
RecordId loc;
BSONObj old = _findEntry(opCtx, fromNS, &loc).getOwned();
{
BSONObjBuilder b;
b.append("ns", toNS);
BSONCollectionCatalogEntry::MetaData md;
md.parse(old["md"].Obj());
md.rename(toNS);
if (!stayTemp)
md.options.temp = false;
b.append("md", md.toBSON());
b.appendElementsUnique(old);
BSONObj obj = b.obj();
StatusWith<RecordId> status =
_rs->updateRecord(opCtx, loc, obj.objdata(), obj.objsize(), false, NULL);
fassert(28522, status.getStatus());
invariant(status.getValue() == loc);
}
stdx::lock_guard<stdx::mutex> lk(_identsLock);
const NSToIdentMap::iterator fromIt = _idents.find(fromNS.toString());
invariant(fromIt != _idents.end());
opCtx->recoveryUnit()->registerChange(new RemoveIdentChange(this, fromNS, fromIt->second));
opCtx->recoveryUnit()->registerChange(new AddIdentChange(this, toNS));
_idents.erase(fromIt);
_idents[toNS.toString()] = Entry(old["ident"].String(), loc);
return Status::OK();
}
Status KVCatalog::newCollection(OperationContext* opCtx,
StringData ns,
const CollectionOptions& options) {
invariant(opCtx->lockState() == NULL ||
opCtx->lockState()->isDbLockedForMode(nsToDatabaseSubstring(ns), MODE_X));
std::unique_ptr<Lock::ResourceLock> rLk;
if (!_isRsThreadSafe && opCtx->lockState()) {
rLk.reset(new Lock::ResourceLock(opCtx->lockState(), resourceIdCatalogMetadata, MODE_X));
}
const string ident = _newUniqueIdent(ns, "collection");
stdx::lock_guard<stdx::mutex> lk(_identsLock);
Entry& old = _idents[ns.toString()];
if (!old.ident.empty()) {
return Status(ErrorCodes::NamespaceExists, "collection already exists");
}
opCtx->recoveryUnit()->registerChange(new AddIdentChange(this, ns));
BSONObj obj;
{
BSONObjBuilder b;
b.append("ns", ns);
b.append("ident", ident);
BSONCollectionCatalogEntry::MetaData md;
md.ns = ns.toString();
md.options = options;
b.append("md", md.toBSON());
obj = b.obj();
}
StatusWith<RecordId> res = _rs->insertRecord(opCtx, obj.objdata(), obj.objsize(), false);
if (!res.isOK())
return res.getStatus();
old = Entry(ident, res.getValue());
LOG(1) << "stored meta data for " << ns << " @ " << res.getValue();
return Status::OK();
}
StatusWith<std::string> KVCatalog::newOrphanedIdent(OperationContext* opCtx, std::string ident) {
// The collection will be named local.orphan.xxxxx.
std::string identNs = ident;
std::replace(identNs.begin(), identNs.end(), '-', '_');
std::string ns = NamespaceString(NamespaceString::kOrphanCollectionDb,
NamespaceString::kOrphanCollectionPrefix + identNs)
.ns();
stdx::lock_guard<stdx::mutex> lk(_identsLock);
Entry& old = _idents[ns];
if (!old.ident.empty()) {
return Status(ErrorCodes::NamespaceExists,
str::stream() << ns << " already exists in the catalog");
}
opCtx->recoveryUnit()->registerChange(new AddIdentChange(this, ns));
// Generate a new UUID for the orphaned collection.
CollectionOptions optionsWithUUID;
optionsWithUUID.uuid.emplace(CollectionUUID::gen());
BSONObj obj;
{
BSONObjBuilder b;
b.append("ns", ns);
b.append("ident", ident);
BSONCollectionCatalogEntry::MetaData md;
md.ns = ns;
// Default options with newly generated UUID.
md.options = optionsWithUUID;
// Not Prefixed.
md.prefix = KVPrefix::kNotPrefixed;
b.append("md", md.toBSON());
obj = b.obj();
}
StatusWith<RecordId> res = _rs->insertRecord(opCtx, obj.objdata(), obj.objsize(), Timestamp());
if (!res.isOK())
return res.getStatus();
old = Entry(ident, res.getValue());
LOG(1) << "stored meta data for orphaned collection " << ns << " @ " << res.getValue();
return StatusWith<std::string>(std::move(ns));
}
Status KVCatalog::newCollection(OperationContext* opCtx,
StringData ns,
const CollectionOptions& options,
KVPrefix prefix) {
invariant(opCtx->lockState()->isDbLockedForMode(nsToDatabaseSubstring(ns), MODE_X));
const string ident = _newUniqueIdent(ns, "collection");
stdx::lock_guard<stdx::mutex> lk(_identsLock);
Entry& old = _idents[ns.toString()];
if (!old.ident.empty()) {
return Status(ErrorCodes::NamespaceExists, "collection already exists");
}
opCtx->recoveryUnit()->registerChange(new AddIdentChange(this, ns));
BSONObj obj;
{
BSONObjBuilder b;
b.append("ns", ns);
b.append("ident", ident);
BSONCollectionCatalogEntry::MetaData md;
md.ns = ns.toString();
md.options = options;
md.prefix = prefix;
b.append("md", md.toBSON());
obj = b.obj();
}
const bool enforceQuota = false;
// TODO SERVER-30638: using timestamp 0 for these inserts.
StatusWith<RecordId> res =
_rs->insertRecord(opCtx, obj.objdata(), obj.objsize(), Timestamp(), enforceQuota);
if (!res.isOK())
return res.getStatus();
old = Entry(ident, res.getValue());
LOG(1) << "stored meta data for " << ns << " @ " << res.getValue();
return Status::OK();
}
Status KVCatalog::renameCollection( OperationContext* opCtx,
const StringData& fromNS,
const StringData& toNS,
bool stayTemp ) {
DiskLoc loc;
BSONObj old = _findEntry( opCtx, fromNS, &loc ).getOwned();
{
BSONObjBuilder b;
b.append( "ns", toNS );
BSONCollectionCatalogEntry::MetaData md;
md.parse( old["md"].Obj() );
md.rename( toNS );
if ( !stayTemp )
md.options.temp = false;
b.append( "md", md.toBSON() );
b.appendElementsUnique( old );
BSONObj obj = b.obj();
StatusWith<DiskLoc> status = _rs->updateRecord( opCtx,
loc,
obj.objdata(),
obj.objsize(),
false,
NULL );
fassert( 28522, status.getStatus() );
invariant( status.getValue() == loc );
}
boost::mutex::scoped_lock lk( _identsLock );
_idents.erase( fromNS.toString() );
_idents[toNS.toString()] = Entry( old["ident"].String(), loc );
return Status::OK();
}
Status KVCatalog::renameCollection(OperationContext* opCtx,
StringData fromNS,
StringData toNS,
bool stayTemp) {
RecordId loc;
BSONObj old = _findEntry(opCtx, fromNS, &loc).getOwned();
{
BSONObjBuilder b;
b.append("ns", toNS);
BSONCollectionCatalogEntry::MetaData md;
md.parse(old["md"].Obj());
md.rename(toNS);
if (!stayTemp)
md.options.temp = false;
b.append("md", md.toBSON());
b.appendElementsUnique(old);
BSONObj obj = b.obj();
Status status = _rs->updateRecord(opCtx, loc, obj.objdata(), obj.objsize());
fassert(28522, status.isOK());
}
stdx::lock_guard<stdx::mutex> lk(_identsLock);
const NSToIdentMap::iterator fromIt = _idents.find(fromNS.toString());
invariant(fromIt != _idents.end());
opCtx->recoveryUnit()->registerChange(new RemoveIdentChange(this, fromNS, fromIt->second));
opCtx->recoveryUnit()->registerChange(new AddIdentChange(this, toNS));
_idents.erase(fromIt);
_idents[toNS.toString()] = Entry(old["ident"].String(), loc);
return Status::OK();
}
void KVCatalog::putMetaData( OperationContext* opCtx,
const StringData& ns,
BSONCollectionCatalogEntry::MetaData& md ) {
DiskLoc loc;
BSONObj obj = _findEntry( opCtx, ns, &loc );
{
// rebuilt doc
BSONObjBuilder b;
b.append( "md", md.toBSON() );
BSONObjBuilder newIdentMap;
BSONObj oldIdentMap;
if ( obj["idxIdent"].isABSONObj() )
oldIdentMap = obj["idxIdent"].Obj();
// fix ident map
for ( size_t i = 0; i < md.indexes.size(); i++ ) {
string name = md.indexes[i].name();
BSONElement e = oldIdentMap[name];
if ( e.type() == String ) {
newIdentMap.append( e );
continue;
}
// missing, create new
std::stringstream ss;
ss << getCollectionIdent( ns ) << '$' << name
<< '-' << _rand << '-' << _next.fetchAndAdd( 1 );
newIdentMap.append( name, ss.str() );
}
b.append( "idxIdent", newIdentMap.obj() );
// add whatever is left
b.appendElementsUnique( obj );
obj = b.obj();
}
StatusWith<DiskLoc> status = _rs->updateRecord( opCtx,
loc,
obj.objdata(),
obj.objsize(),
false,
NULL );
fassert( 28521, status.getStatus() );
invariant( status.getValue() == loc );
}
void KVCatalog::putMetaData(OperationContext* opCtx,
StringData ns,
BSONCollectionCatalogEntry::MetaData& md) {
std::unique_ptr<Lock::ResourceLock> rLk;
if (!_isRsThreadSafe && opCtx->lockState()) {
rLk.reset(new Lock::ResourceLock(opCtx->lockState(), resourceIdCatalogMetadata, MODE_X));
}
RecordId loc;
BSONObj obj = _findEntry(opCtx, ns, &loc);
{
// rebuilt doc
BSONObjBuilder b;
b.append("md", md.toBSON());
BSONObjBuilder newIdentMap;
BSONObj oldIdentMap;
if (obj["idxIdent"].isABSONObj())
oldIdentMap = obj["idxIdent"].Obj();
// fix ident map
for (size_t i = 0; i < md.indexes.size(); i++) {
string name = md.indexes[i].name();
BSONElement e = oldIdentMap[name];
if (e.type() == String) {
newIdentMap.append(e);
continue;
}
// missing, create new
newIdentMap.append(name, _newUniqueIdent(ns, "index"));
}
b.append("idxIdent", newIdentMap.obj());
// add whatever is left
b.appendElementsUnique(obj);
obj = b.obj();
}
LOG(3) << "recording new metadata: " << obj;
StatusWith<RecordId> status =
_rs->updateRecord(opCtx, loc, obj.objdata(), obj.objsize(), false, NULL);
fassert(28521, status.getStatus());
invariant(status.getValue() == loc);
}
void KVCatalog::putMetaData(OperationContext* opCtx,
StringData ns,
BSONCollectionCatalogEntry::MetaData& md) {
RecordId loc;
BSONObj obj = _findEntry(opCtx, ns, &loc);
{
// rebuilt doc
BSONObjBuilder b;
b.append("md", md.toBSON());
BSONObjBuilder newIdentMap;
BSONObj oldIdentMap;
if (obj["idxIdent"].isABSONObj())
oldIdentMap = obj["idxIdent"].Obj();
// fix ident map
for (size_t i = 0; i < md.indexes.size(); i++) {
string name = md.indexes[i].name();
BSONElement e = oldIdentMap[name];
if (e.type() == String) {
newIdentMap.append(e);
continue;
}
// missing, create new
newIdentMap.append(name, _newUniqueIdent(ns, "index"));
}
b.append("idxIdent", newIdentMap.obj());
// add whatever is left
b.appendElementsUnique(obj);
obj = b.obj();
}
LOG(3) << "recording new metadata: " << obj;
Status status = _rs->updateRecord(opCtx, loc, obj.objdata(), obj.objsize(), false, NULL);
fassert(28521, status.isOK());
}
TEST_F(KVCatalogTest, RestartForPrefixes) {
storageGlobalParams.groupCollections = true;
ON_BLOCK_EXIT([&] { storageGlobalParams.groupCollections = false; });
KVPrefix abCollPrefix = KVPrefix::getNextPrefix(NamespaceString("a.b"));
KVPrefix fooIndexPrefix = KVPrefix::getNextPrefix(NamespaceString("a.b"));
unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create());
KVEngine* engine = helper->getEngine();
{
unique_ptr<RecordStore> rs;
unique_ptr<KVCatalog> catalog;
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
ASSERT_OK(engine->createRecordStore(&opCtx, "catalog", "catalog", CollectionOptions()));
rs = engine->getRecordStore(&opCtx, "catalog", "catalog", CollectionOptions());
catalog.reset(new KVCatalog(rs.get(), false, false, nullptr));
uow.commit();
}
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
ASSERT_OK(newCollection(
&opCtx, NamespaceString("a.b"), CollectionOptions(), abCollPrefix, catalog.get()));
ASSERT_NOT_EQUALS("a.b", catalog->getCollectionIdent("a.b"));
ASSERT_TRUE(catalog->isUserDataIdent(catalog->getCollectionIdent("a.b")));
uow.commit();
}
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
BSONCollectionCatalogEntry::MetaData md;
md.ns = "a.b";
BSONCollectionCatalogEntry::IndexMetaData imd;
imd.spec = BSON("name"
<< "foo");
imd.ready = false;
imd.head = RecordId();
imd.multikey = false;
imd.prefix = fooIndexPrefix;
imd.isBackgroundSecondaryBuild = false;
md.indexes.push_back(imd);
md.prefix = abCollPrefix;
catalog->putMetaData(&opCtx, "a.b", md);
uow.commit();
}
}
engine = helper->restartEngine();
{
MyOperationContext opCtx(engine);
WriteUnitOfWork uow(&opCtx);
unique_ptr<RecordStore> rs =
engine->getRecordStore(&opCtx, "catalog", "catalog", CollectionOptions());
unique_ptr<KVCatalog> catalog =
stdx::make_unique<KVCatalog>(rs.get(), false, false, nullptr);
catalog->init(&opCtx);
const BSONCollectionCatalogEntry::MetaData md = catalog->getMetaData(&opCtx, "a.b");
ASSERT_EQ("a.b", md.ns);
ASSERT_EQ(abCollPrefix, md.prefix);
ASSERT_EQ(fooIndexPrefix, md.indexes[md.findIndexOffset("foo")].prefix);
}
}