Freeze::MapDb::MapDb(const ConnectionIPtr& connection, const string& dbName, const string& key, const string& value, const KeyCompareBasePtr& keyCompare, const vector<MapIndexBasePtr>& indices, bool createDb) : Db(connection->dbEnv()->getEnv(), 0), _communicator(connection->communicator()), _dbName(dbName), _trace(connection->trace()), _keyCompare(keyCompare) { if(_trace >= 1) { Trace out(_communicator->getLogger(), "Freeze.Map"); out << "opening Db \"" << _dbName << "\""; } Catalog catalog(connection, _catalogName); TransactionPtr tx = connection->currentTransaction(); bool ownTx = (tx == 0); for(;;) { try { if(ownTx) { tx = 0; tx = connection->beginTransaction(); } Catalog::iterator ci = catalog.find(_dbName); if(ci != catalog.end()) { if(ci->second.evictor) { throw DatabaseException(__FILE__, __LINE__, _dbName + " is an evictor database"); } _key = ci->second.key; _value = ci->second.value; checkTypes(key, value); } else { _key = key; _value = value; } set_app_private(this); if(_keyCompare->compareEnabled()) { set_bt_compare(&customCompare); } PropertiesPtr properties = _communicator->getProperties(); string propPrefix = "Freeze.Map." + _dbName + "."; int btreeMinKey = properties->getPropertyAsInt(propPrefix + "BtreeMinKey"); if(btreeMinKey > 2) { if(_trace >= 1) { Trace out(_communicator->getLogger(), "Freeze.Map"); out << "Setting \"" << _dbName << "\"'s btree minkey to " << btreeMinKey; } set_bt_minkey(btreeMinKey); } bool checksum = properties->getPropertyAsInt(propPrefix + "Checksum") > 0; if(checksum) { if(_trace >= 1) { Trace out(_communicator->getLogger(), "Freeze.Map"); out << "Turning checksum on for \"" << _dbName << "\""; } set_flags(DB_CHKSUM); } int pageSize = properties->getPropertyAsInt(propPrefix + "PageSize"); if(pageSize > 0) { if(_trace >= 1) { Trace out(_communicator->getLogger(), "Freeze.Map"); out << "Setting \"" << _dbName << "\"'s pagesize to " << pageSize; } set_pagesize(pageSize); } DbTxn* txn = getTxn(tx); u_int32_t flags = DB_THREAD; if(createDb) { flags |= DB_CREATE; } open(txn, Ice::nativeToUTF8(_communicator, _dbName).c_str(), 0, DB_BTREE, flags, FREEZE_DB_MODE); StringSeq oldIndices; StringSeq newIndices; size_t oldSize = 0; CatalogIndexList catalogIndexList(connection, _catalogIndexListName); if(createDb) { CatalogIndexList::iterator cil = catalogIndexList.find(_dbName); if(cil != catalogIndexList.end()) { oldIndices = cil->second; oldSize = oldIndices.size(); } } for(vector<MapIndexBasePtr>::const_iterator p = indices.begin(); p != indices.end(); ++p) { const MapIndexBasePtr& indexBase = *p; assert(indexBase->_impl == 0); assert(indexBase->_communicator == 0); indexBase->_communicator = connection->communicator(); auto_ptr<MapIndexI> indexI; try { indexI.reset(new MapIndexI(connection, *this, txn, createDb, indexBase)); } catch(const DbDeadlockException&) { throw; } catch(const DbException& dx) { string message = "Error while opening index \"" + _dbName + "." + indexBase->name() + "\": " + dx.what(); throw DatabaseException(__FILE__, __LINE__, message); } #ifndef NDEBUG bool inserted = #endif _indices.insert(IndexMap::value_type(indexBase->name(), indexI.get())).second; assert(inserted); indexBase->_impl = indexI.release(); if(createDb) { newIndices.push_back(indexBase->name()); oldIndices.erase(std::remove(oldIndices.begin(), oldIndices.end(), indexBase->name()), oldIndices.end()); } } if(ci == catalog.end()) { CatalogData catalogData; catalogData.evictor = false; catalogData.key = key; catalogData.value = value; catalog.put(Catalog::value_type(_dbName, catalogData)); } if(createDb) { // // Remove old indices and write the new ones // bool indexRemoved = false; for(StringSeq::const_iterator q = oldIndices.begin(); q != oldIndices.end(); ++q) { const string& index = *q; if(_trace >= 1) { Trace out(_communicator->getLogger(), "Freeze.Map"); out << "removing old index \"" << index << "\" on Db \"" << _dbName << "\""; } try { connection->removeMapIndex(_dbName, *q); indexRemoved = true; } catch(const IndexNotFoundException&) { // Ignored if(_trace >= 1) { Trace out(_communicator->getLogger(), "Freeze.Map"); out << "index \"" << index << "\" on Db \"" << _dbName << "\" does not exist"; } } } if(indexRemoved || oldSize != newIndices.size()) { if(newIndices.size() == 0) { catalogIndexList.erase(_dbName); if(_trace >= 1) { Trace out(_communicator->getLogger(), "Freeze.Map"); out << "Removed catalogIndexList entry for Db \"" << _dbName << "\""; } } else { catalogIndexList.put(CatalogIndexList::value_type(_dbName, newIndices)); if(_trace >= 1) { Trace out(_communicator->getLogger(), "Freeze.Map"); out << "Updated catalogIndexList entry for Db \"" << _dbName << "\""; } } } } if(ownTx) { tx->commit(); } break; // for(;;) } catch(const DbDeadlockException& dx) { if(ownTx) { if(connection->deadlockWarning()) { Warning out(connection->communicator()->getLogger()); out << "Deadlock in Freeze::MapDb::MapDb on Map \"" << _dbName << "\"; retrying ..."; } // // Ignored, try again // } else { throw DeadlockException(__FILE__, __LINE__, dx.what(), tx); } } catch(const DbException& dx) { if(ownTx) { try { tx->rollback(); } catch(...) { } } string message = "Error while opening Db \"" + _dbName + "\": " + dx.what(); throw DatabaseException(__FILE__, __LINE__, message); } catch(...) { if(ownTx && tx != 0) { try { tx->rollback(); } catch(...) { } } throw; } } }
Freeze::ObjectStoreBase::ObjectStoreBase(const string& facet, const string& facetType, bool createDb, EvictorIBase* evictor, const vector<IndexPtr>& indices, bool populateEmptyIndices) : _facet(facet), _evictor(evictor), _indices(indices), _communicator(evictor->communicator()), _encoding(evictor->encoding()), _keepStats(false) { if(facet == "") { _dbName = EvictorIBase::defaultDb; } else { _dbName = facet; } if(!facetType.empty()) { // // Create a sample servant with this type // ValueFactoryPtr factory = _communicator->getValueFactoryManager()->find(facetType); if(factory == 0) { throw DatabaseException(__FILE__, __LINE__, "No object factory registered for type-id '" + facetType + "'"); } _sampleServant = factory->create(facetType); } ConnectionPtr catalogConnection = createConnection(_communicator, evictor->dbEnv()->getEnvName()); Catalog catalog(catalogConnection, catalogName()); Catalog::iterator p = catalog.find(evictor->filename()); if(p != catalog.end()) { if(p->second.evictor) { // // empty means the value is ::Freeze::ObjectRecord // _keepStats = p->second.value.empty(); } else { DatabaseException ex(__FILE__, __LINE__); ex.message = evictor->filename() + " is not an evictor database"; throw ex; } } DbEnv* dbEnv = evictor->dbEnv()->getEnv(); try { _db.reset(new Db(dbEnv, 0)); Ice::PropertiesPtr properties = evictor->communicator()->getProperties(); string propPrefix = "Freeze.Evictor." + evictor->filename() + "."; int btreeMinKey = properties->getPropertyAsInt(propPrefix + _dbName + ".BtreeMinKey"); if(btreeMinKey > 2) { if(evictor->trace() >= 1) { Trace out(evictor->communicator()->getLogger(), "Freeze.Evictor"); out << "Setting \"" << evictor->filename() + "." + _dbName << "\"'s btree minkey to " << btreeMinKey; } _db->set_bt_minkey(btreeMinKey); } bool checksum = properties->getPropertyAsInt(propPrefix + "Checksum") > 0; if(checksum) { if(evictor->trace() >= 1) { Trace out(evictor->communicator()->getLogger(), "Freeze.Evictor"); out << "Turning checksum on for \"" << evictor->filename() << "\""; } _db->set_flags(DB_CHKSUM); } int pageSize = properties->getPropertyAsInt(propPrefix + "PageSize"); if(pageSize > 0) { if(evictor->trace() >= 1) { Trace out(evictor->communicator()->getLogger(), "Freeze.Evictor"); out << "Setting \"" << evictor->filename() << "\"'s pagesize to " << pageSize; } _db->set_pagesize(pageSize); } TransactionPtr tx = catalogConnection->beginTransaction(); DbTxn* txn = getTxn(tx); u_int32_t flags = DB_THREAD; if(createDb) { flags |= DB_CREATE; } // // Berkeley DB expects file paths to be UTF8 encoded. We keep // _dbName as a native string here, while it might have // been better to convert it to UTF-8, changing this isn't // possible without potentially breaking backward compatibility // with deployed databases. // _db->open(txn, nativeToUTF8(evictor->filename(), getProcessStringConverter()).c_str(), _dbName.c_str(), DB_BTREE, flags, FREEZE_DB_MODE); for(size_t i = 0; i < _indices.size(); ++i) { _indices[i]->_impl->associate(this, txn, createDb, populateEmptyIndices); } if(p == catalog.end()) { CatalogData catalogData; catalogData.evictor = true; catalogData.key = "Ice::Identity"; catalogData.value = "Object"; catalog.put(Catalog::value_type(evictor->filename(), catalogData)); } tx->commit(); } catch(const DbException& dx) { TransactionPtr tx = catalogConnection->currentTransaction(); if(tx != 0) { try { tx->rollback(); } catch(...) { } } if(dx.get_errno() == ENOENT) { NotFoundException ex(__FILE__, __LINE__); ex.message = dx.what(); throw ex; } else { DatabaseException ex(__FILE__, __LINE__); ex.message = dx.what(); throw ex; } } catch(...) { TransactionPtr tx = catalogConnection->currentTransaction(); if(tx != 0) { try { tx->rollback(); } catch(...) { } } throw; } }
/*static*/ void Freeze::MapHelper::recreate(const Freeze::ConnectionPtr& connection, const string& dbName, const string& key, const string& value, const Freeze::KeyCompareBasePtr& keyCompare, const std::vector<MapIndexBasePtr>& indices) { Freeze::ConnectionIPtr connectionI = Freeze::ConnectionIPtr::dynamicCast(connection.get()); if(connectionI == 0) { throw DatabaseException(__FILE__, __LINE__, "Invalid connection"); } if(dbName == catalogName() || dbName == catalogIndexListName()) { throw DatabaseException(__FILE__, __LINE__, "You cannot destroy recreate the \"" + dbName + "\" database"); } if(connectionI->trace() >= 1) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Recreating \"" << dbName << "\""; } TransactionPtr tx = connectionI->currentTransaction(); bool ownTx = (tx == 0); Dbt keyDbt; keyDbt.set_flags(DB_DBT_REALLOC); Dbt valueDbt; valueDbt.set_flags(DB_DBT_REALLOC); try { for(;;) { try { if(ownTx) { tx = 0; tx = connectionI->beginTransaction(); } DbTxn* txn = connectionI->dbTxn(); if(connectionI->trace() >= 2) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Removing all existing indices for \"" << dbName << "\""; } CatalogIndexList catalogIndexList(connection, catalogIndexListName()); CatalogIndexList::iterator p = catalogIndexList.find(dbName); if(p != catalogIndexList.end()) { const StringSeq& idxs = p->second; for(size_t i = 0; i < idxs.size(); ++i) { try { connection->removeMapIndex(dbName, idxs[i]); } catch(const IndexNotFoundException&) { // // Ignored // } } catalogIndexList.erase(p); } // // Rename existing database // string oldDbName = dbName + ".old-" + generateUUID(); if(connectionI->trace() >= 2) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Renaming \"" << dbName << "\" to \"" << oldDbName << "\""; } connectionI->dbEnv()->getEnv()->dbrename(txn, dbName.c_str(), 0, oldDbName.c_str(), 0); // // Fortunately, DB closes oldDb automatically when it goes out of scope // Db oldDb(connectionI->dbEnv()->getEnv(), 0); // // Berkeley DB expects file paths to be UTF8 encoded. // oldDb.open(txn, nativeToUTF8(oldDbName, getProcessStringConverter()).c_str(), 0, DB_BTREE, DB_THREAD, FREEZE_DB_MODE); IceInternal::UniquePtr<MapDb> newDb(new MapDb(connectionI, dbName, key, value, keyCompare, indices, true)); if(connectionI->trace() >= 2) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Writing contents of \"" << oldDbName << "\" to fresh \"" << dbName << "\""; } // // Now simply write all of oldDb into newDb // Dbc* dbc = 0; oldDb.cursor(txn, &dbc, 0); try { while(dbc->get(&keyDbt, &valueDbt, DB_NEXT) == 0) { newDb->put(txn, &keyDbt, &valueDbt, 0); } } catch(...) { dbc->close(); throw; } dbc->close(); if(connectionI->trace() >= 2) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Transfer complete; removing \"" << oldDbName << "\""; } connectionI->dbEnv()->getEnv()->dbremove(txn, oldDbName.c_str(), 0, 0); if(ownTx) { tx->commit(); } break; // for (;;) } catch(const DbDeadlockException& dx) { if(ownTx) { if(connectionI->deadlockWarning()) { Warning out(connectionI->communicator()->getLogger()); out << "Deadlock in Freeze::MapHelperI::recreate on Db \"" << dbName << "\"; retrying ..."; } // // Ignored, try again // } else { throw DeadlockException(__FILE__, __LINE__, dx.what(), tx); } } catch(const DbException& dx) { if(ownTx) { try { tx->rollback(); } catch(...) { } } throw DatabaseException(__FILE__, __LINE__, dx.what()); } catch(...) { if(ownTx && tx != 0) { try { tx->rollback(); } catch(...) { } } throw; } } free(keyDbt.get_data()); free(valueDbt.get_data()); } catch(...) { free(keyDbt.get_data()); free(valueDbt.get_data()); throw; } }