int Freeze::MapIndexI::untypedCount(const Key& k, const ConnectionIPtr& connection) const { Dbt dbKey; initializeInDbt(k, dbKey); // // When we have a custom-comparison function, Berkeley DB returns // the key on-disk (when it finds one). We disable this behavior: // (ref Oracle SR 5925672.992) // dbKey.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL); Dbt dbValue; dbValue.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL); int result = 0; DbTxn* txn = connection->dbTxn(); try { for(;;) { Dbc* dbc = 0; try { // // Move to the first record // _db->cursor(txn, &dbc, 0); bool found = (dbc->get(&dbKey, &dbValue, DB_SET) == 0); if(found) { db_recno_t count = 0; dbc->count(&count, 0); result = static_cast<int>(count); } Dbc* toClose = dbc; dbc = 0; toClose->close(); break; // for (;;) } catch(const DbDeadlockException&) { if(dbc != 0) { try { dbc->close(); } catch(const DbDeadlockException&) { if(txn != 0) { throw; } else { // // Ignored // } } } if(connection->deadlockWarning()) { Warning out(connection->communicator()->getLogger()); out << "Deadlock in Freeze::MapIndexI::untypedCount while searching \"" << _dbName << "\""; } if(txn != 0) { throw; } // // Otherwise retry // } catch(...) { if(dbc != 0) { try { dbc->close(); } catch(const DbDeadlockException&) { if(txn != 0) { throw; } else { // // Ignored // } } } throw; } } } catch(const DbDeadlockException& dx) { throw DeadlockException(__FILE__, __LINE__, dx.what(), connection->currentTransaction()); } catch(const DbException& dx) { throw DatabaseException(__FILE__, __LINE__, dx.what()); } return result; }
void Freeze::IteratorHelperI::get(const Key*& key, const Value*& value) const { key = &_key; value = &_value; size_t keySize = _key.size(); if(keySize < 1024) { keySize = 1024; } _key.resize(keySize); Dbt dbKey; initializeOutDbt(_key, dbKey); size_t valueSize = _value.size(); if(valueSize < 1024) { valueSize = 1024; } _value.resize(valueSize); Dbt dbValue; initializeOutDbt(_value, dbValue); for(;;) { try { int err; if(_indexed) { // // Not interested in getting the index's key // Dbt iKey; iKey.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL); err = _dbc->pget(&iKey, &dbKey, &dbValue, DB_CURRENT); } else { err = _dbc->get(&dbKey, &dbValue, DB_CURRENT); } if(err == 0) { _key.resize(dbKey.get_size()); _value.resize(dbValue.get_size()); break; // for(;;) } else if(err == DB_KEYEMPTY) { throw InvalidPositionException(__FILE__, __LINE__); } else { // // Bug in Freeze // assert(0); throw DatabaseException(__FILE__, __LINE__); } } catch(const ::DbDeadlockException& dx) { if(_tx != 0) { _tx->dead(); } DeadlockException ex(__FILE__, __LINE__); ex.message = dx.what(); throw ex; } catch(const ::DbException& dx) { handleDbException(dx, _key, dbKey, _value, dbValue, __FILE__, __LINE__); } } }
/*static*/ void Freeze::MapHelper::recreate(const Freeze::ConnectionPtr& connection, const string& dbName, const string& key, const string& value, const Freeze::KeyCompareBasePtr& keyCompare, const std::vector<MapIndexBasePtr>& indices) { Freeze::ConnectionIPtr connectionI = Freeze::ConnectionIPtr::dynamicCast(connection.get()); if(connectionI == 0) { throw DatabaseException(__FILE__, __LINE__, "Invalid connection"); } if(dbName == catalogName() || dbName == catalogIndexListName()) { throw DatabaseException(__FILE__, __LINE__, "You cannot destroy recreate the \"" + dbName + "\" database"); } if(connectionI->trace() >= 1) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Recreating \"" << dbName << "\""; } TransactionPtr tx = connectionI->currentTransaction(); bool ownTx = (tx == 0); Dbt keyDbt; keyDbt.set_flags(DB_DBT_REALLOC); Dbt valueDbt; valueDbt.set_flags(DB_DBT_REALLOC); try { for(;;) { try { if(ownTx) { tx = 0; tx = connectionI->beginTransaction(); } DbTxn* txn = connectionI->dbTxn(); if(connectionI->trace() >= 2) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Removing all existing indices for \"" << dbName << "\""; } CatalogIndexList catalogIndexList(connection, catalogIndexListName()); CatalogIndexList::iterator p = catalogIndexList.find(dbName); if(p != catalogIndexList.end()) { const StringSeq& indices = p->second; for(size_t i = 0; i < indices.size(); ++i) { try { connection->removeMapIndex(dbName, indices[i]); } catch(const IndexNotFoundException&) { // // Ignored // } } catalogIndexList.erase(p); } // // Rename existing database // string oldDbName = dbName + ".old-" + IceUtil::generateUUID(); if(connectionI->trace() >= 2) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Renaming \"" << dbName << "\" to \"" << oldDbName << "\""; } connectionI->dbEnv()->getEnv()->dbrename(txn, dbName.c_str(), 0, oldDbName.c_str(), 0); // // Fortunately, DB closes oldDb automatically when it goes out of scope // Db oldDb(connectionI->dbEnv()->getEnv(), 0); oldDb.open(txn, Ice::nativeToUTF8(connectionI->communicator(), oldDbName).c_str(), 0, DB_BTREE, DB_THREAD, FREEZE_DB_MODE); auto_ptr<MapDb> newDb(new MapDb(connectionI, dbName, key, value, keyCompare, indices, true)); if(connectionI->trace() >= 2) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Writing contents of \"" << oldDbName << "\" to fresh \"" << dbName << "\""; } // // Now simply write all of oldDb into newDb // Dbc* dbc = 0; oldDb.cursor(txn, &dbc, 0); try { while(dbc->get(&keyDbt, &valueDbt, DB_NEXT) == 0) { newDb->put(txn, &keyDbt, &valueDbt, 0); } } catch(...) { dbc->close(); throw; } dbc->close(); if(connectionI->trace() >= 2) { Trace out(connectionI->communicator()->getLogger(), "Freeze.Map"); out << "Transfer complete; removing \"" << oldDbName << "\""; } connectionI->dbEnv()->getEnv()->dbremove(txn, oldDbName.c_str(), 0, 0); if(ownTx) { tx->commit(); } break; // for (;;) } catch(const DbDeadlockException& dx) { if(ownTx) { if(connectionI->deadlockWarning()) { Warning out(connectionI->communicator()->getLogger()); out << "Deadlock in Freeze::MapHelperI::recreate on Db \"" << dbName << "\"; retrying ..."; } // // Ignored, try again // } else { throw DeadlockException(__FILE__, __LINE__, dx.what(), tx); } } catch(const DbException& dx) { if(ownTx) { try { tx->rollback(); } catch(...) { } } throw DatabaseException(__FILE__, __LINE__, dx.what()); } catch(...) { if(ownTx && tx != 0) { try { tx->rollback(); } catch(...) { } } throw; } } free(keyDbt.get_data()); free(valueDbt.get_data()); } catch(...) { free(keyDbt.get_data()); free(valueDbt.get_data()); throw; } }
size_t Freeze::MapHelperI::count(const Key& key) const { Dbt dbKey; initializeInDbt(key, dbKey); // // Keep 0 length since we're not interested in the data // Dbt dbValue; dbValue.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL); for(;;) { try { int err = _db->get(_connection->dbTxn(), &dbKey, &dbValue, 0); if(err == 0) { return 1; } else if(err == DB_NOTFOUND) { return 0; } else { assert(0); throw DatabaseException(__FILE__, __LINE__); } } catch(const ::DbDeadlockException& dx) { if(_connection->dbTxn() != 0) { DeadlockException ex(__FILE__, __LINE__); ex.message = dx.what(); throw ex; } else { if(_connection->deadlockWarning()) { Warning out(_connection->communicator()->getLogger()); out << "Deadlock in Freeze::MapHelperI::count on Map \"" << _dbName << "\"; retrying ..."; } // // Ignored, try again // } } catch(const ::DbException& dx) { DatabaseException ex(__FILE__, __LINE__); ex.message = dx.what(); throw ex; } } }
void AccessExample::run(bool removeExistingDatabase, const char *fileName) { // Remove the previous database. if (removeExistingDatabase) (void)remove(fileName); // Create the database object. // There is no environment for this simple example. Db db(0, 0); db.set_error_stream(&cerr); db.set_errpfx("AccessExample"); db.set_pagesize(1024); /* Page size: 1K. */ db.set_cachesize(0, 32 * 1024, 0); db.open(NULL, fileName, NULL, DB_BTREE, DB_CREATE, 0664); // // Insert records into the database, where the key is the user // input and the data is the user input in reverse order. // char buf[1024], rbuf[1024]; char *p, *t; int ret; u_int32_t len; for (;;) { cout << "input> "; cout.flush(); cin.getline(buf, sizeof(buf)); if (cin.eof()) break; if ((len = (u_int32_t)strlen(buf)) <= 0) continue; for (t = rbuf, p = buf + (len - 1); p >= buf;) *t++ = *p--; *t++ = '\0'; Dbt key(buf, len + 1); Dbt data(rbuf, len + 1); ret = db.put(0, &key, &data, DB_NOOVERWRITE); if (ret == DB_KEYEXIST) { cout << "Key " << buf << " already exists.\n"; } } cout << "\n"; // We put a try block around this section of code // to ensure that our database is properly closed // in the event of an error. // try { // Acquire a cursor for the table. Dbc *dbcp; db.cursor(NULL, &dbcp, 0); // Walk through the table, printing the key/data pairs. Dbt key; Dbt data; while (dbcp->get(&key, &data, DB_NEXT) == 0) { char *key_string = (char *)key.get_data(); char *data_string = (char *)data.get_data(); cout << key_string << " : " << data_string << "\n"; } dbcp->close(); } catch (DbException &dbe) { cerr << "AccessExample: " << dbe.what() << "\n"; } db.close(0); }
// Check that key/data for 0 - count-1 are already present, // and write a key/data for count. The key and data are // both "0123...N" where N == count-1. // // For some reason on Windows, we need to open using the full pathname // of the file when there is no environment, thus the 'has_env' // variable. // void rundb(Db *db, int count, int has_env) { const char *name; if (has_env) name = CONSTRUCT01_DBNAME; else name = CONSTRUCT01_DBFULLPATH; db->set_error_stream(&cerr); // We don't really care about the pagesize, but we do want // to make sure adjusting Db specific variables works before // opening the db. // CHK(db->set_pagesize(1024)); CHK(db->open(NULL, name, NULL, DB_BTREE, count ? 0 : DB_CREATE, 0664)); // The bit map of keys we've seen long bitmap = 0; // The bit map of keys we expect to see long expected = (1 << (count+1)) - 1; char outbuf[10]; int i; for (i=0; i<count; i++) { outbuf[i] = '0' + i; } outbuf[i++] = '\0'; Dbt key(outbuf, i); Dbt data(outbuf, i); DEBUGOUT("Put: " << outbuf); CHK(db->put(0, &key, &data, DB_NOOVERWRITE)); // Acquire a cursor for the table. Dbc *dbcp; CHK(db->cursor(NULL, &dbcp, 0)); // Walk through the table, checking Dbt readkey; Dbt readdata; while (dbcp->get(&readkey, &readdata, DB_NEXT) == 0) { char *key_string = (char *)readkey.get_data(); char *data_string = (char *)readdata.get_data(); DEBUGOUT("Got: " << key_string << ": " << data_string); int len = strlen(key_string); long bit = (1 << len); if (len > count) { ERR("reread length is bad"); } else if (strcmp(data_string, key_string) != 0) { ERR("key/data don't match"); } else if ((bitmap & bit) != 0) { ERR("key already seen"); } else if ((expected & bit) == 0) { ERR("key was not expected"); } else { bitmap |= bit; expected &= ~(bit); for (i=0; i<len; i++) { if (key_string[i] != ('0' + i)) { cout << " got " << key_string << " (" << (int)key_string[i] << ")" << ", wanted " << i << " (" << (int)('0' + i) << ")" << " at position " << i << "\n"; ERR("key is corrupt"); } } } } if (expected != 0) { cout << " expected more keys, bitmap is: " << expected << "\n"; ERR("missing keys in database"); } CHK(dbcp->close()); CHK(db->close(0)); }
void TestKeyRange::run() { // Remove the previous database. (void)unlink(FileName); // Create the database object. // There is no environment for this simple example. Db db(0, 0); db.set_error_stream(&cerr); db.set_errpfx("TestKeyRange"); db.set_pagesize(1024); /* Page size: 1K. */ db.set_cachesize(0, 32 * 1024, 0); db.open(NULL, FileName, NULL, DB_BTREE, DB_CREATE, 0664); // // Insert records into the database, where the key is the user // input and the data is the user input in reverse order. // char buf[1024]; char rbuf[1024]; char *t; char *p; int ret; int len; Dbt *firstkey = NULL; char firstbuf[1024]; for (;;) { cout << "input>"; cout.flush(); cin.getline(buf, sizeof(buf)); if (cin.eof()) break; if ((len = strlen(buf)) <= 0) continue; for (t = rbuf, p = buf + (len - 1); p >= buf;) *t++ = *p--; *t++ = '\0'; Dbt key(buf, len + 1); Dbt data(rbuf, len + 1); if (firstkey == NULL) { strcpy(firstbuf, buf); firstkey = new Dbt(firstbuf, len + 1); } ret = db.put(0, &key, &data, DB_NOOVERWRITE); if (ret == DB_KEYEXIST) { cout << "Key " << buf << " already exists.\n"; } cout << "\n"; } // We put a try block around this section of code // to ensure that our database is properly closed // in the event of an error. // try { // Acquire a cursor for the table. Dbc *dbcp; db.cursor(NULL, &dbcp, 0); /*ADDED...*/ DB_KEY_RANGE range; memset(&range, 0, sizeof(range)); db.key_range(NULL, firstkey, &range, 0); printf("less: %f\n", range.less); printf("equal: %f\n", range.equal); printf("greater: %f\n", range.greater); /*end ADDED*/ Dbt key; Dbt data; // Walk through the table, printing the key/data pairs. while (dbcp->get(&key, &data, DB_NEXT) == 0) { char *key_string = (char *)key.get_data(); char *data_string = (char *)data.get_data(); cout << key_string << " : " << data_string << "\n"; } dbcp->close(); } catch (DbException &dbe) { cerr << "TestKeyRange: " << dbe.what() << "\n"; } db.close(0); }
//############################################################################## //############################################################################## std::string BerkeleyDBCXXDb::get_record(record_type type, const std::string& key) const { RANGE_LOG_TIMED_FUNCTION(); std::string fullkey = key_name(type, key); auto txn = current_txn_.lock(); if(txn) { std::string data; if(txn->get_record(type, key, data)) { return data; } } auto lck = boost::dynamic_pointer_cast<BerkeleyDBCXXLock>(this->read_lock(type, key)); DbTxn * dbtxn = BerkeleyDBCXXLockTxnGetter(lck).txn(); Dbt dbkey { (void*) fullkey.c_str(), (uint32_t) fullkey.size() }; size_t bufsize = 131072; std::unique_ptr<char[]> buf { nullptr }; Dbt dbdata; int dbrval = 0; do { if(buf) { LOG(debug0, "resizing_record_buffer") << bufsize; } buf = std::unique_ptr<char[]>(new char[bufsize]); if(!buf) { std::stringstream s; s << "Unable to allocate buffer of size: " << bufsize; THROW_STACK(DatabaseEnvironmentException(s.str())); } dbdata = Dbt(buf.get(), bufsize); dbdata.set_ulen(bufsize); dbdata.set_flags(DB_DBT_USERMEM); bufsize *= 2; int flags = lck->readonly() ? 0 : DB_RMW; try { dbrval = inst_->get(dbtxn, &dbkey, &dbdata, flags); } catch (DbException &e) { if(e.get_errno() == DB_BUFFER_SMALL) { continue; } THROW_STACK(DatabaseEnvironmentException(std::string("Unable to read record") + e.what())); } catch (std::exception &e) { THROW_STACK(DatabaseEnvironmentException(std::string("Unable to read record") + e.what())); } } while(dbrval == DB_BUFFER_SMALL); switch(dbrval) { case 0: break; case DB_NOTFOUND: return std::string(); case DB_BUFFER_SMALL: THROW_STACK(DatabaseEnvironmentException("The requested item could not be returned due to undersized buffer.")); break; case DB_LOCK_DEADLOCK: THROW_STACK(DatabaseEnvironmentException("A transactional database environment operation was selected to resolve a deadlock.")); break; case DB_LOCK_NOTGRANTED: THROW_STACK(DatabaseEnvironmentException("unable to grant a lock in the allowed time.")); break; case DB_REP_HANDLE_DEAD: THROW_STACK(DatabaseEnvironmentException("Dead handle")); break; default: LOG(error, "unknown dbrval") << dbrval; } std::string rval { (char *) dbdata.get_data(), dbdata.get_size() }; return rval; }
int main(int argc, char *argv[]) { try { Db *db = new Db(NULL, 0); db->open(NULL, "my.db", NULL, DB_BTREE, DB_CREATE, 0644); // populate our massive database. // all our strings include null for convenience. // Note we have to cast for idiomatic // usage, since newer gcc requires it. Dbt *keydbt = new Dbt((char*)"key", 4); Dbt *datadbt = new Dbt((char*)"data", 5); db->put(NULL, keydbt, datadbt, 0); // Now, retrieve. We could use keydbt over again, // but that wouldn't be typical in an application. Dbt *goodkeydbt = new Dbt((char*)"key", 4); Dbt *badkeydbt = new Dbt((char*)"badkey", 7); Dbt *resultdbt = new Dbt(); resultdbt->set_flags(DB_DBT_MALLOC); int ret; if ((ret = db->get(NULL, goodkeydbt, resultdbt, 0)) != 0) { cout << "get: " << DbEnv::strerror(ret) << "\n"; } else { char *result = (char *)resultdbt->get_data(); cout << "got data: " << result << "\n"; } if ((ret = db->get(NULL, badkeydbt, resultdbt, 0)) != 0) { // We expect this... cout << "get using bad key: " << DbEnv::strerror(ret) << "\n"; } else { char *result = (char *)resultdbt->get_data(); cout << "*** got data using bad key!!: " << result << "\n"; } // Now, truncate and make sure that it's really gone. cout << "truncating data...\n"; u_int32_t nrecords; db->truncate(NULL, &nrecords, 0); cout << "truncate returns " << nrecords << "\n"; if ((ret = db->get(NULL, goodkeydbt, resultdbt, 0)) != 0) { // We expect this... cout << "after truncate get: " << DbEnv::strerror(ret) << "\n"; } else { char *result = (char *)resultdbt->get_data(); cout << "got data: " << result << "\n"; } db->close(0); cout << "finished test\n"; } catch (DbException &dbe) { cerr << "Db Exception: " << dbe.what(); } return 0; }
void BtRecExample::run() { db_recno_t recno; int ret; char buf[1024]; // Acquire a cursor for the database. dbp->cursor(NULL, &dbcp, 0); // // Prompt the user for a record number, then retrieve and display // that record. // for (;;) { // Get a record number. cout << "recno #> "; cout.flush(); if (fgets(buf, sizeof(buf), stdin) == NULL) break; recno = atoi(buf); // // Start with a fresh key each time, // the dbp->get() routine returns // the key and data pair, not just the key! // Dbt key(&recno, sizeof(recno)); Dbt data; if ((ret = dbcp->get(&key, &data, DB_SET_RECNO)) != 0) { dbp->err(ret, "DBcursor->get"); throw DbException(ret); } // Display the key and data. show("k/d\t", &key, &data); // Move the cursor a record forward. if ((ret = dbcp->get(&key, &data, DB_NEXT)) != 0) { dbp->err(ret, "DBcursor->get"); throw DbException(ret); } // Display the key and data. show("next\t", &key, &data); // // Retrieve the record number for the following record into // local memory. // data.set_data(&recno); data.set_size(sizeof(recno)); data.set_ulen(sizeof(recno)); data.set_flags(data.get_flags() | DB_DBT_USERMEM); if ((ret = dbcp->get(&key, &data, DB_GET_RECNO)) != 0) { if (ret != DB_NOTFOUND && ret != DB_KEYEMPTY) { dbp->err(ret, "DBcursor->get"); throw DbException(ret); } } else { cout << "retrieved recno: " << (u_long)recno << "\n"; } } dbcp->close(); dbcp = NULL; }
vector<Identity>::const_iterator Freeze::EvictorIteratorI::nextBatch() { DeactivateController::Guard deactivateGuard(_store->evictor()->deactivateController()); _batch.clear(); if(!_more) { return _batch.end(); } vector<EvictorElementPtr> evictorElements; evictorElements.reserve(_batchSize); Key firstKey = _key; CommunicatorPtr communicator = _store->communicator(); try { for(;;) { _batch.clear(); evictorElements.clear(); Dbt dbKey; initializeOutDbt(_key, dbKey); Dbt dbValue; dbValue.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL); Dbc* dbc = 0; try { // // Move to the first record // u_int32_t flags = DB_NEXT; if(_initialized) { // // _key represents the next element not yet returned // if it has been deleted, we want the one after // flags = DB_SET_RANGE; // // Will be used as input as well // dbKey.set_size(static_cast<u_int32_t>(firstKey.size())); } _store->db()->cursor(0, &dbc, 0); bool done = false; do { for(;;) { try { // // It is critical to set key size to key capacity before the // get, as a resize that increases the size inserts 0 // _key.resize(_key.capacity()); _more = (dbc->get(&dbKey, &dbValue, flags) == 0); if(_more) { _key.resize(dbKey.get_size()); _initialized = true; flags = DB_NEXT; Ice::Identity ident; ObjectStore::unmarshal(ident, _key, communicator); if(_batch.size() < _batchSize) { _batch.push_back(ident); } else { // // Keep the last element in _key // done = true; } } break; } catch(const DbDeadlockException&) { throw; } catch(const DbException& dx) { handleDbException(dx, _key, dbKey, __FILE__, __LINE__); } } } while(!done && _more); Dbc* toClose = dbc; dbc = 0; toClose->close(); break; // for (;;) } catch(const DbDeadlockException&) { if(dbc != 0) { try { dbc->close(); } catch(const DbDeadlockException&) { // // Ignored // } } _key = firstKey; // // Retry // } catch(...) { if(dbc != 0) { try { dbc->close(); } catch(const DbDeadlockException&) { // // Ignored // } } throw; } } } catch(const DbException& dx) { handleDbException(dx, __FILE__, __LINE__); } if(_batch.size() == 0) { return _batch.end(); } else { return _batch.begin(); } }