void CDBEnv::CloseDb(const string& strFile) { { LOCK(cs_db); if (mapDb[strFile] != NULL) { // Close the database handle Db* pdb = mapDb[strFile]; pdb->close(0); delete pdb; mapDb[strFile] = NULL; } } }
int main(int argc, const char* argv[]) { if (argc !=3) { cout << "usage: "<< argv[0] << " <filename> <db filename>" << endl; return 1; } const char* kDatabaseName = argv[2]; const char* filename = argv[1]; DbEnv env(0); Db* pdb; try { env.set_error_stream(&cerr); env.open("./", DB_CREATE | DB_INIT_MPOOL, 0); pdb = new Db(&env, 0); pdb->set_flags(DB_DUP); // Create (or clear its content if it exists) the database pdb->open(NULL, kDatabaseName, NULL, DB_BTREE, DB_CREATE | DB_TRUNCATE, 0); readfile(pdb, filename); readfile(pdb, "tables/jp.txt"); // Clean up if (pdb != NULL) { pdb->close(0); delete pdb; } env.close(0); } catch (DbException& e) { cerr << "DbException: " << e.what() << endl; return -1; } catch (std::exception& e) { cerr << e.what() << endl; return -1; } return 0; }
// Note that any of the db calls can throw DbException void db_setup(const char *home, const char *data_dir, ostream& err_stream) { // // Create an environment object and initialize it for error // reporting. // DbEnv *dbenv = new DbEnv(0); dbenv->set_error_stream(&err_stream); dbenv->set_errpfx(progname); // // We want to specify the shared memory buffer pool cachesize, // but everything else is the default. // dbenv->set_cachesize(0, 64 * 1024, 0); // Databases are in a subdirectory. (void)dbenv->set_data_dir(data_dir); // Open the environment with full transactional support. dbenv->open(home, DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN, 0); // Open a database in the environment to verify the data_dir // has been set correctly. // Create a database handle, using the environment. Db *db = new Db(dbenv, 0) ; // Open the database. db->open(NULL, "EvnExample_db1.db", NULL, DB_BTREE, DB_CREATE, 0644); // Close the database handle. db->close(0) ; delete db; // Close the handle. dbenv->close(0); delete dbenv; }
string BDBBackend::admin (const string & op, const string & data) { string ret = ThrudocBackend::admin (op, data); if (!ret.empty ()) { return ret; } else if (op == "create_bucket") { Db * db = NULL; try { db = get_db (data); // this will log an error message if db doesn't exist, ignore it } catch (ThrudocException e) {} if (!db) { T_INFO ("admin: creating db=%s", data.c_str()); u_int32_t db_flags = DB_CREATE | // allow creating db DB_AUTO_COMMIT; // allow auto-commit db = new Db (this->db_env, 0); db->open (NULL, // Txn pointer data.c_str (), // file name NULL, // logical db name DB_BTREE, // database type db_flags, // open flags 0); // file mode, defaults db->close (0); delete db; } return "done"; } // TODO delete_bucket, but have to figure out how to close the db // handles across all of the threads first... return ""; }
int main(void) { // Initialize our handles Db *dbp = NULL; DbEnv *envp = NULL; thread_t writerThreads[NUMWRITERS]; int i; u_int32_t envFlags; // Application name const char *progName = "TxnGuideInMemory"; // Env open flags envFlags = DB_CREATE | // Create the environment if it does not exist DB_RECOVER | // Run normal recovery. DB_INIT_LOCK | // Initialize the locking subsystem DB_INIT_LOG | // Initialize the logging subsystem DB_INIT_TXN | // Initialize the transactional subsystem. This // also turns on logging. DB_INIT_MPOOL | // Initialize the memory pool (in-memory cache) DB_PRIVATE | // Region files are not backed by the filesystem. // Instead, they are backed by heap memory. DB_THREAD; // Cause the environment to be free-threaded try { // Create the environment envp = new DbEnv(0); // Specify in-memory logging envp->log_set_config(DB_LOG_IN_MEMORY, 1); // Specify the size of the in-memory log buffer. envp->set_lg_bsize(10 * 1024 * 1024); // Specify the size of the in-memory cache envp->set_cachesize(0, 10 * 1024 * 1024, 1); // Indicate that we want db to internally perform deadlock // detection. Also indicate that the transaction with // the fewest number of write locks will receive the // deadlock notification in the event of a deadlock. envp->set_lk_detect(DB_LOCK_MINWRITE); // Open the environment envp->open(NULL, envFlags, 0); // If we had utility threads (for running checkpoints or // deadlock detection, for example) we would spawn those // here. However, for a simple example such as this, // that is not required. // Open the database openDb(&dbp, progName, NULL, envp, DB_DUPSORT); // Initialize a mutex. Used to help provide thread ids. (void)mutex_init(&thread_num_lock, NULL); // Start the writer threads. for (i = 0; i < NUMWRITERS; i++) (void)thread_create( &writerThreads[i], NULL, writerThread, (void *)dbp); // Join the writers for (i = 0; i < NUMWRITERS; i++) (void)thread_join(writerThreads[i], NULL); } catch(DbException &e) { std::cerr << "Error opening database environment: " << std::endl; std::cerr << e.what() << std::endl; return (EXIT_FAILURE); } try { // Close our database handle if it was opened. if (dbp != NULL) dbp->close(0); // Close our environment if it was opened. if (envp != NULL) envp->close(0); } catch(DbException &e) { std::cerr << "Error closing database and environment." << std::endl; std::cerr << e.what() << std::endl; return (EXIT_FAILURE); } // Final status message and return. std::cout << "I'm all done." << std::endl; return (EXIT_SUCCESS); }
int main() { u_int32_t env_flags = DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN; const char* home = "envHome"; u_int32_t db_flags = DB_CREATE | DB_AUTO_COMMIT; const char* fileName = "envtest.db"; Db* dbp = NULL; DbEnv myEnv(0); try{ myEnv.open(home,env_flags,0); myEnv.set_flags(DB_MULTIVERSION,1); dbp = new Db(&myEnv,0); dbp->open( NULL, //Txn pointer fileName, //File name NULL, //Logic db name DB_BTREE, //Database type db_flags, //Open flags 0 //file mode ); }catch(DbException &e){ std::cerr<<"Error when opening database and Environment:" <<fileName<<","<<home<<std::endl; std::cerr<<e.what()<<std::endl; } //put data normally char *key1 = "luffy"; char *data1 = "op"; char *key2= "usopp"; char *data2 = "brave"; Dbt pkey1(key1,strlen(key1)+1); Dbt pdata1(data1,strlen(data1)+1); Dbt pkey2(key2,strlen(key2)+1); Dbt pdata2(data2,strlen(data2)+1); dbp->put(NULL,&pkey1,&pdata1,0); dbp->put(NULL,&pkey2,&pdata2,0); //using txn cursor to read and another cursor to modify before commit try{ DbTxn *txn1 = NULL; myEnv.txn_begin(NULL,&txn1,DB_TXN_SNAPSHOT); Dbc *cursorp = NULL; dbp->cursor(txn1,&cursorp,0); Dbt tempData1,tempKey2,tempData2; tempData2.set_flags(DB_DBT_MALLOC); cursorp->get(&pkey1,&tempData1,DB_SET); cursorp->get(&tempKey2,&tempData2,DB_NEXT); //cout just to see if it is right std::cout<<(char*)pkey1.get_data()<<" : "<<(char*)tempData1.get_data()<<std::endl <<(char*)tempKey2.get_data()<<" : "<<(char*)tempData2.get_data()<<std::endl; //we can ignore the lock when operation in //the same transaction dbp->put(NULL,&pkey1,&pdata2,0); dbp->put(NULL,&pkey2,&pdata1,0); //will not block even cursor still on this page //close cursor cursorp->close(); //commit the txn txn1->commit(0); }catch(DbException &e){ std::cerr<<"Error when use a txn"<<std::endl; } try{ dbp->close(0); //dbp should close before environment myEnv.close(0); }catch(DbException &e){ std::cerr<<"Error when closing database and environment:" <<fileName<<","<<home<<std::endl; std::cerr<<e.what()<<std::endl; } return 0; }
int main(int argc, char *argv[]) { // Initialize our handles Db *dbp = NULL; DbEnv *envp = NULL; thread_t writerThreads[NUMWRITERS]; int ch, i; u_int32_t envFlags; char *dbHomeDir; extern char *optarg; // Application name const char *progName = "TxnGuide"; // Database file name const char *fileName = "mydb.db"; // Parse the command line arguments #ifdef _WIN32 dbHomeDir = ".\\"; #else dbHomeDir = (char *)"./"; #endif while ((ch = getopt(argc, argv, "h:")) != EOF) switch (ch) { case 'h': dbHomeDir = optarg; break; case '?': default: return (usage()); } // Env open flags envFlags = DB_CREATE | // Create the environment if it does not exist DB_RECOVER | // Run normal recovery. DB_INIT_LOCK | // Initialize the locking subsystem DB_INIT_LOG | // Initialize the logging subsystem DB_INIT_TXN | // Initialize the transactional subsystem. This // also turns on logging. DB_INIT_MPOOL | // Initialize the memory pool (in-memory cache) DB_THREAD; // Cause the environment to be free-threaded try { // Create and open the environment envp = new DbEnv(0); // Indicate that we want db to internally perform deadlock // detection. Also indicate that the transaction with // the fewest number of write locks will receive the // deadlock notification in the event of a deadlock. envp->set_lk_detect(DB_LOCK_MINWRITE); envp->open(dbHomeDir, envFlags, 0); // If we had utility threads (for running checkpoints or // deadlock detection, for example) we would spawn those // here. However, for a simple example such as this, // that is not required. // Open the database openDb(&dbp, progName, fileName, envp, DB_DUPSORT); // Initialize a mutex. Used to help provide thread ids. (void)mutex_init(&thread_num_lock, NULL); // Start the writer threads. for (i = 0; i < NUMWRITERS; i++) (void)thread_create( &writerThreads[i], NULL, writerThread, (void *)dbp); // Join the writers for (i = 0; i < NUMWRITERS; i++) (void)thread_join(writerThreads[i], NULL); } catch(DbException &e) { std::cerr << "Error opening database environment: " << dbHomeDir << std::endl; std::cerr << e.what() << std::endl; return (EXIT_FAILURE); } try { // Close our database handle if it was opened. if (dbp != NULL) dbp->close(0); // Close our environment if it was opened. if (envp != NULL) envp->close(0); } catch(DbException &e) { std::cerr << "Error closing database and environment." << std::endl; std::cerr << e.what() << std::endl; return (EXIT_FAILURE); } // Final status message and return. std::cout << "I'm all done." << std::endl; return (EXIT_SUCCESS); }
~Impl() { mDB.close(0); }
// Note that any of the db calls can throw DbException void db_setup(const char *home, const char *data_dir, ostream& err_stream) { const char * err1 = "DbEnv::open: No such file or directory"; const char * err2 = "Db::open: No such file or directory"; // // Create an environment object and initialize it for error // reporting. // DbEnv *dbenv = new DbEnv(0); dbenv->set_error_stream(&err_stream); dbenv->set_errpfx(progname); // // We want to specify the shared memory buffer pool cachesize, // but everything else is the default. // dbenv->set_cachesize(0, 64 * 1024, 0); // Databases are in a subdirectory. (void)dbenv->set_data_dir(data_dir); // Open the environment with full transactional support. try { dbenv->open(home, DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN, 0); } catch (DbException &dbe) { cerr << "EnvExample: " << dbe.what() << "\n"; if (!strcmp(dbe.what(), err1)){ cout << "Please check whether " << "home dir \"" << home << "\" exists.\n"; } exit (-1); } // Open a database in the environment to verify the data_dir // has been set correctly. // Create a database handle, using the environment. Db *db = new Db(dbenv, 0) ; // Open the database. try { db->open(NULL, "EvnExample_db1.db", NULL, DB_BTREE, DB_CREATE, 0644); } catch (DbException &dbe) { cerr << "EnvExample: " << dbe.what() << "\n"; if (!strcmp(dbe.what(), err2)){ cout << "Please check whether data dir \"" << data_dir << "\" exists under \"" << home << "\"\n"; } exit (-1); } // Close the database handle. db->close(0) ; delete db; // Close the handle. dbenv->close(0); delete dbenv; }
int main(int argc, const char* argv[]) { if (argc !=3) { cout << "usage: "<< argv[0] << " <filename> <db filename>" << endl; return 1; } const char* kDatabaseName = argv[2]; const char* filename = argv[1]; DbEnv env(0); Db* pdb; string line; try { env.set_error_stream(&cerr); env.open("./", DB_CREATE | DB_INIT_MPOOL, 0); pdb = new Db(&env, 0); // If you want to support duplicated records and make duplicated // records sorted by data, you need to call: // pdb->set_flags(DB_DUPSORT); // Note that only Btree-typed database supports sorted duplicated // records // If the database does not exist, create it. If it exists, clear // its content after openning. //pdb->set_flags( DB_DUP );// | DB_DUPSORT); pdb->open(NULL, kDatabaseName, NULL, DB_BTREE, DB_CREATE | DB_TRUNCATE, 0); ifstream myfile (filename); if (myfile.is_open()) { while ( myfile.good() ) { getline (myfile,line); // Ignore empty and commented lines if ((line.length() == 0) || (startswith(line, std::string("#")))) { continue; } std::vector<std::string> cj = split(line, ' '); int freq = atoi(cj[1].c_str()); Dbt key(const_cast<char*>(cj[0].data()), cj[0].size()); Dbt value(&freq, sizeof(int)); pdb->put(NULL, &key, &value, 0); } myfile.close(); } else { cout << "Unable to open file"; } // You need to set ulen and flags=DB_DBT_USERMEM to prevent Dbt // from allocate its own memory but use the memory provided by you. if (pdb != NULL) { pdb->close(0); delete pdb; } env.close(0); } catch (DbException& e) { cerr << "DbException: " << e.what() << endl; return -1; } catch (std::exception& e) { cerr << e.what() << endl; return -1; } return 0; }
int RepMgr::doloop() { Db *dbp; Dbt key, data; char buf[BUFSIZE], *rbuf; int ret; dbp = NULL; memset(&key, 0, sizeof(key)); memset(&data, 0, sizeof(data)); ret = 0; for (;;) { if (dbp == NULL) { dbp = new Db(&dbenv, 0); try { dbp->open(NULL, DATABASE, NULL, DB_BTREE, DB_CREATE | DB_AUTO_COMMIT, 0); } catch(DbException dbe) { dbenv.err(ret, "DB->open"); throw dbe; } } cout << "QUOTESERVER" ; cout << "> " << flush; if (fgets(buf, sizeof(buf), stdin) == NULL) break; if (strtok(&buf[0], " \t\n") == NULL) { switch ((ret = print_stocks(dbp))) { case 0: continue; default: dbp->err(ret, "Error traversing data"); goto err; } } rbuf = strtok(NULL, " \t\n"); if (rbuf == NULL || rbuf[0] == '\0') { if (strncmp(buf, "exit", 4) == 0 || strncmp(buf, "quit", 4) == 0) break; dbenv.errx("Format: TICKER VALUE"); continue; } key.set_data(buf); key.set_size((u_int32_t)strlen(buf)); data.set_data(rbuf); data.set_size((u_int32_t)strlen(rbuf)); if ((ret = dbp->put(NULL, &key, &data, 0)) != 0) { dbp->err(ret, "DB->put"); if (ret != DB_KEYEXIST) goto err; } } err: if (dbp != NULL) (void)dbp->close(DB_NOSYNC); return (ret); }
int main(int argc, char *argv[]) { try { Db *db = new Db(NULL, 0); db->open(NULL, "my.db", NULL, DB_BTREE, DB_CREATE, 0644); // populate our massive database. // all our strings include null for convenience. // Note we have to cast for idiomatic // usage, since newer gcc requires it. Dbt *keydbt = new Dbt((char*)"key", 4); Dbt *datadbt = new Dbt((char*)"data", 5); db->put(NULL, keydbt, datadbt, 0); // Now, retrieve. We could use keydbt over again, // but that wouldn't be typical in an application. Dbt *goodkeydbt = new Dbt((char*)"key", 4); Dbt *badkeydbt = new Dbt((char*)"badkey", 7); Dbt *resultdbt = new Dbt(); resultdbt->set_flags(DB_DBT_MALLOC); int ret; if ((ret = db->get(NULL, goodkeydbt, resultdbt, 0)) != 0) { cout << "get: " << DbEnv::strerror(ret) << "\n"; } else { char *result = (char *)resultdbt->get_data(); cout << "got data: " << result << "\n"; } if ((ret = db->get(NULL, badkeydbt, resultdbt, 0)) != 0) { // We expect this... cout << "get using bad key: " << DbEnv::strerror(ret) << "\n"; } else { char *result = (char *)resultdbt->get_data(); cout << "*** got data using bad key!!: " << result << "\n"; } // Now, truncate and make sure that it's really gone. cout << "truncating data...\n"; u_int32_t nrecords; db->truncate(NULL, &nrecords, 0); cout << "truncate returns " << nrecords << "\n"; if ((ret = db->get(NULL, goodkeydbt, resultdbt, 0)) != 0) { // We expect this... cout << "after truncate get: " << DbEnv::strerror(ret) << "\n"; } else { char *result = (char *)resultdbt->get_data(); cout << "got data: " << result << "\n"; } db->close(0); cout << "finished test\n"; } catch (DbException &dbe) { cerr << "Db Exception: " << dbe.what(); } return 0; }
int main() { u_int32_t env_flags = DB_CREATE | // If the environment does not exist, create it. DB_INIT_MPOOL; // Initialize the in-memory cache. //std::string envHome("./"); std::string envHome("/home/kirill"); u_int32_t db_flags = DB_RDONLY; // only read std::string dbName("X-po2s.db"); //std::string dbName("X-so2p.db"); //std::string dbName("X-sp2o.db"); DbEnv myEnv(0); Db *myDb; // Instantiate the Db object Dbc *cursorp; // cursor try { cout << "X-po2s.db output:" << endl; myEnv.open(envHome.c_str(), env_flags, 0); myDb = new Db(&myEnv, 0); myDb->open(NULL, dbName.c_str(), NULL, DB_BTREE, db_flags, 0); // Get a cursor myDb->cursor(NULL, &cursorp, 0); Dbt key, data; // Position the cursor to the first record in the database whose // key and data begin with the correct strings. int ret = cursorp->get(&key, &data, DB_NEXT); while (ret != DB_NOTFOUND) { cout << "..." << endl; std::cout << "key: " << (char *)key.get_data() << "data: " << (char *)data.get_data()<< std::endl; ret = cursorp->get(&key, &data, DB_NEXT); } // Cursors must be closed if (cursorp != NULL) cursorp->close(); if (myDb != NULL) { myDb->close(0); } myEnv.close(0); cout << "Closing ..." << endl; } // Must catch both DbException and std::exception catch(DbException &e) { myDb->err(e.get_errno(), "Database open failed %s", dbName.c_str()); throw e; } catch(std::exception &e) { // No DB error number available, so use errx myDb->errx("Error opening database: %s", e.what()); throw e; } return 0; }
int main(int argc, const char* argv[]) { if (argc !=3) { cout << "usage: "<< argv[0] << " <filename> <db filename>" << endl; return 1; } const char* kDatabaseName = argv[2]; const char* filename = argv[1]; DbEnv env(0); Db* pdb; string line; bool datamode = false; try { env.set_error_stream(&cerr); env.open("./", DB_CREATE | DB_INIT_MPOOL, 0); pdb = new Db(&env, 0); // If you want to support duplicated records and make duplicated // records sorted by data, you need to call: // pdb->set_flags(DB_DUPSORT); // Note that only Btree-typed database supports sorted duplicated // records // If the database does not exist, create it. If it exists, clear // its content after openning. pdb->set_flags( DB_DUP );// | DB_DUPSORT); pdb->open(NULL, kDatabaseName, NULL, DB_BTREE, DB_CREATE | DB_TRUNCATE, 0); ifstream myfile (filename); if (myfile.is_open()) { int count = 0; while ( myfile.good() ) { //if (count++ > 10) { // break; //} getline (myfile,line); if (!datamode) { if ("[DATA]" == line ) { datamode = true; } continue; } //cout << (int)line[8] << endl; int index = 0; while (32 != (char)line[index]) { index++; } string cjkey = line.substr(0, index); while (32 == line[index]) { index++; } string cjdata = line.substr(index, (line.size()-index)); cout << cjkey << "---" << cjdata << endl; Dbt key(const_cast<char*>(cjkey.data()), cjkey.size()); Dbt value(const_cast<char*>(cjdata.data()), cjdata.size()); pdb->put(NULL, &key, &value, 0); } myfile.close(); } else { cout << "Unable to open file"; } // You need to set ulen and flags=DB_DBT_USERMEM to prevent Dbt // from allocate its own memory but use the memory provided by you. string search("aa"); Dbt key(const_cast<char*>(search.c_str()), search.size()); char buffer[1024]; Dbt data; data.set_data(buffer); data.set_ulen(1024); data.set_flags(DB_DBT_USERMEM); if (pdb->get(NULL, &key, &data, 0) == DB_NOTFOUND) { cerr << "Not found" << endl; } else { cout << "Found: " << "PPPPPP" <<buffer << "PPPPPP" << endl; } if (pdb != NULL) { pdb->close(0); delete pdb; // You have to close and delete an exisiting handle, then create // a new one before you can use it to remove a database (file). pdb = new Db(NULL, 0); //pdb->remove("access.db", NULL, 0); delete pdb; } env.close(0); } catch (DbException& e) { cerr << "DbException: " << e.what() << endl; return -1; } catch (std::exception& e) { cerr << e.what() << endl; return -1; } return 0; }
int RepMgrGSG::doloop() { Dbt key, data; Db *dbp; char buf[BUFSIZE], *rbuf; int ret; dbp = 0; memset(&key, 0, sizeof(key)); memset(&data, 0, sizeof(data)); ret = 0; for (;;) { if (dbp == 0) { dbp = new Db(&dbenv, 0); try { dbp->open(NULL, DATABASE, NULL, DB_BTREE, app_data.is_master ? DB_CREATE | DB_AUTO_COMMIT : DB_AUTO_COMMIT, 0); } catch(DbException dbe) { // It is expected that this condition will be triggered // when client sites start up. It can take a while for // the master site to be found and synced, and no DB will // be available until then. if (dbe.get_errno() == ENOENT) { cout << "No stock db available yet - retrying." << endl; try { dbp->close(0); } catch (DbException dbe2) { cout << "Unexpected error closing after failed" << " open, message: " << dbe2.what() << endl; dbp = NULL; goto err; } dbp = NULL; sleep(SLEEPTIME); continue; } else { dbenv.err(ret, "DB->open"); throw dbe; } } } cout << "QUOTESERVER" ; if (!app_data.is_master) cout << "(read-only)"; cout << "> " << flush; if (fgets(buf, sizeof(buf), stdin) == NULL) break; if (strtok(&buf[0], " \t\n") == NULL) { switch ((ret = print_stocks(dbp))) { case 0: continue; case DB_REP_HANDLE_DEAD: (void)dbp->close(DB_NOSYNC); cout << "closing db handle due to rep handle dead" << endl; dbp = NULL; continue; default: dbp->err(ret, "Error traversing data"); goto err; } } rbuf = strtok(NULL, " \t\n"); if (rbuf == NULL || rbuf[0] == '\0') { if (strncmp(buf, "exit", 4) == 0 || strncmp(buf, "quit", 4) == 0) break; dbenv.errx("Format: TICKER VALUE"); continue; } if (!app_data.is_master) { dbenv.errx("Can't update at client"); continue; } key.set_data(buf); key.set_size((u_int32_t)strlen(buf)); data.set_data(rbuf); data.set_size((u_int32_t)strlen(rbuf)); if ((ret = dbp->put(NULL, &key, &data, 0)) != 0) { dbp->err(ret, "DB->put"); if (ret != DB_KEYEXIST) goto err; } } err: if (dbp != 0) { (void)dbp->close(DB_NOSYNC); } return (ret); }
// // Initialize the database to the specified number of accounts, branches, // history records, and tellers. // void TpcbExample::populate(int accounts, int branches, int history, int tellers) { Db *dbp; int err; u_int32_t balance, idnum; u_int32_t end_anum, end_bnum, end_tnum; u_int32_t start_anum, start_bnum, start_tnum; idnum = BEGID; balance = 500000; dbp = new Db(this, 0); dbp->set_h_nelem((unsigned int)accounts); if ((err = dbp->open(NULL, "account", NULL, DB_HASH, DB_CREATE, 0644)) != 0) { DbException except("Account file create failed", err); throw except; } start_anum = idnum; populateTable(dbp, idnum, balance, accounts, "account"); idnum += accounts; end_anum = idnum - 1; if ((err = dbp->close(0)) != 0) { DbException except("Account file close failed", err); throw except; } delete dbp; if (verbose) cout << "Populated accounts: " << (long)start_anum << " - " << (long)end_anum << "\n"; dbp = new Db(this, 0); // // Since the number of branches is very small, we want to use very // small pages and only 1 key per page. This is the poor-man's way // of getting key locking instead of page locking. // dbp->set_h_ffactor(1); dbp->set_h_nelem((unsigned int)branches); dbp->set_pagesize(512); if ((err = dbp->open(NULL, "branch", NULL, DB_HASH, DB_CREATE, 0644)) != 0) { DbException except("Branch file create failed", err); throw except; } start_bnum = idnum; populateTable(dbp, idnum, balance, branches, "branch"); idnum += branches; end_bnum = idnum - 1; if ((err = dbp->close(0)) != 0) { DbException except("Close of branch file failed", err); throw except; } delete dbp; if (verbose) cout << "Populated branches: " << (long)start_bnum << " - " << (long)end_bnum << "\n"; dbp = new Db(this, 0); // // In the case of tellers, we also want small pages, but we'll let // the fill factor dynamically adjust itself. // dbp->set_h_ffactor(0); dbp->set_h_nelem((unsigned int)tellers); dbp->set_pagesize(512); if ((err = dbp->open(NULL, "teller", NULL, DB_HASH, DB_CREATE, 0644)) != 0) { DbException except("Teller file create failed", err); throw except; } start_tnum = idnum; populateTable(dbp, idnum, balance, tellers, "teller"); idnum += tellers; end_tnum = idnum - 1; if ((err = dbp->close(0)) != 0) { DbException except("Close of teller file failed", err); throw except; } delete dbp; if (verbose) cout << "Populated tellers: " << (long)start_tnum << " - " << (long)end_tnum << "\n"; dbp = new Db(this, 0); dbp->set_re_len(HISTORY_LEN); if ((err = dbp->open(NULL, "history", NULL, DB_RECNO, DB_CREATE, 0644)) != 0) { DbException except("Create of history file failed", err); throw except; } populateHistory(dbp, history, accounts, branches, tellers); if ((err = dbp->close(0)) != 0) { DbException except("Close of history file failed", err); throw except; } delete dbp; }
bool CDB::Rewrite(const string& strFile, const char* pszSkip) { while (!fShutdown) { { LOCK(bitdb.cs_db); if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) { // Flush log data to the dat file bitdb.CloseDb(strFile); bitdb.CheckpointLSN(strFile); bitdb.mapFileUseCount.erase(strFile); bool fSuccess = true; printf("Rewriting %s...\n", strFile.c_str()); string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} CDB db(strFile.c_str(), "r"); Db* pdbCopy = new Db(&bitdb.dbenv, 0); int ret = pdbCopy->open(NULL, // Txn pointer strFileRes.c_str(), // Filename "main", // Logical db name DB_BTREE, // Database type DB_CREATE, // Flags 0); if (ret > 0) { printf("Cannot create database file %s\n", strFileRes.c_str()); fSuccess = false; } Dbc* pcursor = db.GetCursor(); if (pcursor) while (fSuccess) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT); if (ret == DB_NOTFOUND) { pcursor->close(); break; } else if (ret != 0) { pcursor->close(); fSuccess = false; break; } if (pszSkip && strncmp(&ssKey[0], pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) continue; if (strncmp(&ssKey[0], "\x07version", 8) == 0) { // Update version: ssValue.clear(); ssValue << CLIENT_VERSION; } Dbt datKey(&ssKey[0], ssKey.size()); Dbt datValue(&ssValue[0], ssValue.size()); int ret2 = pdbCopy->put(NULL, &datKey, &datValue, DB_NOOVERWRITE); if (ret2 > 0) fSuccess = false; } if (fSuccess) { db.Close(); bitdb.CloseDb(strFile); if (pdbCopy->close(0)) fSuccess = false; delete pdbCopy; } } if (fSuccess) { Db dbA(&bitdb.dbenv, 0); if (dbA.remove(strFile.c_str(), NULL, 0)) fSuccess = false; Db dbB(&bitdb.dbenv, 0); if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0)) fSuccess = false; } if (!fSuccess) printf("Rewriting of %s FAILED!\n", strFileRes.c_str()); return fSuccess; } } MilliSleep(100); } return false; }
static int run(const Ice::StringSeq& originalArgs, const Ice::CommunicatorPtr& communicator) { vector<string> oldCppArgs; vector<string> newCppArgs; bool debug; bool ice = true; // Needs to be true in order to create default definitions. bool underscore; string outputFile; bool ignoreTypeChanges; bool purgeObjects; bool catastrophicRecover; bool suppress; string inputFile; vector<string> oldSlice; vector<string> newSlice; bool evictor; string keyTypeNames; string valueTypeNames; string dbEnvName, dbName, dbEnvNameNew; bool allDb = false; IceUtilInternal::Options opts; opts.addOpt("h", "help"); opts.addOpt("v", "version"); opts.addOpt("D", "", IceUtilInternal::Options::NeedArg, "", IceUtilInternal::Options::Repeat); opts.addOpt("U", "", IceUtilInternal::Options::NeedArg, "", IceUtilInternal::Options::Repeat); opts.addOpt("d", "debug"); opts.addOpt("", "underscore"); opts.addOpt("o", "", IceUtilInternal::Options::NeedArg); opts.addOpt("i"); opts.addOpt("p"); opts.addOpt("c"); opts.addOpt("w"); opts.addOpt("f", "", IceUtilInternal::Options::NeedArg); opts.addOpt("", "include-old", IceUtilInternal::Options::NeedArg, "", IceUtilInternal::Options::Repeat); opts.addOpt("", "include-new", IceUtilInternal::Options::NeedArg, "", IceUtilInternal::Options::Repeat); opts.addOpt("", "old", IceUtilInternal::Options::NeedArg, "", IceUtilInternal::Options::Repeat); opts.addOpt("", "new", IceUtilInternal::Options::NeedArg, "", IceUtilInternal::Options::Repeat); opts.addOpt("a"); opts.addOpt("e"); opts.addOpt("", "key", IceUtilInternal::Options::NeedArg); opts.addOpt("", "value", IceUtilInternal::Options::NeedArg); const string appName = originalArgs[0]; vector<string> args; try { args = opts.parse(originalArgs); } catch(const IceUtilInternal::BadOptException& e) { cerr << appName << ": " << e.reason << endl; usage(appName); return EXIT_FAILURE; } if(opts.isSet("help")) { usage(appName); return EXIT_SUCCESS; } if(opts.isSet("version")) { cout << ICE_STRING_VERSION << endl; return EXIT_SUCCESS; } if(opts.isSet("D")) { vector<string> optargs = opts.argVec("D"); for(vector<string>::const_iterator i = optargs.begin(); i != optargs.end(); ++i) { oldCppArgs.push_back("-D" + *i); newCppArgs.push_back("-D" + *i); } } if(opts.isSet("U")) { vector<string> optargs = opts.argVec("U"); for(vector<string>::const_iterator i = optargs.begin(); i != optargs.end(); ++i) { oldCppArgs.push_back("-U" + *i); newCppArgs.push_back("-U" + *i); } } debug = opts.isSet("debug"); underscore = opts.isSet("underscore"); if(opts.isSet("o")) { outputFile = opts.optArg("o"); } ignoreTypeChanges = opts.isSet("i"); purgeObjects = opts.isSet("p"); catastrophicRecover = opts.isSet("c"); suppress = opts.isSet("w"); if(opts.isSet("f")) { inputFile = opts.optArg("f"); } if(opts.isSet("include-old")) { vector<string> optargs = opts.argVec("include-old"); for(vector<string>::const_iterator i = optargs.begin(); i != optargs.end(); ++i) { oldCppArgs.push_back("-I" + *i); } } if(opts.isSet("include-new")) { vector<string> optargs = opts.argVec("include-new"); for(vector<string>::const_iterator i = optargs.begin(); i != optargs.end(); ++i) { newCppArgs.push_back("-I" + *i); } } if(opts.isSet("old")) { vector<string> optargs = opts.argVec("old"); for(vector<string>::const_iterator i = optargs.begin(); i != optargs.end(); ++i) { oldSlice.push_back(*i); } } if(opts.isSet("new")) { vector<string> optargs = opts.argVec("new"); for(vector<string>::const_iterator i = optargs.begin(); i != optargs.end(); ++i) { newSlice.push_back(*i); } } evictor = opts.isSet("e"); if(opts.isSet("key")) { keyTypeNames = opts.optArg("key"); } if(opts.isSet("value")) { valueTypeNames = opts.optArg("value"); } if(outputFile.empty()) { if(args.size() == 2) { allDb = true; } else if(args.size() != 3) { usage(appName); return EXIT_FAILURE; } } else { if(args.size() == 1) { allDb = true; } else if(args.size() != 0) { usage(appName); return EXIT_FAILURE; } } if(allDb && (!keyTypeNames.empty() || !valueTypeNames.empty())) { usage(appName); return EXIT_FAILURE; } if(inputFile.empty() && !allDb && !evictor && (keyTypeNames.empty() || valueTypeNames.empty())) { usage(appName); return EXIT_FAILURE; } if(args.size() > 0) { dbEnvName = args[0]; } if(args.size() > 1) { if(allDb) { dbEnvNameNew = args[1]; } else { dbName = args[1]; } } if(args.size() > 2) { dbEnvNameNew = args[2]; } if(args.size() > 3) { cerr << appName << ": too many arguments" << endl; usage(appName); return EXIT_FAILURE; } // // Freeze creates a lock file by default to prevent multiple processes from opening // the same database environment simultaneously. In the case of a read-only program // such as transformdb, however, we still want to be able to open the environment despite // the lock. This assumes of course that the other process has opened the environment // with DbPrivate=0. If DbPrivate=0 is also set for dumpdb, we disable the lock. // if(!catastrophicRecover && outputFile.empty()) { Ice::PropertiesPtr props = communicator->getProperties(); string prefix = "Freeze.DbEnv." + args[0]; if(props->getPropertyAsIntWithDefault(prefix + ".DbPrivate", 1) == 0) { props->setProperty(prefix + ".LockFile", "0"); } } Slice::UnitPtr oldUnit = Slice::Unit::createUnit(true, true, ice, underscore); FreezeScript::Destroyer<Slice::UnitPtr> oldD(oldUnit); if(!FreezeScript::parseSlice(appName, oldUnit, oldSlice, oldCppArgs, debug, "-D__TRANSFORMDB__")) { return EXIT_FAILURE; } Slice::UnitPtr newUnit = Slice::Unit::createUnit(true, true, ice, underscore); FreezeScript::Destroyer<Slice::UnitPtr> newD(newUnit); if(!FreezeScript::parseSlice(appName, newUnit, newSlice, newCppArgs, debug, "-D__TRANSFORMDB__")) { return EXIT_FAILURE; } // // Install the evictor types in the Slice units. // FreezeScript::createEvictorSliceTypes(oldUnit); FreezeScript::createEvictorSliceTypes(newUnit); // // Read the catalog if necessary. // FreezeScript::CatalogDataMap catalog; if(allDb) { try { catalog = FreezeScript::readCatalog(communicator, dbEnvName); } catch(const FreezeScript::FailureException& ex) { cerr << appName << ": " << ex.reason() << endl; return EXIT_FAILURE; } if(catalog.empty()) { cerr << appName << ": no databases in environment `" << dbEnvName << "'" << endl; return EXIT_FAILURE; } } // // If no input file was provided, then we need to analyze the Slice types. // string descriptors; if(inputFile.empty()) { ostringstream out; vector<string> missingTypes; vector<string> analyzeErrors; FreezeScript::TransformAnalyzer analyzer(oldUnit, newUnit, ignoreTypeChanges, out, missingTypes, analyzeErrors); const string evictorKeyName = "::Ice::Identity"; const string oldEvictorValueName = "::Freeze::ObjectRecord"; const string newEvictorValueName = "Object"; if(allDb) { // // Add a <database> element for each database in the catalog. // for(FreezeScript::CatalogDataMap::iterator p = catalog.begin(); p != catalog.end(); ++p) { string keyName, valueName; if(p->second.evictor) { keyName = p->second.key; valueName = p->second.value; if(keyName.empty()) { keyName = evictorKeyName; } if(valueName.empty()) { valueName = oldEvictorValueName; } } else { keyName = p->second.key; valueName = p->second.value; } // // Look up the key and value types in the old and new Slice definitions. // Slice::TypePtr oldKeyType = findType(oldUnit, keyName); if(!oldKeyType) { cerr << appName << ": type `" << keyName << "' from database `" << p->first << "' not found in old Slice definitions" << endl; } Slice::TypePtr newKeyType = findType(newUnit, keyName); if(!newKeyType) { cerr << appName << ": type `" << keyName << "' from database `" << p->first << "' not found in new Slice definitions" << endl; } Slice::TypePtr oldValueType = findType(oldUnit, valueName); if(!oldValueType) { cerr << appName << ": type `" << valueName << "' from database `" << p->first << "' not found in old Slice definitions" << endl; } Slice::TypePtr newValueType; if(p->second.evictor) { // // The new evictor does not keep stats // newValueType = findType(newUnit, newEvictorValueName); } else { newValueType = findType(newUnit, valueName); } if(!newValueType) { cerr << appName << ": type `" << valueName << "' from database `" << p->first << "' not found in new Slice definitions" << endl; } // // If we are generating an output file, we do not consider a missing type to be // an error. Since the type information comes from the catalog of the old // environment, it's possible that the key or value types are not present in // the new Slice definitions. Rather than abort at this point, we simply emit // a partially-defined <database> element that must be edited by the user. // // If we are not generating an output file, we have to stop now. // if(outputFile.empty() && (!oldKeyType || !newKeyType || !oldValueType || !newValueType)) { return EXIT_FAILURE; } analyzer.addDatabase(p->first, oldKeyType, newKeyType, oldValueType, newValueType); } } else { string oldKeyName, newKeyName, oldValueName, newValueName; string::size_type pos; if(!evictor && (keyTypeNames.empty() || valueTypeNames.empty())) { usage(appName); return EXIT_FAILURE; } if(!keyTypeNames.empty()) { pos = keyTypeNames.find(','); if(pos == 0 || pos == keyTypeNames.size()) { usage(appName); return EXIT_FAILURE; } if(pos == string::npos) { oldKeyName = keyTypeNames; newKeyName = keyTypeNames; } else { oldKeyName = keyTypeNames.substr(0, pos); newKeyName = keyTypeNames.substr(pos + 1); } } if(!valueTypeNames.empty()) { pos = valueTypeNames.find(','); if(pos == 0 || pos == valueTypeNames.size()) { usage(appName); return EXIT_FAILURE; } if(pos == string::npos) { oldValueName = valueTypeNames; newValueName = valueTypeNames; } else { oldValueName = valueTypeNames.substr(0, pos); newValueName = valueTypeNames.substr(pos + 1); } } if(evictor) { if(oldKeyName.empty()) { oldKeyName = evictorKeyName; } if(newKeyName.empty()) { newKeyName = evictorKeyName; } if(oldValueName.empty()) { oldValueName = newEvictorValueName; } if(newValueName.empty()) { newValueName = newEvictorValueName; } } // // Look up the key and value types in the old and new Slice definitions. // Slice::TypePtr oldKeyType = findType(oldUnit, oldKeyName); if(!oldKeyType) { cerr << appName << ": type `" << oldKeyName << "' not found in old Slice definitions" << endl; } Slice::TypePtr newKeyType = findType(newUnit, newKeyName); if(!newKeyType) { cerr << appName << ": type `" << newKeyName << "' not found in new Slice definitions" << endl; } Slice::TypePtr oldValueType = findType(oldUnit, oldValueName); if(!oldValueType) { cerr << appName << ": type `" << oldValueName << "' not found in old Slice definitions" << endl; } Slice::TypePtr newValueType = findType(newUnit, newValueName); if(!newValueType) { cerr << appName << ": type `" << newValueName << "' not found in new Slice definitions" << endl; } // // Stop now if any of the types could not be found. // if(!oldKeyType || !newKeyType || !oldValueType || !newValueType) { return EXIT_FAILURE; } analyzer.addDatabase("", oldKeyType, newKeyType, oldValueType, newValueType); } analyzer.finish(); if(!analyzeErrors.empty()) { for(vector<string>::const_iterator p = analyzeErrors.begin(); p != analyzeErrors.end(); ++p) { cerr << appName << ": " << *p << endl; } } if(!missingTypes.empty()) { sort(missingTypes.begin(), missingTypes.end()); unique(missingTypes.begin(), missingTypes.end()); if(!analyzeErrors.empty()) { cerr << endl; } cerr << "The following types had no matching definitions in the new Slice:" << endl; for(vector<string>::const_iterator p = missingTypes.begin(); p != missingTypes.end(); ++p) { cerr << " " << *p << endl; } } if(!analyzeErrors.empty()) { return EXIT_FAILURE; } descriptors = out.str(); if(!outputFile.empty()) { // // No nativeToUTF8 conversion necessary here, no string converter is installed // by wmain() on Windows and args are assumbed to be UTF8 on Unix platforms. // IceUtilInternal::ofstream of(outputFile); if(!of.good()) { cerr << appName << ": unable to open file `" << outputFile << "'" << endl; return EXIT_FAILURE; } of << descriptors; of.close(); return EXIT_SUCCESS; } } else { // // Read the input file. // // No nativeToUTF8 conversion necessary here, no string converter is installed // by wmain() on Windows and args are assumbed to be UTF8 on Unix platforms. // IceUtilInternal::ifstream in(inputFile); char buff[1024]; while(true) { in.read(buff, 1024); descriptors.append(buff, static_cast<size_t>(in.gcount())); if(in.gcount() < 1024) { break; } } in.close(); } if(dbEnvName == dbEnvNameNew) { cerr << appName << ": database environment names must be different" << endl; return EXIT_FAILURE; } FreezeScript::ObjectFactoryPtr objectFactory = new FreezeScript::ObjectFactory; communicator->addObjectFactory(objectFactory, ""); // // Transform the database. // DbEnv dbEnv(0); DbEnv dbEnvNew(0); Freeze::TransactionPtr txNew; Freeze::ConnectionPtr connection; Freeze::ConnectionPtr connectionNew; vector<Db*> dbs; int status = EXIT_SUCCESS; try { #ifdef _WIN32 // // Berkeley DB may use a different C++ runtime. // dbEnv.set_alloc(::malloc, ::realloc, ::free); dbEnvNew.set_alloc(::malloc, ::realloc, ::free); #endif // // Open the old database environment. Use DB_RECOVER_FATAL if -c is specified. // No transaction is created for the old environment. // // DB_THREAD is for compatibility with Freeze (the catalog) // { u_int32_t flags = DB_THREAD | DB_CREATE | DB_INIT_TXN | DB_INIT_MPOOL; if(catastrophicRecover) { flags |= DB_INIT_LOG | DB_RECOVER_FATAL; } dbEnv.open(dbEnvName.c_str(), flags, FREEZE_SCRIPT_DB_MODE); } // // We're creating a connection just to make sure the database environment // isn't locked. // connection = Freeze::createConnection(communicator, dbEnvName, dbEnv); // // Open the new database environment and start a transaction. // // // DB_THREAD is for compatibility with Freeze (the catalog) // { u_int32_t flags = DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_RECOVER | DB_CREATE | DB_THREAD; dbEnvNew.open(dbEnvNameNew.c_str(), flags, FREEZE_SCRIPT_DB_MODE); } // // TODO: handle properly DbHome config (currently it will break if it's set for the new env) // // // Open the catalog of the new environment, and start a transaction. // connectionNew = Freeze::createConnection(communicator, dbEnvNameNew, dbEnvNew); txNew = connectionNew->beginTransaction(); DbTxn* txnNew = Freeze::getTxn(txNew); if(allDb) { // // Transform all databases in the old catalog. // for(FreezeScript::CatalogDataMap::iterator p = catalog.begin(); p != catalog.end(); ++p) { transformDb(p->second.evictor, communicator, objectFactory, dbEnv, dbEnvNew, p->first, connectionNew, dbs, oldUnit, newUnit, txnNew, purgeObjects, suppress, descriptors); } } else { transformDb(evictor, communicator, objectFactory, dbEnv, dbEnvNew, dbName, connectionNew, dbs, oldUnit, newUnit, txnNew, purgeObjects, suppress, descriptors); } } catch(const DbException& ex) { cerr << appName << ": database error: " << ex.what() << endl; status = EXIT_FAILURE; } catch(const IceUtil::FileLockException&) { cerr << appName << ": error: database environment is locked" << endl; status = EXIT_FAILURE; } catch(...) { try { if(txNew != 0) { txNew->rollback(); txNew = 0; } if(connectionNew) { connectionNew->close(); connectionNew = 0; } if(connection) { connection->close(); connection = 0; } for(vector<Db*>::iterator p = dbs.begin(); p != dbs.end(); ++p) { Db* db = *p; db->close(0); delete db; } try { dbEnv.close(0); } catch(const DbException&) { } try { dbEnvNew.close(0); } catch(const DbException&) { } } catch(const DbException& ex) { cerr << appName << ": database error: " << ex.what() << endl; } throw; } if(txNew != 0) { try { if(status == EXIT_FAILURE) { txNew->rollback(); } else { txNew->commit(); // // Checkpoint to migrate changes from the log to the database(s). // dbEnvNew.txn_checkpoint(0, 0, DB_FORCE); } for(vector<Db*>::iterator p = dbs.begin(); p != dbs.end(); ++p) { Db* db = *p; db->close(0); delete db; } } catch(const DbException& ex) { cerr << appName << ": database error: " << ex.what() << endl; status = EXIT_FAILURE; } } // Clear the transaction before closing the database environment. txNew = 0; if(connectionNew) { connectionNew->close(); connectionNew = 0; } if(connection) { connection->close(); connection = 0; } try { dbEnv.close(0); } catch(const DbException&) { } try { dbEnvNew.close(0); } catch(const DbException&) { } return status; }