// Pruning is based on two factors.
// 1. All words with a total number of edits less that minimum_edits are discarded
// 2. All words with a vandal-ness close to 0.5 are discarded (where vandal-ness differs from 0.5 by less than min_median_dev)
void pruneDB(unsigned int minimum_edits = 3, float min_median_dev = 0.3) {
unsigned int total_good;
unsigned int total_bad;
getTotals(total_good, total_bad);
Dbc * cur;
db.cursor(NULL, &cur, 0);
if(cur) {
Dbt key, data;
while(cur->get(&key, &data, DB_NEXT) == 0) {
std::string word((char *)key.get_data(), key.get_size());
if(word == "_EDIT_TOTALS") continue;
BayesDBData bdata;
int s = data.get_size();
if(s != sizeof(bdata)) throw std::runtime_error("Bayesian DB returned wrong record size");
void * vdata = data.get_data();
memcpy((void *)&bdata, vdata, s);
unsigned int nedits = bdata.good_edits + bdata.bad_edits;
float vand_prob = (float)bdata.bad_edits / (float)total_bad;
float good_prob = (float)bdata.good_edits / (float)total_good;
float vandalness = vand_prob / (vand_prob + good_prob);
float vmeddev = 0.5 - vandalness;
if(vmeddev < 0) vmeddev = 0 - vmeddev;
if(vmeddev < min_median_dev || nedits < minimum_edits) {
cur->del(0);
}
}
cur->close();
}
}
bool FileRecord::RemoveLock(const std::string& lock_id, std::list<std::pair<std::string,std::string> >& ids) {
if(!valid_) return false;
Glib::Mutex::Lock lock(lock_);
Dbc* cur = NULL;
if(!dberr("removelock:cursor",db_lock_->cursor(NULL,&cur,DB_WRITECURSOR))) return false;
Dbt key;
Dbt data;
make_string(lock_id,key);
void* pkey = key.get_data();
if(!dberr("removelock:get1",cur->get(&key,&data,DB_SET))) { // TODO: handle errors
::free(pkey);
cur->close(); return false;
};
for(;;) {
std::string id;
std::string owner;
uint32_t size = data.get_size();
void* buf = data.get_data();
buf = parse_string(id,buf,size);
buf = parse_string(owner,buf,size);
ids.push_back(std::pair<std::string,std::string>(id,owner));
if(!dberr("removelock:del",cur->del(0))) {
::free(pkey);
cur->close(); return false;
};
if(!dberr("removelock:get2",cur->get(&key,&data,DB_NEXT_DUP))) break;
};
db_lock_->sync(0);
::free(pkey);
cur->close();
return true;
}
void DatabaseManager::deleteRecordRange(const StringPimpl &key,
const StringPimpl &range)
{
Dbc *cursor = NULL;
try
{
Dbt dbKey((void *)key.c_str(), key.size() + 1);
Dbt dbData((void *) range.c_str(), range.size() + 1);
m_pimpl->checkTSSInit();
m_pimpl->m_database->cursor(&(**m_pimpl->m_transaction), &cursor, m_pimpl->m_cursorFlags);
int ret = cursor->get(&dbKey, &dbData, DB_GET_BOTH_RANGE);
while(ret != DB_NOTFOUND)
{
cursor->del(0);
ret = cursor->get(&dbKey, &dbData, DB_NEXT);
}
cursor->close();
}
catch (DbException &e)
{
std::cerr << "Error in addRecord: "
<< e.what() << e.get_errno() << std::endl;
if(cursor != NULL)
{
cursor->close();
}
}
}
void AvailableGroups::slotDeleteServer(quint16 serverId)
{
Dbt key, data;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
Dbc *cursor = 0;
int ret;
groupDb->cursor(0, &cursor, DB_WRITECURSOR);
while ((ret=cursor->get(&key, &data, DB_NEXT)) != DB_NOTFOUND)
{
g=new AvailableGroup((char*)data.get_data());
g->removeServerPresence(serverId);
//resave the group...
const char *p=g->data();
memset(&data, 0, sizeof(data));
data.set_data((void*)p);
data.set_size(g->size());
if (g->noServerPresence())
{
ret=cursor->del(0);
if (ret != 0)
quban->getLogAlertList()->logMessage(LogAlertList::Error, tr("Cannot delete server ") + servers->value(serverId)->getHostName() + tr(" group record"));
}
else if ((ret=cursor->put(&key, &data, DB_CURRENT)) != 0)
quban->getLogAlertList()->logMessage(LogAlertList::Error, tr("Cannot update group record. Error: ") + dbEnv->strerror(ret));
Q_DELETE(p);
}
cursor->close();
reloadData();
}
bool CDBStrMultiMap::get( const string &key, vector<string> &values )
{
if( db_is_open == false )
{
cout<<"==== Can not get items from BDB.BDB has not been opened."<<endl;
return false;
}
char key1[1024];
memset(key1, 0, 1024);
strcpy(key1, key.c_str());
Dbt dbt_key( key1, key.length() +1 );
try{
Dbc *dbcp;
db->cursor(NULL, &dbcp, 0);
Dbt data;
while( dbcp->get(&dbt_key, &data, DB_SET ) == 0){
char *data_string = (char*)data.get_data();
values.push_back( data_string );
dbcp->del(0);
}
dbcp->close();
}
catch( DbException &e)
{
cerr<<"!! Exception: "<<e.what()<<endl;
return false;
}
return true;
}
bool CDBStrMultiMap::get_any( string &key , vector<string> &values )
{
if( db_is_open == false )
{
cout<<"==== Can not get items from BDB.BDB has not been opened."<<endl;
return false;
}
try{
Dbc *dbcq;
db->cursor(NULL, &dbcq, 0);
Dbt dbt_key1;
Dbt dbt_data1;
char* key_str;
char* data_str;
if( dbcq->get(&dbt_key1, &dbt_data1, DB_FIRST) == 0 )
{
key_str = (char*)dbt_key1.get_data();
for( ; key_str< key_str + strlen(key_str) ; key_str++ )
{
key += *key_str;
}
data_str = (char*)dbt_data1.get_data();
values.push_back( data_str );
dbcq->del(0);
//Dbt dbt_key2( *key_str , strlen(key_str) );
while( dbcq -> get(&dbt_key1, &dbt_data1, DB_SET) == 0 )
{
key_str = (char*)dbt_key1.get_data();
values.push_back( data_str );
dbcq->del(0);
}
}
dbcq->close();
}
catch( DbException &e)
{
cerr<<"!! Exception: "<<e.what()<<endl;
return false;
}
return true;
}
bool Table::Prune(Transaction* tx, const ByteString &prefix)
{
Dbc* cursor = NULL;
u_int32_t flags = DB_NEXT;
DbTxn* txn;
if (tx == NULL)
txn = NULL;
else
txn = tx->txn;
if (db->cursor(txn, &cursor, 0) != 0)
return false;
Dbt key, value;
if (prefix.length > 0)
{
key.set_data(prefix.buffer);
key.set_size(prefix.length);
flags = DB_SET_RANGE;
}
while (cursor->get(&key, &value, flags) == 0)
{
if (key.get_size() < prefix.length)
break;
if (strncmp(prefix.buffer, (char*)key.get_data(), prefix.length) != 0)
break;
// don't delete keys starting with @@
if (!(key.get_size() >= 2 &&
((char*)key.get_data())[0] == '@' &&
((char*)key.get_data())[1] == '@'))
cursor->del(0);
flags = DB_NEXT;
}
if (cursor)
cursor->close();
return true;
}
void MultiPartHeader::removeServer(Db* pDB, quint16 serverId)
{
serverPart.remove(serverId);
serverLowest.remove(serverId);
quint64 multiPartKey = getMultiPartKey();
Header* h = 0;
// Walk the headers to get the PartNumMap
Dbc *cursorp = 0;
// Get a cursor
pDB->cursor(NULL, &cursorp, DB_WRITECURSOR);
// Set up our DBTs
Dbt key(&multiPartKey, sizeof(quint64));
Dbt data;
int ret = cursorp->get(&key, &data, DB_SET);
while (ret != DB_NOTFOUND)
{
h = new Header((char*)data.get_data(), (char*)key.get_data());
if (h->isHeldByServer(serverId))
{
h->removeServerPartNum(serverId);
if (h->isServerNumMapEmpty())
{
cursorp->del(0);
size-=h->getSize();
missingParts++;
}
}
Q_DELETE(h);
ret = cursorp->get(&key, &data, DB_NEXT_DUP);
}
if (cursorp != NULL)
cursorp->close();
}
void MultiPartHeader::deleteAllParts(Db* pDB)
{
int ret = 0;
Dbc *cursorp = 0;
Dbt key;
Dbt data;
//qDebug() << "Deleting all parts for key " << multiPartKey;
key.set_data(&multiPartKey);
key.set_size(sizeof(quint64));
// Get a cursor
pDB->cursor(NULL, &cursorp, DB_WRITECURSOR);
while ((ret = cursorp->get(&key, &data, DB_SET)) == 0)
cursorp->del(0);
pDB->sync(0);
if (cursorp != NULL)
cursorp->close();
}
void DatabaseManager::deleteRecordRange(const StringPimpl &key)
{
Dbc *cursor = NULL;
try
{
Dbt dbKey((void *)key.c_str(), key.size() + 1);
Dbt dbData;
m_pimpl->checkTSSInit();
m_pimpl->m_database->cursor(&(**m_pimpl->m_transaction), &cursor, m_pimpl->m_cursorFlags);
int ret = cursor->get(&dbKey, &dbData, DB_SET_RANGE);
while(ret != DB_NOTFOUND)
{
StringPimpl dataString = StringPimpl((char *) dbData.get_data());
StringPimpl keyString = StringPimpl((char *) dbKey.get_data());
if(!keyString.startsWith(key)) {
//We are at the end of the range
break;
}
cursor->del(0);
ret = cursor->get(&dbKey, &dbData, DB_NEXT);
}
cursor->close();
}
catch (DbException &e)
{
std::cerr << "Error in deleteRecordRange: "
<< e.what() << e.get_errno() << std::endl;
if(cursor != NULL)
{
cursor->close();
}
}
}
void
Freeze::MapHelperI::clear()
{
DbTxn* txn = _connection->dbTxn();
if(txn == 0)
{
closeAllIterators();
}
Dbt dbKey;
dbKey.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL);
Dbt dbValue;
dbValue.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL);
try
{
for(;;)
{
Dbc* dbc = 0;
try
{
IteratorHelperI::TxPtr tx;
if(txn == 0)
{
#ifdef ICE_CPP11_COMPILER
tx.reset(new IteratorHelperI::Tx(*this));
#else
tx = new IteratorHelperI::Tx(*this);
#endif
txn = tx->getTxn();
}
_db->cursor(txn, &dbc, 0);
while(dbc->get(&dbKey, &dbValue, DB_NEXT | DB_RMW) == 0)
{
dbc->del(0);
}
Dbc* toClose = dbc;
dbc = 0;
toClose->close();
break; // for (;;)
}
catch(const DbDeadlockException&)
{
if(dbc != 0)
{
try
{
dbc->close();
}
catch(const DbDeadlockException&)
{
if(txn != 0)
{
throw;
}
else
{
//
// Ignored
//
}
}
}
if(_connection->deadlockWarning())
{
Warning out(_connection->communicator()->getLogger());
out << "Deadlock in Freeze::MapHelperI::clear on Map \""
<< _dbName << "\"; retrying ...";
}
if(txn != 0)
{
throw;
}
//
// Otherwise retry
//
}
catch(...)
{
if(dbc != 0)
{
try
{
dbc->close();
}
catch(const DbDeadlockException&)
{
if(txn != 0)
{
throw;
}
else
{
//
// Ignored
//
}
}
}
throw;
}
}
}
catch(const DbDeadlockException& dx)
{
throw DeadlockException(__FILE__, __LINE__, dx.what(), _connection->currentTransaction());
}
catch(const DbException& dx)
{
throw DatabaseException(__FILE__, __LINE__, dx.what());
}
}
