int bdb_create_queue(char *queue_name) {
pthread_rwlock_wrlock(&qlist_ht_lock);
char *k = strdup(queue_name);
assert(k != NULL);
queue_t *q = (queue_t *)calloc(1, sizeof(queue_t));
assert(q != NULL);
q->dbp = NULL;
q->set_hits = q->old_set_hits = 0;
q->get_hits = q->old_get_hits = 0;
pthread_mutex_init(&(q->lock), NULL);
int ret;
DB_TXN *txnp = NULL;
ret = db_create(&(q->dbp), envp, 0);
CHECK_DB_RET(ret);
if (bdb_settings.q_extentsize != 0){
ret = q->dbp->set_q_extentsize(q->dbp, bdb_settings.q_extentsize);
CHECK_DB_RET(ret);
}
ret = q->dbp->set_re_len(q->dbp, bdb_settings.re_len);
CHECK_DB_RET(ret);
ret = q->dbp->set_pagesize(q->dbp, bdb_settings.page_size);
CHECK_DB_RET(ret);
ret = envp->txn_begin(envp, NULL, &txnp, 0);
CHECK_DB_RET(ret);
ret = q->dbp->open(q->dbp, txnp, queue_name, NULL, DB_QUEUE, DB_CREATE, 0664);
CHECK_DB_RET(ret);
DBT dbkey,dbdata;
qstats_t qs;
BDB_CLEANUP_DBT();
memset(&qs, 0, sizeof(qs));
dbkey.data = (void *)queue_name;
dbkey.size = strlen(queue_name)+1;
dbdata.data = (void *)&qs;
dbdata.size = sizeof(qstats_t);
ret = qlist_dbp->put(qlist_dbp, txnp, &dbkey, &dbdata, 0);
CHECK_DB_RET(ret);
ret = txnp->commit(txnp, 0);
CHECK_DB_RET(ret);
int result = hashtable_insert(qlist_htp, (void *)k, (void *)q);
assert(result != 0);
pthread_rwlock_unlock(&qlist_ht_lock);
return 0;
dberr:
if (txnp != NULL){
txnp->abort(txnp);
}
fprintf(stderr, "bdb_create_queue: %s %s\n", queue_name, db_strerror(ret));
pthread_rwlock_unlock(&qlist_ht_lock);
return -1;
}
/* if return item is not NULL, free by caller */
item *item_get(char *key, size_t nkey){
item *it = NULL;
DBT dbkey, dbdata;
bool stop;
int ret;
/* first, alloc a fixed size */
it = item_alloc2(settings.item_buf_size);
if (it == 0) {
return NULL;
}
BDB_CLEANUP_DBT();
dbkey.data = key;
dbkey.size = nkey;
dbdata.ulen = settings.item_buf_size;
dbdata.data = it;
dbdata.flags = DB_DBT_USERMEM;
stop = false;
/* try to get a item from bdb */
while (!stop) {
switch (ret = dbp->get(dbp, NULL, &dbkey, &dbdata, 0)) {
case DB_BUFFER_SMALL: /* user mem small */
/* free the original smaller buffer */
item_free(it);
/* alloc the correct size */
it = item_alloc2(dbdata.size);
if (it == NULL) {
return NULL;
}
dbdata.ulen = dbdata.size;
dbdata.data = it;
break;
case 0: /* Success. */
stop = true;
break;
case DB_NOTFOUND:
stop = true;
item_free(it);
it = NULL;
break;
default:
/* TODO: may cause bug here, if return DB_BUFFER_SMALL then retun non-zero again
* here 'it' may not a full one. a item buffer larger than item_buf_size may be added to freelist */
stop = true;
item_free(it);
it = NULL;
if (settings.verbose > 1) {
fprintf(stderr, "dbp->get: %s\n", db_strerror(ret));
}
}
}
return it;
}
/* if return item is not NULL, free by caller */
item *bdb_get(char *key){
pthread_rwlock_rdlock(&qlist_ht_lock);
item *it = NULL;
DB_TXN *txnp = NULL;
int ret;
queue_t *q = (queue_t *)hashtable_search(qlist_htp, (void *)key);
/* queue not exsited */
if (q == NULL) {
pthread_rwlock_unlock(&qlist_ht_lock);
return NULL;
} else {
DBT dbkey, dbdata;
db_recno_t recno;
/* first, alloc a fixed size */
it = item_alloc2();
if (it == 0) {
pthread_rwlock_unlock(&qlist_ht_lock);
return NULL;
}
BDB_CLEANUP_DBT();
dbkey.data = &recno;
dbkey.ulen = sizeof(recno);
dbkey.flags = DB_DBT_USERMEM;
dbdata.ulen = bdb_settings.re_len;
dbdata.data = it;
dbdata.flags = DB_DBT_USERMEM;
ret = envp->txn_begin(envp, NULL, &txnp, 0);
CHECK_DB_RET(ret);
ret = q->dbp->get(q->dbp, txnp, &dbkey, &dbdata, DB_CONSUME);
CHECK_DB_RET(ret);
ret = txnp->commit(txnp, 0);
CHECK_DB_RET(ret);
pthread_mutex_lock(&(q->lock));
(q->get_hits)++;
pthread_mutex_unlock(&(q->lock));
}
pthread_rwlock_unlock(&qlist_ht_lock);
return it;
dberr:
item_free(it);
it = NULL;
if (txnp != NULL){
txnp->abort(txnp);
}
if (settings.verbose > 1) {
fprintf(stderr, "bdb_get: %s\n", db_strerror(ret));
}
pthread_rwlock_unlock(&qlist_ht_lock);
return NULL;
}
void bdb_qlist_db_open(void){
int ret;
DBC *cursorp = NULL;
DB_TXN *txnp = NULL;
/* Create queue.list db handle */
ret = db_create(&qlist_dbp, envp, 0);
CHECK_DB_RET(ret);
/* Open and Iterate */
ret = envp->txn_begin(envp, NULL, &txnp, 0);
CHECK_DB_RET(ret);
ret = qlist_dbp->open(qlist_dbp, txnp, "queue.list", NULL, DB_BTREE, DB_CREATE, 0664);
CHECK_DB_RET(ret);
ret = qlist_dbp->cursor(qlist_dbp, txnp, &cursorp, 0);
CHECK_DB_RET(ret);
DBT dbkey, dbdata;
char qname[512];
qstats_t qs;
BDB_CLEANUP_DBT();
memset(qname, 0, 512);
memset(&qs, 0, sizeof(qs));
dbkey.data = (void *)qname;
dbkey.ulen = 512;
dbkey.flags = DB_DBT_USERMEM;
dbdata.data = (void *)&qs;
dbdata.ulen = sizeof(qs);
dbdata.flags = DB_DBT_USERMEM;
while ((ret = cursorp->get(cursorp, &dbkey, &dbdata, DB_NEXT)) == 0) {
open_exsited_queue_db(txnp, qname, &qs);
}
if (ret != DB_NOTFOUND) {
goto dberr;
}
ret = cursorp->close(cursorp);
CHECK_DB_RET(ret);
ret = txnp->commit(txnp, 0);
CHECK_DB_RET(ret);
return;
dberr:
if (cursorp != NULL){
cursorp->close(cursorp);
}
if (txnp != NULL){
txnp->abort(txnp);
}
fprintf(stderr, "bdb_qlist_db_open: %s\n", db_strerror(ret));
exit(EXIT_FAILURE);
}
/* 0 for Success
-1 for SERVER_ERROR
*/
int bdb_set(char *key, item *it){
pthread_rwlock_rdlock(&qlist_ht_lock);
queue_t *q = (queue_t *)hashtable_search(qlist_htp, (void *)key);
DB_TXN *txnp = NULL;
int ret;
if (NULL == q) {
pthread_rwlock_unlock(&qlist_ht_lock);
ret = bdb_create_queue(key);
if (0 != ret){
return -1;
}
/* search again */
pthread_rwlock_rdlock(&qlist_ht_lock);
q = (queue_t *)hashtable_search(qlist_htp, (void *)key);
}
if (NULL != q) {
db_recno_t recno;
DBT dbkey, dbdata;
BDB_CLEANUP_DBT();
dbkey.data = &recno;
dbkey.ulen = sizeof(recno);
dbkey.flags = DB_DBT_USERMEM;
dbdata.data = it;
dbdata.size = ITEM_ntotal(it);
ret = envp->txn_begin(envp, NULL, &txnp, 0);
CHECK_DB_RET(ret);
ret = q->dbp->put(q->dbp, txnp, &dbkey, &dbdata, DB_APPEND);
CHECK_DB_RET(ret);
ret = txnp->commit(txnp, 0);
CHECK_DB_RET(ret);
pthread_mutex_lock(&(q->lock));
(q->set_hits)++;
pthread_mutex_unlock(&(q->lock));
}
pthread_rwlock_unlock(&qlist_ht_lock);
return 0;
dberr:
if (txnp != NULL){
txnp->abort(txnp);
}
if (settings.verbose > 1) {
fprintf(stderr, "bdb_set: %s\n", db_strerror(ret));
}
pthread_rwlock_unlock(&qlist_ht_lock);
return -1;
}
/* 0 for Success
-1 for SERVER_ERROR
*/
int item_put(char *key, size_t nkey, item *it){
int ret;
DBT dbkey, dbdata;
BDB_CLEANUP_DBT();
dbkey.data = key;
dbkey.size = nkey;
dbdata.data = it;
dbdata.size = ITEM_ntotal(it);
ret = dbp->put(dbp, NULL, &dbkey, &dbdata, 0);
if (ret == 0) {
return 0;
} else {
if (settings.verbose > 1) {
fprintf(stderr, "dbp->put: %s\n", db_strerror(ret));
}
return -1;
}
}
static void dump_qstats_to_db(void) {
/* qstats hashtable */
char *kk;
qstats_t *s;
struct hashtable *qstats_htp = NULL;
qstats_htp = create_hashtable(64, hashfromkey, equalkeys);
assert(qstats_htp != NULL);
/* cp hashtable to stats table, this is very fast in-memory */
pthread_rwlock_rdlock(&qlist_ht_lock);
char *k;
queue_t *q;
int result;
struct hashtable_itr *itr = NULL;
itr = hashtable_iterator(qlist_htp);
assert(itr != NULL);
if (hashtable_count(qlist_htp) > 0)
{
do {
k = hashtable_iterator_key(itr);
q = hashtable_iterator_value(itr);
pthread_mutex_lock(&(q->lock));
if (q->old_set_hits == q->set_hits &&
q->old_get_hits == q->get_hits) {
pthread_mutex_unlock(&(q->lock));
continue;
}
q->old_set_hits = q->set_hits;
q->old_get_hits = q->get_hits;
pthread_mutex_unlock(&(q->lock));
kk = strdup(k);
assert(kk);
s = calloc(1, sizeof(qstats_t));
assert(s);
s->set_hits = q->old_set_hits;
s->get_hits = q->old_get_hits;
result = hashtable_insert(qstats_htp, (void *)kk, (void *)s);
assert(result != 0);
} while (hashtable_iterator_advance(itr));
}
free(itr);
itr = NULL;
pthread_rwlock_unlock(&qlist_ht_lock);
/* dump stats hashtable to db */
DBT dbkey, dbdata;
int ret;
DB_TXN *txnp = NULL;
ret = envp->txn_begin(envp, NULL, &txnp, 0);
CHECK_DB_RET(ret);
itr = hashtable_iterator(qstats_htp);
assert(itr != NULL);
if (hashtable_count(qstats_htp) > 0)
{
do {
kk = hashtable_iterator_key(itr);
s = hashtable_iterator_value(itr);
BDB_CLEANUP_DBT();
dbkey.data = (void *)kk;
dbkey.size = strlen(kk) + 1;
dbdata.data = (void *)s;
dbdata.size = sizeof(qstats_t);
ret = qlist_dbp->put(qlist_dbp, txnp, &dbkey, &dbdata, 0);
CHECK_DB_RET(ret);
fprintf(stderr, "dump stats[%s], set_hits: %lld, get_hits: %lld \n",
kk, s->set_hits, s->get_hits);
} while (hashtable_iterator_advance(itr));
}
free(itr);
itr = NULL;
ret = txnp->commit(txnp, 0);
CHECK_DB_RET(ret);
hashtable_destroy(qstats_htp, 1);
qstats_htp = NULL;
return;
dberr:
if(itr != NULL) {
free(itr);
}
if (qstats_htp != NULL) {
hashtable_destroy(qstats_htp, 1);
}
if (txnp != NULL){
txnp->abort(txnp);
}
if (settings.verbose > 1) {
fprintf(stderr, "dump_qstats_to_db: %s\n", db_strerror(ret));
}
}
