void bLSM::flushTable()
{
struct timeval start_tv, stop_tv;
double start, stop;
static double last_start;
static bool first = 1;
static int merge_count = 0;
gettimeofday(&start_tv,0);
start = tv_to_double(start_tv);
c0_flushing = true;
bool blocked = false;
int expmcount = merge_count;
//this waits for the previous merger of the mem-tree
//hopefullly this wont happen
while(get_c0_is_merging()) {
rwlc_cond_wait(&c0_needed, header_mut);
blocked = true;
if(expmcount != merge_count) {
return;
}
}
set_c0_is_merging(true);
merge_mgr->get_merge_stats(0)->handed_off_tree();
merge_mgr->new_merge(0);
gettimeofday(&stop_tv,0);
stop = tv_to_double(stop_tv);
pthread_cond_signal(&c0_ready);
DEBUG("Signaled c0-c1 merge thread\n");
merge_count ++;
merge_mgr->get_merge_stats(0)->starting_merge();
if(blocked && stop - start > 1.0) {
if(first)
{
printf("\nBlocked writes for %f sec\n", stop-start);
first = 0;
}
else
{
printf("\nBlocked writes for %f sec (serviced writes for %f sec)\n",
stop-start, start-last_start);
}
last_start = stop;
} else {
DEBUG("signaled c0-c1 merge\n");
}
c0_flushing = false;
}
void insertProbeIter(size_t NUM_ENTRIES)
{
srand(1000);
unlink("storefile.txt");
unlink("logfile.txt");
system("rm -rf stasis_log/");
bLSM::init_stasis();
//data generation
std::vector<std::string> * data_arr = new std::vector<std::string>;
std::vector<std::string> * key_arr = new std::vector<std::string>;
preprandstr(NUM_ENTRIES, data_arr, 10*8192, true);
preprandstr(NUM_ENTRIES+200, key_arr, 100, true);
std::sort(key_arr->begin(), key_arr->end(), &mycmp);
removeduplicates(key_arr);
scramble(key_arr);
if(key_arr->size() > NUM_ENTRIES)
key_arr->erase(key_arr->begin()+NUM_ENTRIES, key_arr->end());
NUM_ENTRIES=key_arr->size();
if(data_arr->size() > NUM_ENTRIES)
data_arr->erase(data_arr->begin()+NUM_ENTRIES, data_arr->end());
int xid = Tbegin();
bLSM * ltable = new bLSM(10 * 1024 * 1024, 1000, 10000, 5);
mergeScheduler mscheduler(ltable);
recordid table_root = ltable->allocTable(xid);
Tcommit(xid);
mscheduler.start();
printf("Stage 1: Writing %llu keys\n", (unsigned long long)NUM_ENTRIES);
struct timeval start_tv, stop_tv, ti_st, ti_end;
double insert_time = 0;
int64_t datasize = 0;
std::vector<pageid_t> dsp;
gettimeofday(&start_tv,0);
for(size_t i = 0; i < NUM_ENTRIES; i++)
{
//prepare the key
dataTuple *newtuple = dataTuple::create((*key_arr)[i].c_str(), (*key_arr)[i].length()+1,(*data_arr)[i].c_str(), (*data_arr)[i].length()+1);
/*
printf("key: \t, keylen: %u\ndata: datalen: %u\n",
//newtuple.key,
*newtuple.keylen,
//newtuple.data,
*newtuple.datalen);
*/
datasize += newtuple->byte_length();
gettimeofday(&ti_st,0);
ltable->insertTuple(newtuple);
gettimeofday(&ti_end,0);
insert_time += tv_to_double(ti_end) - tv_to_double(ti_st);
dataTuple::freetuple(newtuple);
}
gettimeofday(&stop_tv,0);
printf("insert time: %6.1f\n", insert_time);
printf("insert time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
printf("\nTREE STRUCTURE\n");
//ltable.get_tree_c1()->print_tree(xid);
printf("datasize: %lld\n", (long long)datasize);
//sleep(20);
xid = Tbegin();
printf("Stage 2: Looking up %llu keys:\n", (unsigned long long)NUM_ENTRIES);
int found_tuples=0;
for(int i=NUM_ENTRIES-1; i>=0; i--)
{
int ri = i;
//printf("key index%d\n", i);
fflush(stdout);
//get the key
uint32_t keylen = (*key_arr)[ri].length()+1;
dataTuple::key_t rkey = (dataTuple::key_t) malloc(keylen);
memcpy((byte*)rkey, (*key_arr)[ri].c_str(), keylen);
//for(int j=0; j<keylen-1; j++)
//rkey[j] = (*key_arr)[ri][j];
//rkey[keylen-1]='\0';
//find the key with the given tuple
dataTuple *dt = ltable->findTuple(xid, rkey, keylen);
assert(dt!=0);
//if(dt!=0)
{
found_tuples++;
assert(dt->rawkeylen() == (*key_arr)[ri].length()+1);
assert(dt->datalen() == (*data_arr)[ri].length()+1);
dataTuple::freetuple(dt);
}
dt = 0;
free(rkey);
}
printf("found %d\n", found_tuples);
key_arr->clear();
data_arr->clear();
delete key_arr;
delete data_arr;
mscheduler.shutdown();
printf("merge threads finished.\n");
gettimeofday(&stop_tv,0);
printf("run time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
Tcommit(xid);
delete ltable;
bLSM::deinit_stasis();
}
void insertProbeIter(size_t NUM_ENTRIES)
{
srand(1000);
unlink("storefile.txt");
unlink("logfile.txt");
system("rm -rf stasis_log/");
sync();
bLSM::init_stasis();
double delete_freq = .05;
double update_freq = .15;
//data generation
typedef std::vector<std::string> key_v_t;
const static size_t max_partition_size = 100000;
int KEY_LEN = 100;
std::vector<key_v_t*> *key_v_list = new std::vector<key_v_t*>;
int list_size = NUM_ENTRIES / max_partition_size + 1;
for(int i =0; i<list_size; i++)
{
key_v_t * key_arr = new key_v_t;
if(NUM_ENTRIES < max_partition_size*(i+1))
preprandstr(NUM_ENTRIES-max_partition_size*i, key_arr, KEY_LEN, true);
else
preprandstr(max_partition_size, key_arr, KEY_LEN, true);
std::sort(key_arr->begin(), key_arr->end(), &mycmp);
key_v_list->push_back(key_arr);
printf("size partition %llu is %llu\n", (unsigned long long)i+1, (unsigned long long)key_arr->size());
}
key_v_t * key_arr = new key_v_t;
std::vector<key_v_t::iterator*> iters;
for(int i=0; i<list_size; i++)
{
iters.push_back(new key_v_t::iterator((*key_v_list)[i]->begin()));
}
size_t lc = 0;
while(true)
{
int list_index = -1;
for(int i=0; i<list_size; i++)
{
if(*iters[i] == (*key_v_list)[i]->end())
continue;
if(list_index == -1 || mycmp(**iters[i], **iters[list_index]))
list_index = i;
}
if(list_index == -1)
break;
if(key_arr->size() == 0 || mycmp(key_arr->back(), **iters[list_index]))
key_arr->push_back(**iters[list_index]);
(*iters[list_index])++;
lc++;
if(lc % max_partition_size == 0)
printf("%llu/%llu completed.\n", (unsigned long long)lc, (unsigned long long)NUM_ENTRIES);
}
for(int i=0; i<list_size; i++)
{
(*key_v_list)[i]->clear();
delete (*key_v_list)[i];
delete iters[i];
}
key_v_list->clear();
delete key_v_list;
printf("key arr size: %llu\n", (unsigned long long)key_arr->size());
if(key_arr->size() > NUM_ENTRIES)
key_arr->erase(key_arr->begin()+NUM_ENTRIES, key_arr->end());
NUM_ENTRIES=key_arr->size();
int xid = Tbegin();
bLSM *ltable = new bLSM(10 * 1024 * 1024, 1000, 1000, 40);
mergeScheduler mscheduler(ltable);
recordid table_root = ltable->allocTable(xid);
Tcommit(xid);
mscheduler.start();
printf("Stage 1: Writing %llu keys\n", (unsigned long long)NUM_ENTRIES);
struct timeval start_tv, stop_tv, ti_st, ti_end;
double insert_time = 0;
int delcount = 0, upcount = 0;
int64_t datasize = 0;
std::vector<pageid_t> dsp;
std::vector<int> del_list;
gettimeofday(&start_tv,0);
for(size_t i = 0; i < NUM_ENTRIES; i++)
{
//prepare the data
std::string ditem;
getnextdata(ditem, 8192);
//prepare the key
dataTuple *newtuple = dataTuple::create((*key_arr)[i].c_str(), (*key_arr)[i].length()+1, ditem.c_str(), ditem.length()+1);
datasize += newtuple->byte_length();
gettimeofday(&ti_st,0);
ltable->insertTuple(newtuple);
gettimeofday(&ti_end,0);
insert_time += tv_to_double(ti_end) - tv_to_double(ti_st);
dataTuple::freetuple(newtuple);
double rval = ((rand() % 100)+.0)/100;
if( rval < delete_freq) //delete a key
{
int del_index = i - (rand()%50); //delete one of the last inserted 50 elements
if(del_index >= 0 && std::find(del_list.begin(), del_list.end(), del_index) == del_list.end())
{
delcount++;
dataTuple *deltuple = dataTuple::create((*key_arr)[del_index].c_str(), (*key_arr)[del_index].length()+1);
gettimeofday(&ti_st,0);
ltable->insertTuple(deltuple);
gettimeofday(&ti_end,0);
insert_time += tv_to_double(ti_end) - tv_to_double(ti_st);
dataTuple::freetuple(deltuple);
del_list.push_back(del_index);
}
}
else if(rval < delete_freq + update_freq) //update a record
{
int up_index = i - (rand()%50); //update one of the last inserted 50 elements
if(up_index >= 0 && std::find(del_list.begin(), del_list.end(), up_index) == del_list.end())
{//only update non-deleted elements
getnextdata(ditem, 512);
upcount++;
dataTuple *uptuple = dataTuple::create((*key_arr)[up_index].c_str(), (*key_arr)[up_index].length()+1,
ditem.c_str(), ditem.length()+1);
gettimeofday(&ti_st,0);
ltable->insertTuple(uptuple);
gettimeofday(&ti_end,0);
insert_time += tv_to_double(ti_end) - tv_to_double(ti_st);
dataTuple::freetuple(uptuple);
}
}
}
gettimeofday(&stop_tv,0);
printf("insert time: %6.1f\n", insert_time);
printf("insert time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
printf("#deletions: %d\n#updates: %d\n", delcount, upcount);
printf("\nTREE STRUCTURE\n");
printf("datasize: %llu\n", (unsigned long long)datasize);
xid = Tbegin();
printf("Stage 2: Looking up %llu keys:\n", (unsigned long long)NUM_ENTRIES);
int found_tuples=0;
for(int i=NUM_ENTRIES-1; i>=0; i--)
{
int ri = i;
//get the key
uint32_t keylen = (*key_arr)[ri].length()+1;
dataTuple::key_t rkey = (dataTuple::key_t) malloc(keylen);
memcpy((byte*)rkey, (*key_arr)[ri].c_str(), keylen);
//find the key with the given tuple
dataTuple *dt = ltable->findTuple(xid, rkey, keylen);
if(std::find(del_list.begin(), del_list.end(), i) == del_list.end())
{
assert(dt!=0);
assert(!dt->isDelete());
found_tuples++;
assert(dt->rawkeylen() == (*key_arr)[ri].length()+1);
dataTuple::freetuple(dt);
}
else
{
if(dt!=0)
{
assert(dt->rawkeylen() == (*key_arr)[ri].length()+1);
assert(dt->isDelete());
dataTuple::freetuple(dt);
}
}
dt = 0;
free(rkey);
}
printf("found %d\n", found_tuples);
key_arr->clear();
//data_arr->clear();
delete key_arr;
//delete data_arr;
mscheduler.shutdown();
printf("merge threads finished.\n");
gettimeofday(&stop_tv,0);
printf("run time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
Tcommit(xid);
delete ltable;
bLSM::deinit_stasis();
}
int main(int argc, char * argv[]) {
if(argc != 3) { usage(argv); return -1; }
char ** from_arg = (char**)malloc(2*sizeof(char*));
from_arg[0] = argv[0];
from_arg[1] = argv[1];
char ** to_arg = (char**)malloc(2*sizeof(char*));
to_arg[0] = argv[0];
to_arg[1] = argv[2];
logstore_handle_t * from = util_open_conn(2, from_arg);
logstore_handle_t * to = util_open_conn(2, to_arg);
uint8_t ret = logstore_client_op_returns_many(from, OP_SCAN, 0, 0, -2);
if(ret != LOGSTORE_RESPONSE_SENDING_TUPLES) {
perror("Open database scan failed"); return 3;
}
ret = logstore_client_op_returns_many(to, OP_BULK_INSERT);
if(ret != LOGSTORE_RESPONSE_RECEIVING_TUPLES) {
perror("Open bulk insert failed"); return 3;
}
long long num_tuples = 0;
long long size_copied = 0;
dataTuple *tup;
int bytes_per_dot = 10 * 1024 * 1024;
int dots_per_line = 50;
long long last_dot = 0;
struct timeval load_start_time;
gettimeofday(&load_start_time, 0);
while((tup = logstore_client_next_tuple(from))) {
ret = logstore_client_send_tuple(to, tup);
num_tuples ++;
size_copied += tup->byte_length();
dataTuple::freetuple(tup);
if(ret != LOGSTORE_RESPONSE_SUCCESS) {
perror("Send tuple failed"); return 3;
}
if(last_dot != size_copied / bytes_per_dot) {
printf("."); fflush(stdout);
last_dot = size_copied / bytes_per_dot;
if(last_dot % dots_per_line == 0) {
struct timeval line_stop_time;
gettimeofday(&line_stop_time,0);
double seconds = tv_to_double(line_stop_time) - tv_to_double(load_start_time);
printf("%6lldMB %6.1f s %6.2f mb/s %6.2f tuples/s\n", size_copied / 1024*1024, seconds, (double)size_copied/(1024.0 * 1024.0 * seconds), (double)num_tuples/seconds);
}
}
}
logstore_client_send_tuple(to,NULL);
if(ret != LOGSTORE_RESPONSE_SUCCESS) {
perror("Close bulk load failed"); return 3;
}
logstore_client_close(from);
logstore_client_close(to);
printf("Copy database done. %lld tuples, %lld bytes\n", (long long)num_tuples, (long long)size_copied);
return 0;
}
void get_cpu_usage(double &user, double &system) {
struct rusage usage;
getrusage(RUSAGE_SELF, &usage);
user = tv_to_double(usage.ru_utime);
system = tv_to_double(usage.ru_stime);
}
void insertProbeIter(size_t NUM_ENTRIES)
{
srand(1000);
logstore_handle_t * l = logstore_client_open(svrname, svrport, 100);
//data generation
typedef std::vector<std::string> key_v_t;
const static size_t max_partition_size = 100000;
int KEY_LEN = 100;
std::vector<key_v_t*> *key_v_list = new std::vector<key_v_t*>;
size_t list_size = NUM_ENTRIES / max_partition_size + 1;
for(size_t i =0; i<list_size; i++)
{
key_v_t * key_arr = new key_v_t;
if(NUM_ENTRIES < max_partition_size*(i+1))
preprandstr(NUM_ENTRIES-max_partition_size*i, key_arr, KEY_LEN, true);
else
preprandstr(max_partition_size, key_arr, KEY_LEN, true);
std::sort(key_arr->begin(), key_arr->end(), &mycmp);
key_v_list->push_back(key_arr);
printf("size partition %llu is %llu\n", (unsigned long long)i+1, (unsigned long long)key_arr->size());
}
key_v_t * key_arr = new key_v_t;
std::vector<key_v_t::iterator*> iters;
for(size_t i=0; i<list_size; i++)
{
iters.push_back(new key_v_t::iterator((*key_v_list)[i]->begin()));
}
int lc = 0;
while(true)
{
int list_index = -1;
for(size_t i=0; i<list_size; i++)
{
if(*iters[i] == (*key_v_list)[i]->end())
continue;
if(list_index == -1 || mycmp(**iters[i], **iters[list_index]))
list_index = i;
}
if(list_index == -1)
break;
if(key_arr->size() == 0 || mycmp(key_arr->back(), **iters[list_index]))
key_arr->push_back(**iters[list_index]);
(*iters[list_index])++;
lc++;
if(lc % max_partition_size == 0)
printf("%llu/%llu completed.\n", (unsigned long long)lc, (unsigned long long)NUM_ENTRIES);
}
for(size_t i=0; i<list_size; i++)
{
(*key_v_list)[i]->clear();
delete (*key_v_list)[i];
delete iters[i];
}
key_v_list->clear();
delete key_v_list;
printf("key arr size: %llu\n", (unsigned long long)key_arr->size());
if(key_arr->size() > NUM_ENTRIES)
key_arr->erase(key_arr->begin()+NUM_ENTRIES, key_arr->end());
NUM_ENTRIES=key_arr->size();
printf("Stage 1: Writing %llu keys\n", (unsigned long long)NUM_ENTRIES);
struct timeval start_tv, stop_tv, ti_st, ti_end;
double insert_time = 0;
int delcount = 0, upcount = 0;
int64_t datasize = 0;
std::vector<pageid_t> dsp;
std::vector<int> del_list;
gettimeofday(&start_tv,0);
for(size_t i = 0; i < NUM_ENTRIES; i++)
{
//prepare the key
len_t keylen = (*key_arr)[i].length()+1;
//prepare the data
std::string ditem;
getnextdata(ditem, 8192);
len_t datalen = ditem.length()+1;
dataTuple* newtuple = dataTuple::create((*key_arr)[i].c_str(), keylen,
ditem.c_str(), datalen);
datasize += newtuple->byte_length();
gettimeofday(&ti_st,0);
//send the data
dataTuple * ret = logstore_client_op(l, OP_INSERT, newtuple);
assert(ret);
gettimeofday(&ti_end,0);
insert_time += tv_to_double(ti_end) - tv_to_double(ti_st);
dataTuple::freetuple(newtuple);
if(i % 10000 == 0 && i > 0)
printf("%llu / %llu inserted.\n", (unsigned long long)i, (unsigned long long)NUM_ENTRIES);
}
gettimeofday(&stop_tv,0);
printf("insert time: %6.1f\n", insert_time);
printf("insert time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
printf("#deletions: %d\n#updates: %d\n", delcount, upcount);
printf("Stage 2: Looking up %llu keys:\n", (unsigned long long)NUM_ENTRIES);
int found_tuples=0;
for(int i=NUM_ENTRIES-1; i>=0; i--)
{
int ri = i;
//printf("key index%d\n", i);
//fflush(stdout);
//get the key
len_t keylen = (*key_arr)[ri].length()+1;
dataTuple* searchtuple = dataTuple::create((*key_arr)[ri].c_str(), keylen);
//find the key with the given tuple
dataTuple *dt = logstore_client_op(l, OP_FIND, searchtuple);
assert(dt!=0);
assert(!dt->isDelete());
found_tuples++;
assert(dt->rawkeylen() == (*key_arr)[ri].length()+1);
//free dt
dataTuple::freetuple(dt);
dt = 0;
dataTuple::freetuple(searchtuple);
}
printf("found %d\n", found_tuples);
printf("Stage 3: Initiating scan\n");
network_op_t ret = logstore_client_op_returns_many(l, OP_SCAN, NULL, NULL, 0); // start = NULL stop = NULL limit = NONE
assert(ret == LOGSTORE_RESPONSE_SENDING_TUPLES);
dataTuple * tup;
size_t i = 0;
while((tup = logstore_client_next_tuple(l))) {
assert(!tup->isDelete());
assert(tup->rawkeylen() == (*key_arr)[i].length()+1);
assert(!memcmp(tup->rawkey(), (*key_arr)[i].c_str(), (*key_arr)[i].length()));
dataTuple::freetuple(tup);
i++;
}
assert(i == NUM_ENTRIES);
key_arr->clear();
delete key_arr;
logstore_client_close(l);
gettimeofday(&stop_tv,0);
printf("run time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
}
