int test(void) {
test_pair_t one = { 1, "one" };
test_pair_t two = { 2, "two" };
test_pair_t three = { 3, "three" };
test_pair_t four = { 4, "four" };
int i;
int xid = Tbegin();
jbHashTable_t *ht = jbHtCreate(xid, 79);
if( jbHtInsert(xid, ht, &one.key, sizeof(int), one.value, sizeof(char)*4) ||
jbHtInsert(xid, ht, &two.key, sizeof(int), two.value, sizeof(char)*4) ||
jbHtInsert(xid, ht, &three.key, sizeof(int), three.value, sizeof(char)*6) ||
jbHtInsert(xid, ht, &four.key, sizeof(int), four.value, sizeof(char)*5)
) {
return -1;
}
Tcommit(xid);
xid = Tbegin();
for( i = 1; i <= 4; i++ ) {
char buf[7];
jbHtLookup(xid, ht, &i, sizeof(int), buf);
printf("key %u: %s\n", i, buf);
}
jbHtDelete(xid, ht);
Tcommit(xid);
return 0;
}
} END_TEST
START_TEST(regions_lockSmokeTest) {
Tinit();
int xid = Tbegin();
pageid_t pageid = TregionAlloc(xid, 100,0);
fsckRegions(xid);
Tcommit(xid);
xid = Tbegin();
int xid2 = Tbegin();
TregionDealloc(xid, pageid);
for(int i = 0; i < 50; i++) {
TregionAlloc(xid2, 1, 0);
}
fsckRegions(xid);
Tabort(xid);
fsckRegions(xid2);
Tcommit(xid2);
Tdeinit();
} END_TEST
}END_TEST
/** @test More complicated case of log w/ hard bound; many xacts at a time.
*/
START_TEST(boundedLogConcurrentTest) {
stasis_log_type = LOG_TO_MEMORY;
stasis_log_in_memory_max_entries = 1000;
stasis_log_softcommit = 1;
Tinit();
int xid = Tbegin();
pageid_t region_start = TregionAlloc(xid, NUM_XACTS, 0);
Tcommit(xid);
for(int64_t i = 0; i < NUM_XACTS; i++) {
int xids[NUM_CONCURRENT_XACTS];
for(int j = 0; j < NUM_CONCURRENT_XACTS; j++) {
xids[j] = Tbegin();
}
for(int j = 0; j < NUM_CONCURRENT_XACTS; j++) {
TinitializeFixedPage(xids[j], region_start + i, sizeof(uint64_t));
recordid rid = {region_start + i, 0, sizeof(uint64_t)};
Tset(xids[j], rid, &i);
i++;
}
for(int j = 0; j < NUM_CONCURRENT_XACTS; j++) {
Tcommit(xids[j]);
}
}
Tdeinit();
}END_TEST
int main(int argc, char** argv) {
assert(argc == 3);
int xact_count = atoi(argv[1]);
int count = atoi(argv[2]);
/* unlink("storefile.txt");
unlink("logfile.txt");
unlink("blob0_file.txt");
unlink("blob1_file.txt");*/
Tinit();
int xid = Tbegin();
recordid hash = TnaiveHashCreate(xid, sizeof(int), sizeof(int));
Tcommit(xid);
int i = 0;
int k;
for(k = 0; k < xact_count; k++) {
xid = Tbegin();
for(; i < (count*(k+1)) ; i++) {
TnaiveHashInsert(xid, hash, &i, sizeof(int), &i, sizeof(int));
}
Tcommit(xid);
}
Tdeinit();
}
static void*
threadFunc(void* arg_ptr) {
int j, k, startKey, endKey;
int xid, count = 0;
int bufferCurrentLength, bufferTotalSize;
char* insertBuffer;
/* Allocate the buffer that stores all outstanding hash table insertions. */
bufferTotalSize = BUFFER_INITIAL_LENGTH;
bufferCurrentLength = 0;
insertBuffer = stasis_malloc(bufferTotalSize, char);
xid = Tbegin();
k = (intptr_t) arg_ptr;
startKey = baseKey + (k * opsPerThread);
endKey = startKey + opsPerThread;
for (j = startKey; j < endKey; j++) {
ThashInsert(xid, hashTable, (byte*)&j, sizeof(int), (byte*)&j, sizeof(int));
writeKeyToBuffer(&insertBuffer, &bufferCurrentLength, &bufferTotalSize, j);
count++;
if ((count % opsPerTransaction) == 0) {
/* Prior to committing the transaction, write the hash table insertion buffer to
* the insert-log so that we can keep track of which insertions were possibly
* committed. After the Tcommit() call, write the insertion buffer to the commit-
* log so that we can keep track of which insertions were definitely committed.
*/
writeBufferToLog(insertLog, insertBuffer, bufferCurrentLength);
Tcommit(xid);
writeBufferToLog(commitLog, insertBuffer, bufferCurrentLength);
bufferCurrentLength = 0;
xid = Tbegin();
count = 0;
}
}
/* Prior to committing the transaction, write the hash table insertion buffer to
* the insert-log so that we can keep track of which insertions were possibly
* committed. After the Tcommit() call, write the insertion buffer to the commit-
* log so that we can keep track of which insertions were definitely committed.
*/
writeBufferToLog(insertLog, insertBuffer, bufferCurrentLength);
Tcommit(xid);
writeBufferToLog(commitLog, insertBuffer, bufferCurrentLength);
return NULL;
}
int test() {
int i;
int writeVal, readVal;
/** These need to be reset manually. */
int xid = 2;
recordid rec = { 0,0,sizeof(int) };
InitiateRecovery();
printf("\nRunning prepare testing commit.\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tcommit(xid);
xid = Tbegin();
printf("xid = %d\n", xid);
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tabort(xid);
exit(-1);
}
int test() {
unsigned int i, xid;
const unsigned int INSERT_NUM = 10000; /* should be > JB_HASHTABLE_SIZE */
char value[22]; /* should be log(INSERT_NUM)+8 */
jbHashTable_t *ht;
xid = Tbegin();
ht = jbHtCreate(xid, 1000);
for( i = 0; i < INSERT_NUM; i++ ) {
sprintf(value, "value: %u\n", i);
jbHtInsert(xid, ht, &i, sizeof(int), value, sizeof(char)*strlen(value));
}
Tcommit(xid);
xid = Tbegin();
for( i = 0; i < INSERT_NUM; i++ ) {
jbHtLookup(xid, ht, &i, sizeof(int), value);
printf("%s", value);
}
jbHtDelete(xid, ht);
Tabort(xid);
return 0;
}
int main(int argc, char * argv[]) {
Tinit();
ReferentialAlgebra_init();
recordid rootEntry;
recordid hash;
int xid = Tbegin();
if(TrecordType(xid, ROOT_RECORD) == INVALID_SLOT) {
printf("Creating new store\n");
rootEntry = Talloc(xid, sizeof(recordid));
assert(rootEntry.page == ROOT_RECORD.page);
assert(rootEntry.slot == ROOT_RECORD.slot);
hash = ReferentialAlgebra_allocContext(xid);
Tset(xid, rootEntry, &hash);
} else {
printf("Opened existing store\n");
rootEntry.page = ROOT_RECORD.page;
rootEntry.slot = ROOT_RECORD.slot;
rootEntry.size = sizeof(recordid);
Tread(xid, rootEntry, &hash);
}
context = ReferentialAlgebra_openContext(xid,hash);
Tcommit(xid);
FILE * in;
if(argc == 3) { // listen on socket
if(strcmp("--socket", argv[1])) {
printf("usage:\n\n%s\n%s <filename>\n%s --socket addr:port\n",
argv[0],argv[0],argv[0]);
} else {
startServer(argv[2], hash);
}
printf("Shutting down...\n");
} else {
if(argc == 2) {
in = fopen(argv[1], "r");
if(!in) {
printf("Couldn't open input file.\n");
return 1;
}
} else {
in = stdin;
}
openInterpreter(in, stdout, hash);
}
Tdeinit();
}
int main(int argc, char** argv) {
assert(argc == 3 || argc == 4);
int thread_count = atoi(argv[1]);
count = atoi(argv[2]);
alwaysCommit = (argc==4);
unlink("storefile.txt");
unlink("logfile.txt");
unlink("blob0_file.txt");
unlink("blob1_file.txt");
pthread_t * workers = stasis_malloc(thread_count, pthread_t);
Tinit();
int xid = Tbegin();
// hash = ThashCreate(xid, sizeof(int), sizeof(int));
hash = ThashCreate(xid, VARIABLE_LENGTH, VARIABLE_LENGTH);
Tcommit(xid);
int k;
/* threads have static thread sizes. Ughh. */
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_mutex_init(&mutex, NULL);
pthread_attr_setstacksize (&attr, PTHREAD_STACK_MIN);
// pthread_attr_setschedpolicy(&attr, SCHED_FIFO);
pthread_mutex_lock(&mutex);
for(k = 0; k < thread_count; k++) {
int * k_copy = stasis_alloc(int);
*k_copy = k ;
pthread_create(&workers[k], &attr, go, k_copy);
}
pthread_mutex_unlock(&mutex);
for(k = 0; k < thread_count; k++) {
pthread_join(workers[k],NULL);
}
Tdeinit();
printf("Committed %d times, put %d times.\n", commitCount, putCount);
}
int test(int argc, char **argv) {
int xid;
int writeVals[NUM_TRIALS];
int readVals[NUM_TRIALS];
recordid recs[NUM_TRIALS];
int i;
int failed = 0;
printf("\nRunning %s\n", __FILE__);
xid = Tbegin();
for (i = 0; i < NUM_TRIALS; i++) {
writeVals[i] = rand();
recs[i] = Talloc(xid, sizeof(int));
Tset(xid, recs[i], &writeVals[i]);
}
Tcommit(xid);
xid = Tbegin();
for (i = 0; i < NUM_TRIALS; i++) {
Tread(xid, recs[i], &readVals[i]);
if (VERBOSE)
printf("value set is %d, value read is %d\n", writeVals[i], readVals[i]);
if (writeVals[i] != readVals[i])
failed = 1;
}
Tcommit(xid);
printf("%s\n\n", failed ? "****FAILED****" : "PASSED");
return failed;
}
void insertProbeIter(lsmkey_t NUM_ENTRIES) {
int intcmp = 0;
lsmTreeRegisterComparator(intcmp,cmp);
TlsmRegionAllocConf_t alloc_conf = LSM_REGION_ALLOC_STATIC_INITIALIZER;
stasis_page_impl_register(lsmRootImpl());
Tinit();
int xid = Tbegin();
recordid tree = TlsmCreate(xid, intcmp,
TlsmRegionAlloc, &alloc_conf,
sizeof(lsmkey_t));
long oldpagenum = -1;
for(lsmkey_t i = 0; i < NUM_ENTRIES; i++) {
long pagenum = TlsmFindPage(xid, tree, (byte*)&i);
assert(pagenum == -1 || pagenum == oldpagenum || oldpagenum == -1);
DEBUG("TlsmAppendPage %d\n",i);
TlsmAppendPage(xid, tree, (const byte*)&i, TlsmRegionAlloc, &alloc_conf, i + OFFSET);
pagenum = TlsmFindPage(xid, tree, (byte*)&i);
oldpagenum = pagenum;
assert(pagenum == i + OFFSET);
}
for(lsmkey_t i = 0; i < NUM_ENTRIES; i++) {
long pagenum = TlsmFindPage(xid, tree, (byte*)&i);
assert(pagenum == i + OFFSET);
}
int64_t count = 0;
lladdIterator_t * it = lsmTreeIterator_open(xid, tree);
while(lsmTreeIterator_next(xid, it)) {
lsmkey_t * key;
lsmkey_t **key_ptr = &key;
int size = lsmTreeIterator_key(xid, it, (byte**)key_ptr);
assert(size == sizeof(lsmkey_t));
long *value;
long **value_ptr = &value;
size = lsmTreeIterator_value(xid, it, (byte**)value_ptr);
assert(size == sizeof(pageid_t));
assert(*key + OFFSET == *value);
assert(*key == count);
count++;
}
assert(count == NUM_ENTRIES);
lsmTreeIterator_close(xid, it);
Tcommit(xid);
Tdeinit();
}
int test()
{
struct timeval start, end, total;
recordid rids[TESTS];
long commited[TESTS];
long aborted[TESTS];
int i, j, xid = Tbegin();
for( i = 0; i < TESTS; i++ ) {
rids[i] = Talloc(xid, sizeof(long));
commited[i] = random();
aborted[i] = random();
Tset(xid, rids[i], &commited[i]);
}
gettimeofday(&start, NULL);
Tcommit(xid);
gettimeofday(&end, NULL);
timersub(&end, &start, &total);
printf("commit %u: %ld.%ld\n", TESTS, total.tv_sec, total.tv_usec);
for( j = 0; j < 10; j++ ) {
xid = Tbegin();
for( i = 0; i < TESTS*j; i++ ) {
Tset(xid, rids[random() % TESTS], &aborted[i%TESTS]);
}
gettimeofday(&start, NULL);
Tcommit(xid);
gettimeofday(&end, NULL);
timersub(&end, &start, &total);
printf("commit %u: %ld.%ld\n", TESTS*j, total.tv_sec, total.tv_usec);
}
return 0;
}
} END_TEST
START_TEST(regions_recoveryTest) {
Tinit();
pageid_t pages[50];
int xid1 = Tbegin();
int xid2 = Tbegin();
for(int i = 0; i < 50; i+=2) {
pages[i] = TregionAlloc(xid1, stasis_util_random64(4)+1, 0);
pages[i+1] = TregionAlloc(xid2, stasis_util_random64(2)+1, 0);
}
fsckRegions(xid1);
Tcommit(xid1);
Tdeinit();
if(TdurabilityLevel() == VOLATILE) { return; }
Tinit();
int xid = Tbegin();
fsckRegions(xid);
for(int i = 0; i < 50; i+=2) {
TregionDealloc(xid, pages[i]);
}
fsckRegions(xid);
Tabort(xid);
fsckRegions(Tbegin());
Tdeinit();
Tinit();
fsckRegions(Tbegin());
Tdeinit();
} END_TEST
void insertProbeIter(size_t NUM_ENTRIES)
{
srand(1000);
unlink("storefile.txt");
unlink("logfile.txt");
system("rm -rf stasis_log/");
sync();
bLSM::init_stasis();
int xid = Tbegin();
Tcommit(xid);
xid = Tbegin();
mergeManager merge_mgr(0);
mergeStats * stats = merge_mgr.get_merge_stats(1);
diskTreeComponent *ltable_c1 = new diskTreeComponent(xid, 1000, 10000, 5, stats);
std::vector<std::string> data_arr;
std::vector<std::string> key_arr;
preprandstr(NUM_ENTRIES, data_arr, 5*4096, true);
preprandstr(NUM_ENTRIES+200, key_arr, 50, true);//well i can handle upto 200
std::sort(key_arr.begin(), key_arr.end(), &mycmp);
removeduplicates(key_arr);
if(key_arr.size() > NUM_ENTRIES)
key_arr.erase(key_arr.begin()+NUM_ENTRIES, key_arr.end());
NUM_ENTRIES=key_arr.size();
if(data_arr.size() > NUM_ENTRIES)
data_arr.erase(data_arr.begin()+NUM_ENTRIES, data_arr.end());
printf("Stage 1: Writing %llu keys\n", (unsigned long long)NUM_ENTRIES);
for(size_t i = 0; i < NUM_ENTRIES; i++)
{
//prepare the tuple
dataTuple* newtuple = dataTuple::create(key_arr[i].c_str(), key_arr[i].length()+1, data_arr[i].c_str(), data_arr[i].length()+1);
ltable_c1->insertTuple(xid, newtuple);
dataTuple::freetuple(newtuple);
}
printf("\nTREE STRUCTURE\n");
ltable_c1->print_tree(xid);
printf("Writes complete.\n");
ltable_c1->writes_done();
Tcommit(xid);
xid = Tbegin();
printf("Stage 2: Sequentially reading %llu tuples\n", (unsigned long long)NUM_ENTRIES);
size_t tuplenum = 0;
diskTreeComponent::iterator * tree_itr = ltable_c1->open_iterator();
dataTuple *dt=0;
while( (dt=tree_itr->next_callerFrees()) != NULL)
{
assert(dt->rawkeylen() == key_arr[tuplenum].length()+1);
assert(dt->datalen() == data_arr[tuplenum].length()+1);
tuplenum++;
dataTuple::freetuple(dt);
dt = 0;
}
delete(tree_itr);
assert(tuplenum == key_arr.size());
printf("Sequential Reads completed.\n");
int rrsize=key_arr.size() / 3;
printf("Stage 3: Randomly reading %d tuples by key\n", rrsize);
for(int i=0; i<rrsize; i++)
{
//randomly pick a key
int ri = rand()%key_arr.size();
dataTuple *dt = ltable_c1->findTuple(xid, (const dataTuple::key_t) key_arr[ri].c_str(), (size_t)key_arr[ri].length()+1);
assert(dt!=0);
assert(dt->rawkeylen() == key_arr[ri].length()+1);
assert(dt->datalen() == data_arr[ri].length()+1);
dataTuple::freetuple(dt);
dt = 0;
}
printf("Random Reads completed.\n");
Tcommit(xid);
bLSM::deinit_stasis();
}
int test() {
int xid;
int NUM_TRIALS=100;
recordid rec;
/* counters */
int i;
char c;
/* used for making the random strings*/
double r;
int start, end;
/* used to make a long string */
char first_printable_ascii = ' ';
char last_printable_ascii = '~';
int ascii_length = (int)(last_printable_ascii - first_printable_ascii);
char *ASCII = (char *)malloc(sizeof(char) * ascii_length);
/* an array of strings */
char **strings = (char **)malloc(sizeof(char*) * NUM_TRIALS);
/* output buffer */
char *out_buffer = (char *)malloc(sizeof(char) * ascii_length);
srand(SEED);
/* create the long string from which random strings will be made */
for (c = 0; c < ascii_length; c++) {
ASCII[(int)c] = first_printable_ascii + c;
}
for (i = 0; i < NUM_TRIALS; i++) {
r = ((double)rand()/(double)((double)RAND_MAX+1)); /* returns [0, 1)*/
r = r*ascii_length;
start = (int)r; /* an int in the rand [0, ascii_length) */
r = ((double)rand()/(double)((double)RAND_MAX+1)); /* returns [0, 1)*/
r = r*ascii_length;
end = (int)r; /* an int in the rand [0, ascii_length) */
if (start == end) {
i--;
continue;
}
if (end < start) {
int swap = start;
start = end;
end = swap;
}
strings[i] = (char *)malloc(sizeof(char) * (end - start) + 1);
strncpy(strings[i], ASCII + start, end-start);
strings[i][end-start-1] = '\0'; /* make the string null terminated */
}
for (i = 0; i < NUM_TRIALS; i++) {
xid = Tbegin();
rec = Talloc(xid, sizeof(char)*(strlen(strings[i]) +1));
Tset(xid, rec, strings[i]);
Tread(xid, rec, out_buffer);
if (VERBOSE) {
printf("xid %d set rid (%d, %d) to [%s], and read [%s]\n", xid, rec.page, rec.slot, strings[i], out_buffer);
}
assert(strcmp(strings[i], out_buffer) == 0);
Tcommit(xid);
}
return 0;
}
int
runSubject(void) {
pthread_t* threads;
pthread_attr_t attr;
intptr_t k;
int xid, fd;
/* Need the following sleep call for debugging. */
//sleep(45);
printf("\tRunning Subject Process\n"); fflush(stdout);
readGlobalsFromSharedBuffer();
initHashTable = 1;
Tinit();
if (iteration == 0) {
/* Create the transactional hash table data structure. */
xid = Tbegin();
hashTable = ThashCreate(xid, sizeof(int), sizeof(int));
Tcommit(xid);
/* Write the hash table recordid to a file, so it can be
* reloaded during subsequent iterations.
*/
fd = open("recordid.txt", O_CREAT | O_WRONLY | O_SYNC, 0777);
write(fd, (char*) &hashTable, sizeof(recordid));
close(fd);
}
else {
/* Otherwise, read in the hash table from disk. */
fd = open("recordid.txt", O_RDONLY, 0777);
read(fd, (char*) &hashTable, sizeof(recordid));
close(fd);
}
initHashTable = 0;
/* Open the insert and commit log files. The insert-log keeps track of all insertions
* that were made to the hash-table, not all of which may have been committed. The
* commit-log keeps track of all insertions that were definitely committed.
*/
insertLog = open("inserts.txt", O_CREAT | O_RDWR | O_APPEND, 0777);
commitLog = open("commits.txt", O_CREAT | O_RDWR | O_APPEND, 0777);
/* Allocate the worker threads. */
threads = stasis_malloc(numThreads, pthread_t);
pthread_attr_init(&attr);
pthread_attr_setstacksize (&attr, PTHREAD_STACK_MIN);
pthread_mutex_init(&hashTableMutex, NULL);
for (k = 0; k < numThreads; k++) {
pthread_create(&threads[k], &attr, threadFunc, (void*) k);
}
for (k = 0; k < numThreads; k++) {
pthread_join(threads[k], NULL);
}
/* Close the insert and commit log files. */
close(insertLog);
close(commitLog);
Tdeinit();
printf("\t\tSubject Process Exiting Normally\n"); fflush(stdout);
return 0;
}
int test() {
int xid;
char *handle_in = "jkit";
char *name_in = "Jimmy Kittiyachavalit";
int phone_in = 8821417;
char *handle_out = (char *)malloc(strlen(handle_in)+1);
char *name_out = (char *)malloc(strlen(name_in)+1);
int phone_out;
recordid r_name, r_handle, r_phone;
xid = Tbegin();
r_handle = Talloc(xid, strlen(handle_in)+1);
r_name = Talloc(xid, strlen(name_in)+1);
r_phone = Talloc(xid, sizeof(int));
Tset(xid, r_handle, handle_in);
printf("set handle to [%s]\n", handle_in);
Tset(xid, r_name, name_in);
printf("set name to [%s]\n", name_in);
Tset(xid, r_phone, &phone_in);
printf("set name to [%d]\n", phone_in);
Tread(xid, r_handle, handle_out);
printf("read handle is [%s]\n", handle_out);
Tread(xid, r_name, name_out);
printf("read name is [%s]\n", name_out);
Tread(xid, r_phone, &phone_out);
printf("read name is [%d]\n", phone_out);
Tcommit(xid);
xid = Tbegin();
handle_in = "tikj";
name_in = "tilavahcayittik ymmij";
phone_in = 3142116;
handle_out = (char *)malloc(strlen(handle_in)+1);
name_out = (char *)malloc(strlen(name_in)+1);
phone_out = 0;
Tset(xid, r_handle, handle_in);
printf("set handle to [%s]\n", handle_in);
Tset(xid, r_name, name_in);
printf("set name to [%s]\n", name_in);
Tset(xid, r_phone, &phone_in);
printf("set name to [%d]\n", phone_in);
Tread(xid, r_handle, handle_out);
printf("read handle is [%s]\n", handle_out);
Tread(xid, r_name, name_out);
printf("read name is [%s]\n", name_out);
Tread(xid, r_phone, &phone_out);
printf("read name is [%d]\n", phone_out);
printf("aborted the transaction\n");
Tabort(xid);
xid = Tbegin();
handle_out = (char *)malloc(strlen(handle_in)+1);
name_out = (char *)malloc(strlen(name_in)+1);
phone_out = 0;
Tread(xid, r_handle, handle_out);
printf("read handle is [%s]\n", handle_out);
Tread(xid, r_name, name_out);
printf("read name is [%s]\n", name_out);
Tread(xid, r_phone, &phone_out);
printf("read name is [%d]\n", phone_out);
Tcommit(xid);
return 0;
}
int openInterpreter(FILE * in, FILE * out, recordid hash) {
char * line = NULL;
size_t len = 0;
int AUTOCOMMIT = 1;
int debug = 0;
int xid = -1;
size_t read;
fprintf(out, "> ");
int ret = 0;
if(!AUTOCOMMIT) { xid = Tbegin(); }
while((read = getline(&line, &len, in)) != -1) {
if(line[0] == '!') {
if(!strncmp(line+1,"debug",strlen("debug"))) {
debug = !debug;
if(debug)
fprintf(out, "Enabling debugging\n");
else
fprintf(out, "Disabling debugging\n");
} else if(!strncmp(line+1,"regions",strlen("regions"))) {
fprintf(out, "Boundary tag pages:\n");
pageid_t pid = REGION_FIRST_TAG;
boundary_tag tag;
TregionReadBoundaryTag(-1,pid,&tag);
int done = 0;
while(!done) {
fprintf(out, "\tpageid=%lld\ttype=%d\tsize=%lld\n", pid, tag.allocation_manager, tag.size);
if(tag.size == UINT32_MAX) { fprintf(out, "\t[EOF]\n"); }
int err = TregionNextBoundaryTag(-1,&pid,&tag,0);
if(!err) { done = 1; }
}
} else if(!strncmp(line+1,"autocommit",strlen("autocommit"))) {
if(AUTOCOMMIT) {
// we're not in a transaction
fprintf(out, "Disabling autocommit\n");
AUTOCOMMIT = 0;
xid = Tbegin();
} else {
fprintf(out, "Enabling autocommit\n");
AUTOCOMMIT = 1;
Tcommit(xid);
}
/* } else if(!strncmp(line+1,"parseTuple",strlen("parseToken"))) {
char * c = line + 1 + strlen("parseToken");
char ** toks = parseTuple(&c);
for(int i = 0; toks[i]; i++) {
fprintf(out, "col %d = ->%s<-\n", i, toks[i]);
}
fprintf(out, "trailing stuff: %s", c);
} else if(!strncmp(line+1,"parseExpression",
strlen("parseExpression"))) {
char * c = line + 1 + strlen("parseExpression");
lladdIterator_t * it = parseExpression(xid, hash, &c);
it = 0; */
} else if(!strncmp(line+1,"exit",strlen("exit"))) {
break;
} else if(!strncmp(line+1,"shutdown",strlen("shutdown"))) {
ret = SHUTDOWN_SERVER;
break;
}
} else {
expr_list * results = 0;
parse(line, &results);
for(int i = 0; results && i < results->count; i++) {
expr * e = results->ents[i];
switch(e->typ) {
case query_typ: {
lladdIterator_t * it = ReferentialAlgebra_ExecuteQuery(xid, context, e->u.q);
if(it) {
while(Titerator_next(xid,it)) {
byte * tup;
Titerator_value(xid,it, &tup);
char * tupleString = stringTuple(*(tuple_t*)tup);
fprintf(out, "%s\n", tupleString);
free(tupleString);
}
Titerator_close(xid,it);
}
} break;
case insert_typ: {
if(AUTOCOMMIT) { xid = Tbegin(); }
ReferentialDML_ExecuteInsert(xid, context, e->u.i);
if(AUTOCOMMIT) { Tcommit(xid); xid = -1;}
} break;
case delete_typ: {
if(AUTOCOMMIT) { xid = Tbegin(); }
ReferentialDML_ExecuteDelete(xid, context, e->u.d);
if(AUTOCOMMIT) { Tcommit(xid); xid = -1;}
} break;
case create_typ: {
if(AUTOCOMMIT) { xid = Tbegin(); }
ReferentialDDL_CreateTable(xid, context, e->u.c);
if(AUTOCOMMIT) { xid = Tcommit(xid); xid = -1;}
} break;
default:
abort();
}
}
//XXX typecheck(context, results);
if(results) {
char * str = pp_expr_list(results);
printf("%s", str);
free(str);
} else {
printf("No results\n");
}
}
fprintf(out, "> ");
}
if(!AUTOCOMMIT) { Tcommit(xid); }
free(line);
fprintf(out, "\n");
return ret;
}
JNIEXPORT void JNICALL Java_stasis_jni_Stasis_commit
(JNIEnv *e, jclass c, jlong xid) {
printf("commiting %d\n", xid);
Tcommit((int)xid);
}
int test() {
int i;
int xid, writeVal, readVal;
recordid rec;
printf("\nRunning test1\n");
writeVal = 10;
xid = Tbegin();
rec = Talloc(xid, sizeof(int));
Tset(xid, rec, &writeVal);
Tread(xid, rec, &readVal);
printf("value set is %d, value read is %d\n", writeVal, readVal);
Tcommit(xid);
xid = Tbegin();
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tincrement(xid, rec);
printf("calling Tincrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tincrement(xid, rec);
printf("calling Tincrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tabort(xid);
printf("aborting\n");
xid = Tbegin();
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tincrement(xid, rec);
printf("calling Tincrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tdecrement(xid, rec);
printf("calling Tdecrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tdecrement(xid, rec);
printf("calling Tdecrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tdecrement(xid, rec);
printf("calling Tdecrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
for (i = 0; i < 10; i++) {
Tdecrement(xid, rec);
printf("calling Tdecrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
}
Tcommit(xid);
printf("committing\n");
xid = Tbegin();
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tdecrement(xid, rec);
printf("calling Tdecrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tincrement(xid, rec);
printf("calling Tincrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
Tdecrement(xid, rec);
printf("calling Tdecrement\n");
Tread(xid, rec, &readVal);
printf("value read is %d\n", readVal);
printf("\n");
return 0;
}
} END_TEST
START_TEST(regions_lockRandomizedTest) {
Tinit();
const int NUM_XACTS = 100;
const int NUM_OPS = 10000;
const int FUDGE = 10;
int xids[NUM_XACTS];
pageid_t * xidRegions[NUM_XACTS + FUDGE];
int xidRegionCounts[NUM_XACTS + FUDGE];
int longXid = Tbegin();
time_t seed = time(0);
printf("\nSeed = %ld\n", seed);
srandom(seed);
for(int i = 0; i < NUM_XACTS; i++) {
xids[i] = Tbegin();
assert(xids[i] < NUM_XACTS + FUDGE);
xidRegions[xids[i]] = stasis_malloc(NUM_OPS, pageid_t);
xidRegionCounts[xids[i]] = 0;
}
int activeXacts = NUM_XACTS;
for(int i = 0; i < NUM_OPS; i++) {
pageid_t j;
if(!(i % (NUM_OPS/NUM_XACTS))) {
// abort or commit one transaction randomly.
activeXacts --;
j = stasis_util_random64(activeXacts);
if(stasis_util_random64(2)) {
Tcommit(xids[j]);
} else {
Tabort(xids[j]);
}
if(activeXacts == 0) {
break;
}
for(; j < activeXacts; j++) {
xids[j] = xids[j+1];
}
fsckRegions(longXid);
}
j = stasis_util_random64(activeXacts);
if(stasis_util_random64(2)) {
// alloc
xidRegions[xids[j]][xidRegionCounts[xids[j]]] = TregionAlloc(xids[j], stasis_util_random64(100), 0);
xidRegionCounts[xids[j]]++;
} else {
// free
if(xidRegionCounts[xids[j]]) {
pageid_t k = stasis_util_random64(xidRegionCounts[xids[j]]);
TregionDealloc(xids[j], xidRegions[xids[j]][k]);
xidRegionCounts[xids[j]]--;
for(; k < xidRegionCounts[xids[j]]; k++) {
xidRegions[xids[j]][k] = xidRegions[xids[j]][k+1];
}
}
}
}
for(int i = 0; i < activeXacts; i++) {
Tabort(i);
fsckRegions(longXid);
}
Tcommit(longXid);
Tdeinit();
} END_TEST
int test(int argc, char **argv) {
int xids[NUM_TRIALS];
int initVals[NUM_TRIALS];
int writeVals[NUM_TRIALS];
int readVals[NUM_TRIALS];
int commits[NUM_TRIALS];
int i;
recordid recs[NUM_TRIALS];
int failed = 0;
printf("\nRunning %s\n", __FILE__);
for (i = 0; i < NUM_TRIALS; i++) {
xids[i] = Tbegin();
initVals[i] = rand();
recs[i] = Talloc(xids[i], sizeof(int));
Tset(xids[i], recs[i], &initVals[i]);
Tcommit(xids[i]);
}
for (i = 0; i < NUM_TRIALS; i++) {
xids[i] = Tbegin();
commits[i] = 0;
writeVals[i] = rand();
}
for (i = 0; i < NUM_TRIALS; i++) {
Tset(xids[i], recs[i], &writeVals[i]);
}
for (i = 0; i < NUM_TRIALS; i++) {
if (rand() % 2) {
Tcommit(xids[i]);
commits[i] = 1;
} else {
Tabort(xids[i]);
}
}
for (i = 0; i < NUM_TRIALS; i++) {
Tread(xids[i], recs[i], &readVals[i]);
}
for (i = 0; i < NUM_TRIALS; i++) {
if (VERBOSE) {
if (commits[i])
printf("xid %d commited value %d, and read %d\n", xids[i], writeVals[i], readVals[i]);
else
printf("xid %d aborted while setting value %d, and read %d and should've read %d\n", xids[i], writeVals[i], readVals[i], initVals[i]);
}
if (commits[i]) {
if (writeVals[i] != readVals[i])
failed = 1;
} else {
if (initVals[i] != readVals[i])
failed = 1;
}
}
printf("%s\n\n", failed ? "****FAILED****" : "PASSED");
return failed;
}
END_TEST
START_TEST(regions_randomizedTest) {
Tinit();
time_t seed = time(0);
printf("Seed = %ld: ", seed);
srandom(seed);
int xid = Tbegin();
pageid_t pagesAlloced = 0;
pageid_t regionsAlloced = 0;
double max_blowup = 0;
pageid_t max_region_count = 0;
pageid_t max_waste = 0;
pageid_t max_size = 0;
pageid_t max_ideal_size = 0;
for(int i = 0; i < 10000; i++) {
if(!(i % 100)) {
Tcommit(xid);
xid = Tbegin();
}
if(!(i % 100)) {
fsckRegions(xid);
}
if(stasis_util_random64(2)) {
unsigned int size = stasis_util_random64(100);
TregionAlloc(xid, size, 0);
pagesAlloced += size;
regionsAlloced ++;
} else {
if(regionsAlloced) {
pageid_t victim = stasis_util_random64(regionsAlloced);
pageid_t victimSize;
pageid_t victimPage;
TregionFindNthActive(xid, victim, &victimPage, &victimSize);
TregionDealloc(xid, victimPage);
pagesAlloced -= victimSize;
regionsAlloced --;
} else {
i--;
}
}
if(regionsAlloced) {
pageid_t lastRegionStart;
pageid_t lastRegionSize;
TregionFindNthActive(xid, regionsAlloced-1, &lastRegionStart, &lastRegionSize);
pageid_t length = lastRegionStart + lastRegionSize+1;
pageid_t ideal = pagesAlloced + regionsAlloced + 1;
double blowup = (double)length/(double)ideal;
unsigned long long bytes_wasted = length - ideal;
// printf("Region count = %d, blowup = %d / %d = %5.2f\n", regionsAlloced, length, ideal, blowup);
if(max_blowup < blowup) {
max_blowup = blowup;
}
if(max_waste < bytes_wasted) {
max_waste = bytes_wasted;
}
if(max_size < length) {
max_size = length;
}
if(max_ideal_size < ideal) {
max_ideal_size = ideal;
}
if(max_region_count < regionsAlloced) {
max_region_count = regionsAlloced;
}
}
}
fsckRegions(xid);
Tcommit(xid);
Tdeinit();
printf("Max # of regions = %lld, page file size = %5.2fM, ideal page file size = %5.2fM, (blowup = %5.2f)\n",
//peak bytes wasted = %5.2fM, blowup = %3.2f\n",
max_region_count,
((double)max_size * PAGE_SIZE)/(1024.0*1024.0),
((double)max_ideal_size * PAGE_SIZE)/(1024.0*1024.0),
(double)max_size/(double)max_ideal_size);
// ((double)max_waste * PAGE_SIZE)/(1024.0*1024.0),
// max_blowup);
if((double)max_size/(double)max_ideal_size > 5) {
// max_blowup isn't what we want here; it measures the peak
// percentage of the file that is unused. Instead, we want to
// measure the actual and ideal page file sizes for this run.
printf("ERROR: Excessive blowup (max allowable is 5)\n");
abort();
}
} END_TEST
void insertProbeIter(size_t NUM_ENTRIES)
{
srand(1000);
unlink("storefile.txt");
unlink("logfile.txt");
system("rm -rf stasis_log/");
bLSM::init_stasis();
//data generation
std::vector<std::string> * data_arr = new std::vector<std::string>;
std::vector<std::string> * key_arr = new std::vector<std::string>;
preprandstr(NUM_ENTRIES, data_arr, 10*8192, true);
preprandstr(NUM_ENTRIES+200, key_arr, 100, true);
std::sort(key_arr->begin(), key_arr->end(), &mycmp);
removeduplicates(key_arr);
scramble(key_arr);
if(key_arr->size() > NUM_ENTRIES)
key_arr->erase(key_arr->begin()+NUM_ENTRIES, key_arr->end());
NUM_ENTRIES=key_arr->size();
if(data_arr->size() > NUM_ENTRIES)
data_arr->erase(data_arr->begin()+NUM_ENTRIES, data_arr->end());
int xid = Tbegin();
bLSM * ltable = new bLSM(10 * 1024 * 1024, 1000, 10000, 5);
mergeScheduler mscheduler(ltable);
recordid table_root = ltable->allocTable(xid);
Tcommit(xid);
mscheduler.start();
printf("Stage 1: Writing %llu keys\n", (unsigned long long)NUM_ENTRIES);
struct timeval start_tv, stop_tv, ti_st, ti_end;
double insert_time = 0;
int64_t datasize = 0;
std::vector<pageid_t> dsp;
gettimeofday(&start_tv,0);
for(size_t i = 0; i < NUM_ENTRIES; i++)
{
//prepare the key
dataTuple *newtuple = dataTuple::create((*key_arr)[i].c_str(), (*key_arr)[i].length()+1,(*data_arr)[i].c_str(), (*data_arr)[i].length()+1);
/*
printf("key: \t, keylen: %u\ndata: datalen: %u\n",
//newtuple.key,
*newtuple.keylen,
//newtuple.data,
*newtuple.datalen);
*/
datasize += newtuple->byte_length();
gettimeofday(&ti_st,0);
ltable->insertTuple(newtuple);
gettimeofday(&ti_end,0);
insert_time += tv_to_double(ti_end) - tv_to_double(ti_st);
dataTuple::freetuple(newtuple);
}
gettimeofday(&stop_tv,0);
printf("insert time: %6.1f\n", insert_time);
printf("insert time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
printf("\nTREE STRUCTURE\n");
//ltable.get_tree_c1()->print_tree(xid);
printf("datasize: %lld\n", (long long)datasize);
//sleep(20);
xid = Tbegin();
printf("Stage 2: Looking up %llu keys:\n", (unsigned long long)NUM_ENTRIES);
int found_tuples=0;
for(int i=NUM_ENTRIES-1; i>=0; i--)
{
int ri = i;
//printf("key index%d\n", i);
fflush(stdout);
//get the key
uint32_t keylen = (*key_arr)[ri].length()+1;
dataTuple::key_t rkey = (dataTuple::key_t) malloc(keylen);
memcpy((byte*)rkey, (*key_arr)[ri].c_str(), keylen);
//for(int j=0; j<keylen-1; j++)
//rkey[j] = (*key_arr)[ri][j];
//rkey[keylen-1]='\0';
//find the key with the given tuple
dataTuple *dt = ltable->findTuple(xid, rkey, keylen);
assert(dt!=0);
//if(dt!=0)
{
found_tuples++;
assert(dt->rawkeylen() == (*key_arr)[ri].length()+1);
assert(dt->datalen() == (*data_arr)[ri].length()+1);
dataTuple::freetuple(dt);
}
dt = 0;
free(rkey);
}
printf("found %d\n", found_tuples);
key_arr->clear();
data_arr->clear();
delete key_arr;
delete data_arr;
mscheduler.shutdown();
printf("merge threads finished.\n");
gettimeofday(&stop_tv,0);
printf("run time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
Tcommit(xid);
delete ltable;
bLSM::deinit_stasis();
}
void * mergeScheduler::diskMergeThread()
{
int xid;
assert(ltable_->get_tree_c2());
int merge_count =0;
mergeStats * stats = ltable_->merge_mgr->get_merge_stats(2);
while(true)
{
// 2: wait for input
rwlc_writelock(ltable_->header_mut);
ltable_->merge_mgr->new_merge(2);
int done = 0;
// get a new input for merge
while(!ltable_->get_tree_c1_mergeable())
{
pthread_cond_signal(<able_->c1_needed);
if(!ltable_->is_still_running()){
done = 1;
break;
}
DEBUG("dmt:\twaiting for block ready cond\n");
rwlc_cond_wait(<able_->c1_ready, ltable_->header_mut);
DEBUG("dmt:\tblock ready\n");
}
if(done==1)
{
rwlc_unlock(ltable_->header_mut);
break;
}
stats->starting_merge();
// 3: begin
xid = Tbegin();
// 4: do the merge.
//create the iterators
diskTreeComponent::iterator *itrA = ltable_->get_tree_c2()->open_iterator();
diskTreeComponent::iterator *itrB = ltable_->get_tree_c1_mergeable()->open_iterator(ltable_->merge_mgr, 0.05, <able_->c1_flushing);
//create a new tree
diskTreeComponent * c2_prime = new diskTreeComponent(xid, ltable_->internal_region_size, ltable_->datapage_region_size, ltable_->datapage_size, stats, (uint64_t)(ltable_->max_c0_size * *ltable_->R() + stats->base_size)/ 1000);
// diskTreeComponent * c2_prime = new diskTreeComponent(xid, ltable_->internal_region_size, ltable_->datapage_region_size, ltable_->datapage_size, stats);
rwlc_unlock(ltable_->header_mut);
//do the merge
DEBUG("dmt:\tMerging:\n");
merge_iterators<typeof(*itrA),typeof(*itrB)>(xid, c2_prime, itrA, itrB, ltable_, c2_prime, stats, true);
delete itrA;
delete itrB;
//5: force write the new region to disk
c2_prime->force(xid);
// (skip 6, 7, 8, 8.5, 9))
rwlc_writelock(ltable_->header_mut);
//12
ltable_->get_tree_c2()->dealloc(xid);
delete ltable_->get_tree_c2();
//11.5
ltable_->get_tree_c1_mergeable()->dealloc(xid);
//11
delete ltable_->get_tree_c1_mergeable();
ltable_->set_tree_c1_mergeable(0);
//writes complete
//now atomically replace the old c2 with new c2
//pthread_mutex_lock(a->block_ready_mut);
merge_count++;
//update the current optimal R value
*(ltable_->R()) = std::max(MIN_R, sqrt( ((double)stats->output_size()) / ((double)ltable_->mean_c0_run_length) ) );
DEBUG("\nR = %f\n", *(ltable_->R()));
DEBUG("dmt:\tmerge_count %lld\t#written bytes: %lld\n optimal r %.2f", stats.stats_merge_count, stats.output_size(), *(a->r_i));
// 10: C2 is never too big
ltable_->set_tree_c2(c2_prime);
stats->handed_off_tree();
DEBUG("dmt:\tUpdated C2's position on disk to %lld\n",(long long)-1);
// 13
ltable_->update_persistent_header(xid);
Tcommit(xid);
rwlc_unlock(ltable_->header_mut);
// stats->pretty_print(stdout);
ltable_->merge_mgr->finished_merge(2);
}
return 0;
}
int main (int argc, char * argv[]) {
double MB = 1024 * 1024;
uint64_t mb = 20000; // size of run, in megabytes.
enum run_type mode = ALL;
const uint64_t num_pages = mb * (MB / PAGE_SIZE);
stasis_buffer_manager_size = (512 * MB) / PAGE_SIZE;
// stasis_buffer_manager_hint_writes_are_sequential = 1;
// stasis_dirty_page_table_flush_quantum = (8 * MB) / PAGE_SIZE; // XXX if set to high-> segfault
// stasis_dirty_page_count_hard_limit = (16 * MB) / PAGE_SIZE;
// stasis_dirty_page_count_soft_limit = (10 * MB) / PAGE_SIZE;
// stasis_dirty_page_low_water_mark = (8 * MB) / PAGE_SIZE;
// Hard disk preferred.
/* stasis_dirty_page_table_flush_quantum = (4 * MB) / PAGE_SIZE; // XXX if set to high-> segfault
stasis_dirty_page_count_hard_limit = (12 * MB) / PAGE_SIZE;
stasis_dirty_page_count_soft_limit = (8 * MB) / PAGE_SIZE;
stasis_dirty_page_low_water_mark = (4 * MB) / PAGE_SIZE;*/
// SSD preferred.
stasis_dirty_page_table_flush_quantum = (4 * MB) / PAGE_SIZE; // XXX if set to high-> segfault
stasis_dirty_page_count_hard_limit = (40 * MB) / PAGE_SIZE;
stasis_dirty_page_count_soft_limit = (32 * MB) / PAGE_SIZE;
stasis_dirty_page_low_water_mark = (16 * MB) / PAGE_SIZE;
stasis_dirty_page_table_flush_quantum = (4 * MB) / PAGE_SIZE; // XXX if set to high-> segfault
stasis_dirty_page_count_hard_limit = (48 * MB) / PAGE_SIZE;
stasis_dirty_page_count_soft_limit = (40 * MB) / PAGE_SIZE;
stasis_dirty_page_low_water_mark = (32 * MB) / PAGE_SIZE;
printf("stasis_buffer_manager_size=%lld\n", (long long)stasis_buffer_manager_size * PAGE_SIZE);
printf("Hard limit=%lld\n", (long long)((stasis_dirty_page_count_hard_limit*PAGE_SIZE)/MB));
printf("Hard limit is %f pct.\n", 100.0 * ((double)stasis_dirty_page_count_hard_limit)/((double)stasis_buffer_manager_size));
bLSM::init_stasis();
regionAllocator * readableAlloc = NULL;
if(!mode) {
int xid = Tbegin();
regionAllocator * alloc = new regionAllocator(xid, num_pages);
printf("Starting first write of %lld mb\n", (long long)mb);
struct timeval start, start_sync, stop; double elapsed;
gettimeofday(&start, 0);
pageid_t extent = alloc->alloc_extent(xid, num_pages);
for(uint64_t i = 0; i < num_pages; i++) {
Page * p = loadUninitializedPage(xid, i+extent);
stasis_dirty_page_table_set_dirty((stasis_dirty_page_table_t*)stasis_runtime_dirty_page_table(), p);
releasePage(p);
}
gettimeofday(&start_sync,0);
alloc->force_regions(xid);
readableAlloc = alloc;
Tcommit(xid);
// alloc = new RegionAllocator(xid, num_pages);
gettimeofday(&stop, 0);
elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)mb)/elapsed);
printf("Sync took %f seconds.\n", stasis_timeval_to_double(stasis_subtract_timeval(stop, start_sync)));
}
if(!mode) {
int xid = Tbegin();
regionAllocator * alloc = new regionAllocator(xid, num_pages);
printf("Starting write with parallel read of %lld mb\n", (long long)mb);
struct timeval start, start_sync, stop; double elapsed;
gettimeofday(&start, 0);
pageid_t region_length;
pageid_t region_count;
pageid_t * old_extents = readableAlloc->list_regions(xid, ®ion_length, ®ion_count);
pageid_t extent = alloc->alloc_extent(xid, num_pages);
assert(region_count == 1);
for(uint64_t i = 0; i < num_pages/2; i++) {
Page * p = loadUninitializedPage(xid, i+extent);
stasis_dirty_page_table_set_dirty((stasis_dirty_page_table_t*)stasis_runtime_dirty_page_table(), p);
releasePage(p);
p = loadPage(xid, i+old_extents[0]);
releasePage(p);
}
gettimeofday(&start_sync,0);
alloc->force_regions(xid);
delete alloc;
Tcommit(xid);
// alloc = new RegionAllocator(xid, num_pages);
gettimeofday(&stop, 0);
elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)mb)/elapsed);
printf("Sync took %f seconds.\n", stasis_timeval_to_double(stasis_subtract_timeval(stop, start_sync)));
}
if(!mode) {
int xid = Tbegin();
struct timeval start, start_sync, stop; double elapsed;
printf("Starting write of giant datapage\n");
gettimeofday(&start, 0);
regionAllocator * alloc = new regionAllocator(xid, num_pages);
dataPage * dp = new DataPage(xid, num_pages-1, alloc);
byte * key = (byte*)calloc(100, 1);
byte * val = (byte*)calloc(900, 1);
dataTuple * tup = dataTuple::create(key, 100, val, 900);
free(key);
free(val);
while(1) {
if(!dp->append(tup)) {
break;
}
}
gettimeofday(&start_sync,0);
alloc->force_regions(xid);
gettimeofday(&stop, 0);
Tcommit(xid);
elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)mb)/elapsed);
printf("Sync took %f seconds.\n", stasis_timeval_to_double(stasis_subtract_timeval(stop, start_sync)));
}
if(!mode) {
int xid = Tbegin();
struct timeval start, start_sync, stop; double elapsed;
printf("Starting write of many small datapages\n");
gettimeofday(&start, 0);
regionAllocator * alloc = new regionAllocator(xid, num_pages);
byte * key = (byte*)calloc(100, 1);
byte * val = (byte*)calloc(900, 1);
dataTuple * tup = dataTuple::create(key, 100, val, 900);
free(key);
free(val);
dataPage * dp = 0;
uint64_t this_count = 0;
uint64_t count = 0;
uint64_t dp_count = 0;
while((count * 1000) < (mb * 1024*1024)) {
if((!dp) || !dp->append(tup)) {
dp = new DataPage(xid, 2, alloc);
dp_count++;
}
count++;
this_count++;
// if(((this_count * 1000) > (1024 * 1024 * 16))) {
// alloc->force_regions(xid);
// this_count = 0;
// gettimeofday(&stop, 0);
// elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
// printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)(count*1000))/(1024*1024*elapsed));
// }
}
gettimeofday(&start_sync,0);
alloc->force_regions(xid);
gettimeofday(&stop, 0);
Tcommit(xid);
elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)(count*1000))/(elapsed*1024*1024));
printf("Sync took %f seconds.\n", stasis_timeval_to_double(stasis_subtract_timeval(stop, start_sync)));
}
if(!mode) {
int xid = Tbegin();
struct timeval start, start_sync, stop; double elapsed;
printf("Starting two parallel writes of many small datapages\n");
gettimeofday(&start, 0);
regionAllocator * alloc = new regionAllocator(xid, num_pages/2);
regionAllocator * alloc2 = new regionAllocator(xid, num_pages/2);
byte * key = (byte*)calloc(100, 1);
byte * val = (byte*)calloc(900, 1);
dataTuple * tup = dataTuple::create(key, 100, val, 900);
free(key);
free(val);
dataPage * dp = 0;
dataPage * dp2 = 0;
uint64_t this_count = 0;
uint64_t count = 0;
uint64_t dp_count = 0;
while((count * 1000) < (mb * 1024*1024)) {
if((!dp) || !dp->append(tup)) {
dp = new DataPage(xid, 2, alloc);
dp_count++;
}
if((!dp2) || !dp2->append(tup)) {
dp2 = new DataPage(xid, 2, alloc2);
//dp_count++;
}
count += 2;
this_count++;
// if(((this_count * 1000) > (1024 * 1024 * 16))) {
// alloc->force_regions(xid);
// this_count = 0;
// gettimeofday(&stop, 0);
// elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
// printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)(count*1000))/(1024*1024*elapsed));
// }
}
gettimeofday(&start_sync,0);
alloc->force_regions(xid);
alloc2->force_regions(xid);
gettimeofday(&stop, 0);
Tcommit(xid);
elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)(count*1000))/(elapsed*1024*1024));
printf("Sync took %f seconds.\n", stasis_timeval_to_double(stasis_subtract_timeval(stop, start_sync)));
}
regionAllocator * read_alloc = NULL;
regionAllocator * read_alloc2 = NULL;
regionAllocator * read_alloc3 = NULL;
regionAllocator * read_alloc4 = NULL;
if(!mode) {
int xid = Tbegin();
struct timeval start, start_sync, stop; double elapsed;
printf("Starting four parallel writes of many small datapages\n");
gettimeofday(&start, 0);
regionAllocator * alloc = new regionAllocator(xid, num_pages/4);
regionAllocator * alloc2 = new regionAllocator(xid, num_pages/4);
regionAllocator * alloc3 = new regionAllocator(xid, num_pages/4);
regionAllocator * alloc4 = new regionAllocator(xid, num_pages/4);
byte * key = (byte*)calloc(100, 1);
byte * val = (byte*)calloc(900, 1);
dataTuple * tup = dataTuple::create(key, 100, val, 900);
free(key);
free(val);
dataPage * dp = 0;
dataPage * dp2 = 0;
dataPage * dp3 = 0;
dataPage * dp4 = 0;
uint64_t this_count = 0;
uint64_t count = 0;
uint64_t dp_count = 0;
while((count * 1000) < (mb * 1024*1024)) {
if((!dp) || !dp->append(tup)) {
dp = new DataPage(xid, 2, alloc);
dp_count++;
}
if((!dp2) || !dp2->append(tup)) {
dp2 = new DataPage(xid, 2, alloc2);
//dp_count++;
}
if((!dp3) || !dp3->append(tup)) {
dp3 = new DataPage(xid, 2, alloc3);
//dp_count++;
}
if((!dp4) || !dp4->append(tup)) {
dp4 = new DataPage(xid, 2, alloc4);
//dp_count++;
}
count += 4;
this_count++;
// if(((this_count * 1000) > (1024 * 1024 * 16))) {
// alloc->force_regions(xid);
// this_count = 0;
// gettimeofday(&stop, 0);
// elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
// printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)(count*1000))/(1024*1024*elapsed));
// }
}
gettimeofday(&start_sync,0);
alloc->force_regions(xid);
alloc2->force_regions(xid);
alloc3->force_regions(xid);
alloc4->force_regions(xid);
gettimeofday(&stop, 0);
Tcommit(xid);
elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)(count*1000))/(elapsed*1024*1024));
printf("Sync took %f seconds.\n", stasis_timeval_to_double(stasis_subtract_timeval(stop, start_sync)));
read_alloc = alloc;
read_alloc2 = alloc2;
read_alloc3 = alloc3;
read_alloc4 = alloc4;
}
if(!mode) {
int xid = Tbegin();
struct timeval start, start_sync, stop; double elapsed;
printf("Starting four parallel writes of many small datapages\n");
gettimeofday(&start, 0);
regionAllocator * alloc = new regionAllocator(xid, num_pages/4);
regionAllocator * alloc2 = new regionAllocator(xid, num_pages/4);
regionAllocator * alloc3 = new regionAllocator(xid, num_pages/4);
regionAllocator * alloc4 = new regionAllocator(xid, num_pages/4);
byte * key = (byte*)calloc(100, 1);
byte * val = (byte*)calloc(900, 1);
dataTuple * tup = dataTuple::create(key, 100, val, 900);
free(key);
free(val);
dataPage * dp = 0;
dataPage * dp2 = 0;
dataPage * dp3 = 0;
dataPage * dp4 = 0;
uint64_t this_count = 0;
uint64_t count = 0;
uint64_t dp_count = 0;
pageid_t n1, n2, n3, n4;
pageid_t l1, l2, l3, l4;
pageid_t * regions1, * regions2, * regions3, * regions4;
regions1 = read_alloc->list_regions(xid, &l1, &n1);
regions2 = read_alloc2->list_regions(xid, &l2, &n2);
regions3 = read_alloc3->list_regions(xid, &l3, &n3);
regions4 = read_alloc4->list_regions(xid, &l4, &n4);
pageid_t i1 = regions1[0];
pageid_t i2 = regions2[0];
pageid_t i3 = regions3[0];
pageid_t i4 = regions4[0];
dataPage * rdp = new DataPage(xid, 0, i1);
dataPage * rdp2 = new DataPage(xid, 0, i2);
dataPage * rdp3 = new DataPage(xid, 0, i3);
dataPage * rdp4 = new DataPage(xid, 0, i4);
dataPage::iterator it1 = rdp->begin();
dataPage::iterator it2 = rdp2->begin();
dataPage::iterator it3 = rdp3->begin();
dataPage::iterator it4 = rdp4->begin();
while((count * 1000) < (mb * 1024*1024)) {
if((!dp) || !dp->append(tup)) {
dp = new DataPage(xid, 2, alloc);
dp_count++;
}
if((!dp2) || !dp2->append(tup)) {
dp2 = new DataPage(xid, 2, alloc2);
//dp_count++;
}
if((!dp3) || !dp3->append(tup)) {
dp3 = new DataPage(xid, 2, alloc3);
//dp_count++;
}
if((!dp4) || !dp4->append(tup)) {
dp4 = new DataPage(xid, 2, alloc4);
//dp_count++;
}
dataTuple * t;
if((!rdp) || !(t = it1.getnext())) {
i1+= rdp->get_page_count();
if(rdp) delete rdp;
rdp = new DataPage(xid, 0, i1);
// i1++;
it1 = rdp->begin();
t = it1.getnext();
}
if(t) dataTuple::freetuple(t);
if((!rdp2) || !(t = it2.getnext())) {
i2+= rdp2->get_page_count();
if(rdp2) delete rdp2;
rdp2 = new DataPage(xid, 0, i2);
// i2++;
it2 = rdp2->begin();
t = it2.getnext();
}
if(t) dataTuple::freetuple(t);
if((!rdp3) || !(t = it3.getnext())) {
i3+= rdp3->get_page_count();
if(rdp3) delete rdp3;
rdp3 = new DataPage(xid, 0, i3);
// i3++;
it3 = rdp3->begin();
t = it3.getnext();
}
if(t) dataTuple::freetuple(t);
if((!rdp4) || !(t = it4.getnext())) {
i4+= rdp4->get_page_count();
if(rdp4) delete rdp4;
rdp4 = new DataPage(xid, 0, i4);
// i4++;
it4 = rdp4->begin();
t = it4.getnext();
}
if(t) dataTuple::freetuple(t);
count += 8;
this_count++;
// if(((this_count * 1000) > (1024 * 1024 * 16))) {
// alloc->force_regions(xid);
// this_count = 0;
// gettimeofday(&stop, 0);
// elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
// printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)(count*1000))/(1024*1024*elapsed));
// }
}
gettimeofday(&start_sync,0);
alloc->force_regions(xid);
alloc2->force_regions(xid);
alloc3->force_regions(xid);
alloc4->force_regions(xid);
gettimeofday(&stop, 0);
Tcommit(xid);
elapsed = stasis_timeval_to_double(stasis_subtract_timeval(stop, start));
printf("Write took %f seconds (%f mb/sec)\n", elapsed, ((double)(count*1000))/(elapsed*1024*1024));
printf("Sync took %f seconds.\n", stasis_timeval_to_double(stasis_subtract_timeval(stop, start_sync)));
read_alloc = alloc;
read_alloc2 = alloc2;
read_alloc3 = alloc3;
read_alloc4 = alloc4;
}
bLSM::deinit_stasis();
}
JNIEXPORT void JNICALL Java_stasis_jni_Stasis_commit
(JNIEnv *e, jclass c, jlong xid) {
DEBUG("commiting %lld\n", (long long)xid);
Tcommit((int)xid);
}
} END_TEST
START_TEST(pagedListCheck) {
Tinit();
int xid = Tbegin();
recordid list = TpagedListAlloc(xid);
int a;
recordid b;
int i;
printf("\n");
for(i = 0; i < NUM_ENTRIES; i++) {
if(!(i % (NUM_ENTRIES/10))) {
printf("."); fflush(stdout);
}
a = i;
b.page = i+1;
b.slot = i+2;
b.size = i+3;
int ret;
{
byte * t;
ret = TpagedListFind(xid, list, (byte*)&a, sizeof(int), &t);
assert(-1 == ret);
}
ret = TpagedListInsert(xid, list, (byte*)&a, sizeof(int), (byte*)&b, sizeof(recordid));
assert(!ret);
recordid * bb;
recordid ** bbb = &bb;
ret = TpagedListFind(xid, list, (byte*)&a, sizeof(int), (byte**)bbb);
assert(ret == sizeof(recordid));
assert(!memcmp(bb, &b, sizeof(recordid)));
}
Tcommit(xid);
printf("\n");
xid = Tbegin();
for(i = 0; i < NUM_ENTRIES; i++ ) {
if(!(i % (NUM_ENTRIES/10))) {
printf("."); fflush(stdout);
}
a = i;
b.page = i+1;
b.slot = i+2;
b.size = i+3;
recordid * bb;
recordid ** bbb = &bb;
int ret = TpagedListFind(xid, list, (byte*)&a, sizeof(int), (byte**)bbb);
assert(ret == sizeof(recordid));
assert(!memcmp(bb, &b, sizeof(recordid)));
if(!(i % 10)) {
ret = TpagedListRemove(xid, list, (byte*)&a, sizeof(int));
assert(ret);
free(bb);
bb = 0;
ret = TpagedListFind(xid, list, (byte*)&a, sizeof(int), (byte**)bbb);
assert(-1==ret);
assert(!bb);
}
}
Tabort(xid);
xid = Tbegin();
printf("\n");
for(i = 0; i < NUM_ENTRIES; i++) {
if(!(i % (NUM_ENTRIES/10))) {
printf("."); fflush(stdout);
}
a = i;
b.page = i+1;
b.slot = i+2;
b.size = i+3;
recordid * bb;
recordid ** bbb = &bb;
int ret = TpagedListFind(xid, list, (byte*)&a, sizeof(int), (byte**)bbb);
assert(ret == sizeof(recordid));
assert(!memcmp(bb, &b, sizeof(recordid)));
}
byte * seen = stasis_calloc(NUM_ENTRIES, byte);
lladd_pagedList_iterator * it = TpagedListIterator(xid, list);
int keySize;
int valueSize;
int * key = 0;
int ** bkey = &key;
recordid * value = 0;
recordid ** bvalue = &value;
while(TpagedListNext(xid, it, (byte**)bkey, &keySize, (byte**)bvalue, &valueSize)) {
assert(!seen[*key]);
seen[*key] = 1;
assert(value->page == *key+1);
assert(value->slot == *key+2);
assert(value->size == *key+3);
free(key);
free(value);
key = 0;
value = 0;
}
for(i = 0; i < NUM_ENTRIES; i++) {
assert(seen[i] == 1);
}
Tcommit(xid);
Tdeinit();
} END_TEST
void insertProbeIter(size_t NUM_ENTRIES)
{
srand(1000);
unlink("storefile.txt");
unlink("logfile.txt");
system("rm -rf stasis_log/");
sync();
bLSM::init_stasis();
double delete_freq = .05;
double update_freq = .15;
//data generation
typedef std::vector<std::string> key_v_t;
const static size_t max_partition_size = 100000;
int KEY_LEN = 100;
std::vector<key_v_t*> *key_v_list = new std::vector<key_v_t*>;
int list_size = NUM_ENTRIES / max_partition_size + 1;
for(int i =0; i<list_size; i++)
{
key_v_t * key_arr = new key_v_t;
if(NUM_ENTRIES < max_partition_size*(i+1))
preprandstr(NUM_ENTRIES-max_partition_size*i, key_arr, KEY_LEN, true);
else
preprandstr(max_partition_size, key_arr, KEY_LEN, true);
std::sort(key_arr->begin(), key_arr->end(), &mycmp);
key_v_list->push_back(key_arr);
printf("size partition %llu is %llu\n", (unsigned long long)i+1, (unsigned long long)key_arr->size());
}
key_v_t * key_arr = new key_v_t;
std::vector<key_v_t::iterator*> iters;
for(int i=0; i<list_size; i++)
{
iters.push_back(new key_v_t::iterator((*key_v_list)[i]->begin()));
}
size_t lc = 0;
while(true)
{
int list_index = -1;
for(int i=0; i<list_size; i++)
{
if(*iters[i] == (*key_v_list)[i]->end())
continue;
if(list_index == -1 || mycmp(**iters[i], **iters[list_index]))
list_index = i;
}
if(list_index == -1)
break;
if(key_arr->size() == 0 || mycmp(key_arr->back(), **iters[list_index]))
key_arr->push_back(**iters[list_index]);
(*iters[list_index])++;
lc++;
if(lc % max_partition_size == 0)
printf("%llu/%llu completed.\n", (unsigned long long)lc, (unsigned long long)NUM_ENTRIES);
}
for(int i=0; i<list_size; i++)
{
(*key_v_list)[i]->clear();
delete (*key_v_list)[i];
delete iters[i];
}
key_v_list->clear();
delete key_v_list;
printf("key arr size: %llu\n", (unsigned long long)key_arr->size());
if(key_arr->size() > NUM_ENTRIES)
key_arr->erase(key_arr->begin()+NUM_ENTRIES, key_arr->end());
NUM_ENTRIES=key_arr->size();
int xid = Tbegin();
bLSM *ltable = new bLSM(10 * 1024 * 1024, 1000, 1000, 40);
mergeScheduler mscheduler(ltable);
recordid table_root = ltable->allocTable(xid);
Tcommit(xid);
mscheduler.start();
printf("Stage 1: Writing %llu keys\n", (unsigned long long)NUM_ENTRIES);
struct timeval start_tv, stop_tv, ti_st, ti_end;
double insert_time = 0;
int delcount = 0, upcount = 0;
int64_t datasize = 0;
std::vector<pageid_t> dsp;
std::vector<int> del_list;
gettimeofday(&start_tv,0);
for(size_t i = 0; i < NUM_ENTRIES; i++)
{
//prepare the data
std::string ditem;
getnextdata(ditem, 8192);
//prepare the key
dataTuple *newtuple = dataTuple::create((*key_arr)[i].c_str(), (*key_arr)[i].length()+1, ditem.c_str(), ditem.length()+1);
datasize += newtuple->byte_length();
gettimeofday(&ti_st,0);
ltable->insertTuple(newtuple);
gettimeofday(&ti_end,0);
insert_time += tv_to_double(ti_end) - tv_to_double(ti_st);
dataTuple::freetuple(newtuple);
double rval = ((rand() % 100)+.0)/100;
if( rval < delete_freq) //delete a key
{
int del_index = i - (rand()%50); //delete one of the last inserted 50 elements
if(del_index >= 0 && std::find(del_list.begin(), del_list.end(), del_index) == del_list.end())
{
delcount++;
dataTuple *deltuple = dataTuple::create((*key_arr)[del_index].c_str(), (*key_arr)[del_index].length()+1);
gettimeofday(&ti_st,0);
ltable->insertTuple(deltuple);
gettimeofday(&ti_end,0);
insert_time += tv_to_double(ti_end) - tv_to_double(ti_st);
dataTuple::freetuple(deltuple);
del_list.push_back(del_index);
}
}
else if(rval < delete_freq + update_freq) //update a record
{
int up_index = i - (rand()%50); //update one of the last inserted 50 elements
if(up_index >= 0 && std::find(del_list.begin(), del_list.end(), up_index) == del_list.end())
{//only update non-deleted elements
getnextdata(ditem, 512);
upcount++;
dataTuple *uptuple = dataTuple::create((*key_arr)[up_index].c_str(), (*key_arr)[up_index].length()+1,
ditem.c_str(), ditem.length()+1);
gettimeofday(&ti_st,0);
ltable->insertTuple(uptuple);
gettimeofday(&ti_end,0);
insert_time += tv_to_double(ti_end) - tv_to_double(ti_st);
dataTuple::freetuple(uptuple);
}
}
}
gettimeofday(&stop_tv,0);
printf("insert time: %6.1f\n", insert_time);
printf("insert time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
printf("#deletions: %d\n#updates: %d\n", delcount, upcount);
printf("\nTREE STRUCTURE\n");
printf("datasize: %llu\n", (unsigned long long)datasize);
xid = Tbegin();
printf("Stage 2: Looking up %llu keys:\n", (unsigned long long)NUM_ENTRIES);
int found_tuples=0;
for(int i=NUM_ENTRIES-1; i>=0; i--)
{
int ri = i;
//get the key
uint32_t keylen = (*key_arr)[ri].length()+1;
dataTuple::key_t rkey = (dataTuple::key_t) malloc(keylen);
memcpy((byte*)rkey, (*key_arr)[ri].c_str(), keylen);
//find the key with the given tuple
dataTuple *dt = ltable->findTuple(xid, rkey, keylen);
if(std::find(del_list.begin(), del_list.end(), i) == del_list.end())
{
assert(dt!=0);
assert(!dt->isDelete());
found_tuples++;
assert(dt->rawkeylen() == (*key_arr)[ri].length()+1);
dataTuple::freetuple(dt);
}
else
{
if(dt!=0)
{
assert(dt->rawkeylen() == (*key_arr)[ri].length()+1);
assert(dt->isDelete());
dataTuple::freetuple(dt);
}
}
dt = 0;
free(rkey);
}
printf("found %d\n", found_tuples);
key_arr->clear();
//data_arr->clear();
delete key_arr;
//delete data_arr;
mscheduler.shutdown();
printf("merge threads finished.\n");
gettimeofday(&stop_tv,0);
printf("run time: %6.1f\n", (tv_to_double(stop_tv) - tv_to_double(start_tv)));
Tcommit(xid);
delete ltable;
bLSM::deinit_stasis();
}
/**
* Merge algorithm: Outsider's view
*<pre>
1: while(1)
2: wait for c0_mergable
3: begin
4: merge c0_mergable and c1 into c1' # Blocks; tree must be consistent at this point
5: force c1' # Blocks
6: if c1' is too big # Blocks; tree must be consistent at this point.
7: c1_mergable = c1'
8: c1 = new_empty
8.5: delete old c1_mergeable # Happens in other thread (not here)
9: else
10: c1 = c1'
11: c0_mergeable = NULL
11.5: delete old c0_mergeable
12: delete old c1
13: commit
</pre>
Merge algorithm: actual order: 1 2 3 4 5 6 12 11.5 11 [7 8 (9) 10] 13
*/
void * mergeScheduler::memMergeThread() {
int xid;
assert(ltable_->get_tree_c1());
int merge_count =0;
mergeStats * stats = ltable_->merge_mgr->get_merge_stats(1);
while(true) // 1
{
rwlc_writelock(ltable_->header_mut);
ltable_->merge_mgr->new_merge(1);
int done = 0;
// 2: wait for c0_mergable
// the merge iterator will wait until c0 is big enough for us to proceed.
if(!ltable_->is_still_running()) {
done = 1;
}
if(done==1)
{
pthread_cond_signal(<able_->c1_ready); // no block is ready. this allows the other thread to wake up, and see that we're shutting down.
rwlc_unlock(ltable_->header_mut);
break;
}
stats->starting_merge();
lsn_t merge_start = ltable_->get_log_offset();
printf("\nstarting memory merge. log offset is %lld\n", merge_start);
// 3: Begin transaction
xid = Tbegin();
// 4: Merge
//create the iterators
diskTreeComponent::iterator *itrA = ltable_->get_tree_c1()->open_iterator();
const int64_t min_bloom_target = ltable_->max_c0_size;
//create a new tree
diskTreeComponent * c1_prime = new diskTreeComponent(xid, ltable_->internal_region_size, ltable_->datapage_region_size, ltable_->datapage_size, stats, (stats->target_size < min_bloom_target ? min_bloom_target : stats->target_size) / 100);
ltable_->set_tree_c1_prime(c1_prime);
rwlc_unlock(ltable_->header_mut);
// needs to be past the rwlc_unlock...
memTreeComponent::batchedRevalidatingIterator *itrB =
new memTreeComponent::batchedRevalidatingIterator(ltable_->get_tree_c0(), ltable_->merge_mgr, ltable_->max_c0_size, <able_->c0_flushing, 100, <able_->rb_mut);
//: do the merge
DEBUG("mmt:\tMerging:\n");
merge_iterators<typeof(*itrA),typeof(*itrB)>(xid, c1_prime, itrA, itrB, ltable_, c1_prime, stats, false);
delete itrA;
delete itrB;
// 5: force c1'
//force write the new tree to disk
c1_prime->force(xid);
rwlc_writelock(ltable_->header_mut);
merge_count++;
DEBUG("mmt:\tmerge_count %lld #bytes written %lld\n", stats.stats_merge_count, stats.output_size());
// Immediately clean out c0 mergeable so that writers may continue.
// first, we need to move the c1' into c1.
// 12: delete old c1
ltable_->get_tree_c1()->dealloc(xid);
delete ltable_->get_tree_c1();
// 10: c1 = c1'
ltable_->set_tree_c1(c1_prime);
ltable_->set_tree_c1_prime(0);
ltable_->set_c0_is_merging(false);
double new_c1_size = stats->output_size();
pthread_cond_signal(<able_->c0_needed);
ltable_->update_persistent_header(xid, merge_start);
Tcommit(xid);
ltable_->truncate_log();
//TODO: this is simplistic for now
//6: if c1' is too big, signal the other merger
// XXX move this to mergeManager, and make bytes_in_small be protected.
if(stats->bytes_in_small) {
// update c0 effective size.
double frac = 1.0/(double)merge_count;
ltable_->num_c0_mergers = merge_count;
ltable_->mean_c0_run_length=
(int64_t) (
((double)ltable_->mean_c0_run_length)*(1-frac) +
((double)stats->bytes_in_small*frac));
//ltable_->merge_mgr->get_merge_stats(0)->target_size = ltable_->mean_c0_run_length;
}
printf("\nMerge done. R = %f MemSize = %lld Mean = %lld, This = %lld, Count = %d factor %3.3fcur%3.3favg\n", *ltable_->R(), (long long)ltable_->max_c0_size, (long long int)ltable_->mean_c0_run_length, stats->bytes_in_small, merge_count, ((double)stats->bytes_in_small) / (double)ltable_->max_c0_size, ((double)ltable_->mean_c0_run_length) / (double)ltable_->max_c0_size);
assert(*ltable_->R() >= MIN_R);
// XXX don't hardcode 1.05, which will break for R > ~20.
bool signal_c2 = (1.05 * new_c1_size / ltable_->mean_c0_run_length > *ltable_->R());
DEBUG("\nc1 size %f R %f\n", new_c1_size, *ltable_->R());
if( signal_c2 )
{
DEBUG("mmt:\tsignaling C2 for merge\n");
DEBUG("mmt:\tnew_c1_size %.2f\tMAX_C0_SIZE %lld\ta->max_size %lld\t targetr %.2f \n", new_c1_size,
ltable_->max_c0_size, a->max_size, target_R);
// XXX need to report backpressure here!
while(ltable_->get_tree_c1_mergeable()) {
ltable_->c1_flushing = true;
rwlc_cond_wait(<able_->c1_needed, ltable_->header_mut);
ltable_->c1_flushing = false;
}
xid = Tbegin();
// we just set c1 = c1'. Want to move c1 -> c1 mergeable, clean out c1.
// 7: and perhaps c1_mergeable
ltable_->set_tree_c1_mergeable(ltable_->get_tree_c1()); // c1_prime == c1.
stats->handed_off_tree();
// 8: c1 = new empty.
ltable_->set_tree_c1(new diskTreeComponent(xid, ltable_->internal_region_size, ltable_->datapage_region_size, ltable_->datapage_size, stats, 10));
pthread_cond_signal(<able_->c1_ready);
ltable_->update_persistent_header(xid);
Tcommit(xid);
}
// DEBUG("mmt:\tUpdated C1's position on disk to %lld\n",ltable_->get_tree_c1()->get_root_rec().page);
// 13
rwlc_unlock(ltable_->header_mut);
ltable_->merge_mgr->finished_merge(1);
// stats->pretty_print(stdout);
//TODO: get the freeing outside of the lock
}
return 0;
}
