skiplist*
skiplist_clone(skiplist* list, dict_key_datum_clone_func clone_func)
{
ASSERT(list != NULL);
skiplist* clone = skiplist_new(list->cmp_func, list->del_func,
list->max_link);
if (clone) {
skip_node* node = list->head->link[0];
while (node) {
bool inserted = false;
void** datum = skiplist_insert(clone, node->key, &inserted);
if (!datum || !inserted) {
skiplist_free(clone);
return NULL;
}
*datum = node->datum;
node = node->link[0];
}
if (clone_func) {
node = clone->head->link[0];
while (node) {
clone_func(&node->key, &node->datum);
node = node->link[0];
}
}
}
return clone;
}
int
main(void)
{
int i;
int *key = malloc(N * sizeof(int));
if (key == NULL) {
exit(-1);
}
struct skiplist *list = skiplist_new();
if (list == NULL) {
exit(-1);
}
printf("Test start!\n");
printf("Add %d nodes...\n", N);
/* Insert test */
srandom(time(NULL));
for (i = 0; i < N; i++) {
int value = key[i] = (int)random();
skiplist_add(list, key[i], value);
}
#ifdef SKIPLIST_DEBUG
skiplist_dump(list);
#endif
/* Search test */
printf("Now search each node...\n");
for (i = 0; i < N; i++) {
int value = skiplist_search(list, key[i]);
if (value != -1) {
#ifdef SKIPLIST_DEBUG
printf("key:0x%08x value:0x%08x\n", key[i], value);
#endif
} else {
printf("Not found:0x%08x\n", key[i]);
}
}
/* Delete test */
printf("Now remove all nodes...\n");
for (i = 0; i < N; i++) {
skiplist_remove(list, key[i]);
}
#ifdef SKIPLIST_DEBUG
skiplist_dump(list);
#endif
printf("End of Test.\n");
skiplist_delete(list);
return 0;
}
/* Measure insertions. */
static void ins(void) {
skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL);
TIME(pre);
for (intptr_t i=0; i < lim; i++) {
skiplist_add(sl, (void *) i, (void *) i);
}
TIME(post);
TDIFF();
skiplist_free(sl, NULL, NULL);
}
int main(int argc, char **argv)
{
skiplist *list = skiplist_new(compare_word);
assert(list != NULL);
FILE *f = fopen("../_test/wordlist.txt", "r");
assert(f != NULL);
char *key;
int i = 0;
long long start, stop;
start = get_ustime_sec();
while (!feof(f)) {
i++;
key = (char *)malloc(sizeof(char) * 30);
assert(key != NULL);
fscanf(f, "%s\n", key);
skiplist_insert(list, key, &value);
}
stop = get_ustime_sec();
fclose(f);
printf("put %d word in skiplist cost time %lldμs\n", i, stop-start);
/* 查询 */
char test[30] = "comically";
int *t = skiplist_search(list, test);
if (t) {
printf("%s in wordlist and value = %d\n", test, *t);
} else {
printf("%s not in wordlist\n", test);
}
skiplist_delete(list, test);
t = skiplist_search(list, test);
if (t) {
printf("%s in wordlist\n", test);
} else {
printf("%s not in wordlist\n", test);
}
/**
* 在删除 Hashmap 前，需要自己管理 Key, Value 指向的内存
* 这里并没有释放 Key 内存
*/
skiplist_free(list);
return 0;
}
dict*
skiplist_dict_new(dict_compare_func cmp_func, dict_delete_func del_func,
unsigned max_link) {
dict* dct = MALLOC(sizeof(*dct));
if (dct) {
if (!(dct->_object = skiplist_new(cmp_func, del_func, max_link))) {
FREE(dct);
return NULL;
}
dct->_vtable = &skiplist_vtable;
}
return dct;
}
static void set(void) {
skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL);
TIME(pre);
for (intptr_t i=0; i < lim; i++) {
intptr_t k = i % (lim / 2);
skiplist_set(sl, (void *) k, (void *) k, NULL);
}
TIME(post);
TDIFF();
skiplist_free(sl, NULL, NULL);
}
struct skiplist *_read_mmap(struct sst *sst, size_t count)
{
int i;
int fd;
int result;
int fcount;
int blk_sizes;
char file[FILE_PATH_SIZE];
struct sst_block *blks;
struct skiplist *merge = NULL;
struct footer footer;
int fsize = sizeof(struct footer);
memset(file, 0, FILE_PATH_SIZE);
snprintf(file, FILE_PATH_SIZE, "%s/%s", sst->basedir, sst->name);
fd = open(file, O_RDWR, 0644);
result = lseek(fd, -fsize, SEEK_END);
if(result == -1)
abort();
result = read(fd, &footer, fsize);
if(result == -1)
abort();
fcount = from_be32(footer.count);
blk_sizes = fcount * sizeof(struct sst_block);
blks = mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if(blks == MAP_FAILED) {
__DEBUG(LEVEL_ERROR, "%s", "Error: sst_bloom un-mmapping the file");
goto out;
}
merge = skiplist_new(fcount + count + 1);
for(i = 0; i < fcount; i++) {
skiplist_insert(merge, blks[i].key, from_be64(blks[i].offset), ADD);
}
if(munmap(blks, blk_sizes) == -1) {
__DEBUG(LEVEL_ERROR, "%s", "read_map:un-mmapping the file");
}
out:
close(fd);
return merge;
}
int index_add(struct index *idx, struct slice *sk, struct slice *sv)
{
uint64_t value_offset;
struct skiplist *list;
struct skiplist *new_list;
value_offset = log_append(idx->log, sk, sv);
list = idx->list;
if(!list) {
__DEBUG(LEVEL_ERROR, "ERROR: List<%d> is NULL", idx->lsn);
return 0;
}
if(!skiplist_notfull(list)) {
idx->bg_merge_count++;
/* If the detached-merge thread isnot finished, hold on it
* Notice: it will block the current process,
* but it happens only once in a thousand years on production environment.
*/
pthread_mutex_lock(&idx->merge_mutex);
/* start to merge with detached thread */
pthread_t tid;
idx->park->list = list;
idx->park->lsn = idx->lsn;
pthread_mutex_unlock(&idx->merge_mutex);
pthread_create(&tid, &idx->attr, _merge_job, idx);
idx->mtbl_rem_count = 0;
new_list = skiplist_new(idx->max_mtbl_size);
idx->list = new_list;
idx->lsn++;
log_next(idx->log, idx->lsn);
}
skiplist_insert(idx->list, sk->data, value_offset, sv == NULL ? DEL : ADD);
idx->mtbl_rem_count++;
return 1;
}
/* Measure getting _nonexistent_ values (lookup failure). */
static void get_nonexistent(void) {
skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL);
for (intptr_t i=0; i < lim; i++) {
skiplist_add(sl, (void *) i, (void *) i);
}
TIME(pre);
for (intptr_t i=0; i < lim; i++) {
intptr_t k = (i * largeish_prime) + lim;
intptr_t v = 0;
skiplist_get(sl, (void *) k, (void **)&v);
assert(v == 0);
if (0) { printf("%lu %lu\n", k, v); }
}
TIME(post);
TDIFF();
skiplist_free(sl, NULL, NULL);
}
static void set_and_get(void) {
skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL);
TIME(pre);
for (intptr_t i=0; i < lim; i++) {
intptr_t k = i % (lim / 2);
skiplist_set(sl, (void *) k, (void *) k, NULL);
}
for (intptr_t i=0; i < lim; i++) {
intptr_t k = (i * largeish_prime) % (lim / 2);
intptr_t v = (intptr_t)0;
skiplist_get(sl, (void *) k, (void **)&v);
if (0) { printf("%lu %lu\n", k, v); }
assert(v == k);
}
TIME(post);
TDIFF();
skiplist_free(sl, NULL, NULL);
}
struct index *index_new(const char *basedir, int max_mtbl_size, int tolog)
{
char dbfile[FILE_PATH_SIZE];
struct index *idx = malloc(sizeof(struct index));
struct idx_park *park = malloc(sizeof(struct idx_park));
ensure_dir_exists(basedir);
idx->lsn = 0;
idx->bloom_hits = 0;
idx->bg_merge_count = 0;
idx->max_mtbl = 1;
idx->max_mtbl_size = max_mtbl_size;
memset(idx->basedir, 0, FILE_PATH_SIZE);
memcpy(idx->basedir, basedir, FILE_PATH_SIZE);
/* sst */
idx->sst = sst_new(idx->basedir);
idx->list = skiplist_new(max_mtbl_size);
pthread_mutex_init(&idx->merge_mutex, NULL);
/* container */
park->list = NULL;
park->lsn = idx->lsn;
idx->park = park;
/* log */
idx->log = log_new(idx->basedir, idx->lsn, tolog);
/*
* Log Recovery Processes :
* 1) read old log file and add entries to memtable
* 2) read new log file abd add entries to memtable
* 3) merge the current active log's memtable
* 4) remove old log file, new log file
* 5) create new memtable and log file
*/
if (log_recovery(idx->log, idx->list)) {
__DEBUG(LEVEL_DEBUG, "prepare to merge logs, merge count #%d....", idx->list->count);
sst_merge(idx->sst, idx->list, 1);
remove(idx->log->log_new);
remove(idx->log->log_old);
idx->list = skiplist_new(idx->max_mtbl_size);
}
/* Create new log : 0.log */
log_next(idx->log, 0);
memset(dbfile, 0, FILE_PATH_SIZE);
snprintf(dbfile, FILE_PATH_SIZE, "%s/ness.db", idx->basedir);
idx->db_rfd = open(dbfile, LSM_OPEN_FLAGS, 0644);
/* Detached thread attr */
pthread_attr_init(&idx->attr);
pthread_attr_setdetachstate(&idx->attr, PTHREAD_CREATE_DETACHED);
return idx;
}
int
main(void)
{
int i;
struct timeval start, end;
int *key = (int *)malloc(N * sizeof(int));
if (key == NULL) {
exit(-1);
}
struct skiplist *list = skiplist_new();
if (list == NULL) {
exit(-1);
}
printf("Test start!\n");
printf("Add %d nodes...\n", N);
/* Insert test */
srandom(time(NULL));
gettimeofday(&start, NULL);
for (i = 0; i < N; i++) {
int value = key[i] = (int)random();
skiplist_insert(list, key[i], value);
}
gettimeofday(&end, NULL);
printf("time span: %ldms\n", (end.tv_sec - start.tv_sec)*1000 + (end.tv_usec - start.tv_usec)/1000);
#ifdef SKIPLIST_DEBUG
skiplist_dump(list);
#endif
/* Search test 1 */
printf("Now search each node by key...\n");
gettimeofday(&start, NULL);
for (i = 0; i < N; i++) {
struct skipnode *node = skiplist_search_by_key(list, key[i]);
if (node != NULL) {
#ifdef SKIPLIST_DEBUG
printf("key:0x%08x value:0x%08x\n", node->key, node->value);
#endif
} else {
printf("Not found:0x%08x\n", key[i]);
}
#ifdef SKIPLIST_DEBUG
printf("key rank:%d\n", skiplist_key_rank(list, key[i]));
#else
//skiplist_key_rank(list, key[i]);
#endif
}
gettimeofday(&end, NULL);
printf("time span: %ldms\n", (end.tv_sec - start.tv_sec)*1000 + (end.tv_usec - start.tv_usec)/1000);
/* Search test 2 */
printf("Now search each node by rank...\n");
gettimeofday(&start, NULL);
for (i = 0; i < N; i++) {
struct skipnode *node = skiplist_search_by_rank(list, i + 1);
if (node != NULL) {
#ifdef SKIPLIST_DEBUG
printf("rank:%d value:0x%08x\n", i + 1, node->value);
#endif
} else {
printf("Not found:%d\n", i + 1);
}
}
gettimeofday(&end, NULL);
printf("time span: %ldms\n", (end.tv_sec - start.tv_sec)*1000 + (end.tv_usec - start.tv_usec)/1000);
/* Delete test */
printf("Now remove all nodes...\n");
gettimeofday(&start, NULL);
for (i = 0; i < N; i++) {
skiplist_remove(list, key[i]);
}
gettimeofday(&end, NULL);
printf("time span: %ldms\n", (end.tv_sec - start.tv_sec)*1000 + (end.tv_usec - start.tv_usec)/1000);
#ifdef SKIPLIST_DEBUG
skiplist_dump(list);
#endif
printf("End of Test.\n");
skiplist_delete(list);
return 0;
}
for (intptr_t i=0; i < lim; i++) {
intptr_t k = (i * largeish_prime) % (lim / 2);
intptr_t v = (intptr_t)0;
skiplist_get(sl, (void *) k, (void **)&v);
if (0) { printf("%lu %lu\n", k, v); }
assert(v == k);
}
TIME(post);
TDIFF();
skiplist_free(sl, NULL, NULL);
}
static void delete(void) {
skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL);
for (intptr_t i=0; i < lim; i++) {
skiplist_add(sl, (void *) i, (void *) i);
}
TIME(pre);
for (intptr_t i=0; i < lim; i++) {
intptr_t k = (i * largeish_prime) % lim;
intptr_t v = 0;
skiplist_delete(sl, (void *) k, (void **)&v);
if (0) { printf("%lu %lu\n", k, v); }
assert(v == k);
}
TIME(post);