skiplist* skiplist_clone(skiplist* list, dict_key_datum_clone_func clone_func) { ASSERT(list != NULL); skiplist* clone = skiplist_new(list->cmp_func, list->del_func, list->max_link); if (clone) { skip_node* node = list->head->link[0]; while (node) { bool inserted = false; void** datum = skiplist_insert(clone, node->key, &inserted); if (!datum || !inserted) { skiplist_free(clone); return NULL; } *datum = node->datum; node = node->link[0]; } if (clone_func) { node = clone->head->link[0]; while (node) { clone_func(&node->key, &node->datum); node = node->link[0]; } } } return clone; }
int main(void) { int i; int *key = malloc(N * sizeof(int)); if (key == NULL) { exit(-1); } struct skiplist *list = skiplist_new(); if (list == NULL) { exit(-1); } printf("Test start!\n"); printf("Add %d nodes...\n", N); /* Insert test */ srandom(time(NULL)); for (i = 0; i < N; i++) { int value = key[i] = (int)random(); skiplist_add(list, key[i], value); } #ifdef SKIPLIST_DEBUG skiplist_dump(list); #endif /* Search test */ printf("Now search each node...\n"); for (i = 0; i < N; i++) { int value = skiplist_search(list, key[i]); if (value != -1) { #ifdef SKIPLIST_DEBUG printf("key:0x%08x value:0x%08x\n", key[i], value); #endif } else { printf("Not found:0x%08x\n", key[i]); } } /* Delete test */ printf("Now remove all nodes...\n"); for (i = 0; i < N; i++) { skiplist_remove(list, key[i]); } #ifdef SKIPLIST_DEBUG skiplist_dump(list); #endif printf("End of Test.\n"); skiplist_delete(list); return 0; }
/* Measure insertions. */ static void ins(void) { skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL); TIME(pre); for (intptr_t i=0; i < lim; i++) { skiplist_add(sl, (void *) i, (void *) i); } TIME(post); TDIFF(); skiplist_free(sl, NULL, NULL); }
int main(int argc, char **argv) { skiplist *list = skiplist_new(compare_word); assert(list != NULL); FILE *f = fopen("../_test/wordlist.txt", "r"); assert(f != NULL); char *key; int i = 0; long long start, stop; start = get_ustime_sec(); while (!feof(f)) { i++; key = (char *)malloc(sizeof(char) * 30); assert(key != NULL); fscanf(f, "%s\n", key); skiplist_insert(list, key, &value); } stop = get_ustime_sec(); fclose(f); printf("put %d word in skiplist cost time %lldμs\n", i, stop-start); /* 查询 */ char test[30] = "comically"; int *t = skiplist_search(list, test); if (t) { printf("%s in wordlist and value = %d\n", test, *t); } else { printf("%s not in wordlist\n", test); } skiplist_delete(list, test); t = skiplist_search(list, test); if (t) { printf("%s in wordlist\n", test); } else { printf("%s not in wordlist\n", test); } /** * 在删除 Hashmap 前,需要自己管理 Key, Value 指向的内存 * 这里并没有释放 Key 内存 */ skiplist_free(list); return 0; }
dict* skiplist_dict_new(dict_compare_func cmp_func, dict_delete_func del_func, unsigned max_link) { dict* dct = MALLOC(sizeof(*dct)); if (dct) { if (!(dct->_object = skiplist_new(cmp_func, del_func, max_link))) { FREE(dct); return NULL; } dct->_vtable = &skiplist_vtable; } return dct; }
static void set(void) { skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL); TIME(pre); for (intptr_t i=0; i < lim; i++) { intptr_t k = i % (lim / 2); skiplist_set(sl, (void *) k, (void *) k, NULL); } TIME(post); TDIFF(); skiplist_free(sl, NULL, NULL); }
struct skiplist *_read_mmap(struct sst *sst, size_t count) { int i; int fd; int result; int fcount; int blk_sizes; char file[FILE_PATH_SIZE]; struct sst_block *blks; struct skiplist *merge = NULL; struct footer footer; int fsize = sizeof(struct footer); memset(file, 0, FILE_PATH_SIZE); snprintf(file, FILE_PATH_SIZE, "%s/%s", sst->basedir, sst->name); fd = open(file, O_RDWR, 0644); result = lseek(fd, -fsize, SEEK_END); if(result == -1) abort(); result = read(fd, &footer, fsize); if(result == -1) abort(); fcount = from_be32(footer.count); blk_sizes = fcount * sizeof(struct sst_block); blks = mmap(0, blk_sizes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if(blks == MAP_FAILED) { __DEBUG(LEVEL_ERROR, "%s", "Error: sst_bloom un-mmapping the file"); goto out; } merge = skiplist_new(fcount + count + 1); for(i = 0; i < fcount; i++) { skiplist_insert(merge, blks[i].key, from_be64(blks[i].offset), ADD); } if(munmap(blks, blk_sizes) == -1) { __DEBUG(LEVEL_ERROR, "%s", "read_map:un-mmapping the file"); } out: close(fd); return merge; }
int index_add(struct index *idx, struct slice *sk, struct slice *sv) { uint64_t value_offset; struct skiplist *list; struct skiplist *new_list; value_offset = log_append(idx->log, sk, sv); list = idx->list; if(!list) { __DEBUG(LEVEL_ERROR, "ERROR: List<%d> is NULL", idx->lsn); return 0; } if(!skiplist_notfull(list)) { idx->bg_merge_count++; /* If the detached-merge thread isnot finished, hold on it * Notice: it will block the current process, * but it happens only once in a thousand years on production environment. */ pthread_mutex_lock(&idx->merge_mutex); /* start to merge with detached thread */ pthread_t tid; idx->park->list = list; idx->park->lsn = idx->lsn; pthread_mutex_unlock(&idx->merge_mutex); pthread_create(&tid, &idx->attr, _merge_job, idx); idx->mtbl_rem_count = 0; new_list = skiplist_new(idx->max_mtbl_size); idx->list = new_list; idx->lsn++; log_next(idx->log, idx->lsn); } skiplist_insert(idx->list, sk->data, value_offset, sv == NULL ? DEL : ADD); idx->mtbl_rem_count++; return 1; }
/* Measure getting _nonexistent_ values (lookup failure). */ static void get_nonexistent(void) { skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL); for (intptr_t i=0; i < lim; i++) { skiplist_add(sl, (void *) i, (void *) i); } TIME(pre); for (intptr_t i=0; i < lim; i++) { intptr_t k = (i * largeish_prime) + lim; intptr_t v = 0; skiplist_get(sl, (void *) k, (void **)&v); assert(v == 0); if (0) { printf("%lu %lu\n", k, v); } } TIME(post); TDIFF(); skiplist_free(sl, NULL, NULL); }
static void set_and_get(void) { skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL); TIME(pre); for (intptr_t i=0; i < lim; i++) { intptr_t k = i % (lim / 2); skiplist_set(sl, (void *) k, (void *) k, NULL); } for (intptr_t i=0; i < lim; i++) { intptr_t k = (i * largeish_prime) % (lim / 2); intptr_t v = (intptr_t)0; skiplist_get(sl, (void *) k, (void **)&v); if (0) { printf("%lu %lu\n", k, v); } assert(v == k); } TIME(post); TDIFF(); skiplist_free(sl, NULL, NULL); }
struct index *index_new(const char *basedir, int max_mtbl_size, int tolog) { char dbfile[FILE_PATH_SIZE]; struct index *idx = malloc(sizeof(struct index)); struct idx_park *park = malloc(sizeof(struct idx_park)); ensure_dir_exists(basedir); idx->lsn = 0; idx->bloom_hits = 0; idx->bg_merge_count = 0; idx->max_mtbl = 1; idx->max_mtbl_size = max_mtbl_size; memset(idx->basedir, 0, FILE_PATH_SIZE); memcpy(idx->basedir, basedir, FILE_PATH_SIZE); /* sst */ idx->sst = sst_new(idx->basedir); idx->list = skiplist_new(max_mtbl_size); pthread_mutex_init(&idx->merge_mutex, NULL); /* container */ park->list = NULL; park->lsn = idx->lsn; idx->park = park; /* log */ idx->log = log_new(idx->basedir, idx->lsn, tolog); /* * Log Recovery Processes : * 1) read old log file and add entries to memtable * 2) read new log file abd add entries to memtable * 3) merge the current active log's memtable * 4) remove old log file, new log file * 5) create new memtable and log file */ if (log_recovery(idx->log, idx->list)) { __DEBUG(LEVEL_DEBUG, "prepare to merge logs, merge count #%d....", idx->list->count); sst_merge(idx->sst, idx->list, 1); remove(idx->log->log_new); remove(idx->log->log_old); idx->list = skiplist_new(idx->max_mtbl_size); } /* Create new log : 0.log */ log_next(idx->log, 0); memset(dbfile, 0, FILE_PATH_SIZE); snprintf(dbfile, FILE_PATH_SIZE, "%s/ness.db", idx->basedir); idx->db_rfd = open(dbfile, LSM_OPEN_FLAGS, 0644); /* Detached thread attr */ pthread_attr_init(&idx->attr); pthread_attr_setdetachstate(&idx->attr, PTHREAD_CREATE_DETACHED); return idx; }
int main(void) { int i; struct timeval start, end; int *key = (int *)malloc(N * sizeof(int)); if (key == NULL) { exit(-1); } struct skiplist *list = skiplist_new(); if (list == NULL) { exit(-1); } printf("Test start!\n"); printf("Add %d nodes...\n", N); /* Insert test */ srandom(time(NULL)); gettimeofday(&start, NULL); for (i = 0; i < N; i++) { int value = key[i] = (int)random(); skiplist_insert(list, key[i], value); } gettimeofday(&end, NULL); printf("time span: %ldms\n", (end.tv_sec - start.tv_sec)*1000 + (end.tv_usec - start.tv_usec)/1000); #ifdef SKIPLIST_DEBUG skiplist_dump(list); #endif /* Search test 1 */ printf("Now search each node by key...\n"); gettimeofday(&start, NULL); for (i = 0; i < N; i++) { struct skipnode *node = skiplist_search_by_key(list, key[i]); if (node != NULL) { #ifdef SKIPLIST_DEBUG printf("key:0x%08x value:0x%08x\n", node->key, node->value); #endif } else { printf("Not found:0x%08x\n", key[i]); } #ifdef SKIPLIST_DEBUG printf("key rank:%d\n", skiplist_key_rank(list, key[i])); #else //skiplist_key_rank(list, key[i]); #endif } gettimeofday(&end, NULL); printf("time span: %ldms\n", (end.tv_sec - start.tv_sec)*1000 + (end.tv_usec - start.tv_usec)/1000); /* Search test 2 */ printf("Now search each node by rank...\n"); gettimeofday(&start, NULL); for (i = 0; i < N; i++) { struct skipnode *node = skiplist_search_by_rank(list, i + 1); if (node != NULL) { #ifdef SKIPLIST_DEBUG printf("rank:%d value:0x%08x\n", i + 1, node->value); #endif } else { printf("Not found:%d\n", i + 1); } } gettimeofday(&end, NULL); printf("time span: %ldms\n", (end.tv_sec - start.tv_sec)*1000 + (end.tv_usec - start.tv_usec)/1000); /* Delete test */ printf("Now remove all nodes...\n"); gettimeofday(&start, NULL); for (i = 0; i < N; i++) { skiplist_remove(list, key[i]); } gettimeofday(&end, NULL); printf("time span: %ldms\n", (end.tv_sec - start.tv_sec)*1000 + (end.tv_usec - start.tv_usec)/1000); #ifdef SKIPLIST_DEBUG skiplist_dump(list); #endif printf("End of Test.\n"); skiplist_delete(list); return 0; }
for (intptr_t i=0; i < lim; i++) { intptr_t k = (i * largeish_prime) % (lim / 2); intptr_t v = (intptr_t)0; skiplist_get(sl, (void *) k, (void **)&v); if (0) { printf("%lu %lu\n", k, v); } assert(v == k); } TIME(post); TDIFF(); skiplist_free(sl, NULL, NULL); } static void delete(void) { skiplist *sl = skiplist_new(intptr_cmp, NULL, NULL); for (intptr_t i=0; i < lim; i++) { skiplist_add(sl, (void *) i, (void *) i); } TIME(pre); for (intptr_t i=0; i < lim; i++) { intptr_t k = (i * largeish_prime) % lim; intptr_t v = 0; skiplist_delete(sl, (void *) k, (void **)&v); if (0) { printf("%lu %lu\n", k, v); } assert(v == k); } TIME(post);