void bc_scan(Bitcask* bc)
{
const char* path = mgr_base(bc->mgr);
char dname[20], hname[20], datapath[255], hintpath[255];
int i=0;
struct stat st;
for (i=0; i<MAX_BUCKET_COUNT; i++) {
sprintf(dname, DATA_FILE, i);
sprintf(datapath, "%s/%s", path, dname);
if (stat(datapath, &st) != 0) {
break;
}
bc->bytes += st.st_size;
sprintf(hname, HINT_FILE, i);
sprintf(hintpath, "%s/%s", path, hname);
if (bc->before == 0){
if (0 == stat(hintpath, &st)){
scanHintFile(bc->tree, i, hintpath, NULL);
}else{
sprintf(hintpath, "%s/%s", mgr_alloc(bc->mgr, hname), hname);
scanDataFile(bc->tree, i, datapath, hintpath);
}
}else{
if (0 == stat(hintpath, &st) &&
(st.st_mtime < bc->before || 0 == stat(datapath, &st) && st.st_mtime < bc->before)){
scanHintFile(bc->tree, i, hintpath, NULL);
}else{
scanDataFileBefore(bc->tree, i, datapath, bc->before);
}
}
}
bc->curr = i;
}
/*
* bc_close() is not thread safe, should stop other threads before call it.
* */
void bc_close(Bitcask *bc)
{
int i=0;
pthread_mutex_lock(&bc->write_lock);
bc_flush(bc, 0, 0);
if (NULL != bc->curr_tree) {
uint32_t curr;
ht_get_hash(bc->curr_tree, "@", &curr);
if (curr > 0) {
char name[255], buf[255];
sprintf(name, HINT_FILE, bc->curr);
sprintf(buf, "%s/%s", mgr_alloc(bc->mgr, name), name);
build_hint(bc->curr_tree, buf);
}else{
ht_destroy(bc->curr_tree);
}
bc->curr_tree = NULL;
}
bc->curr = 0;
ht_destroy(bc->tree);
mgr_destroy(bc->mgr);
free(bc->write_buffer);
free(bc);
}
static inline char *new_path(char *dst, Mgr *mgr, const char *fmt, int i)
{
char *path = gen_path(dst, mgr_base(mgr), fmt, i);
if (!file_exists(dst))
{
char name[16];
sprintf(name, fmt, i);
sprintf(path, "%s/%s", mgr_alloc(mgr, name), name);
}
return path;
}
void bc_optimize(Bitcask *bc, int limit)
{
int i;
for (i=0; i < bc->curr; i++) {
char data[20], hint[20], datapath[255], hintpath[255];
sprintf(data, DATA_FILE, i);
sprintf(hint, HINT_FILE, i);
sprintf(datapath, "%s/%s", mgr_alloc(bc->mgr, data), data);
sprintf(hintpath, "%s/%s", mgr_alloc(bc->mgr, hint), hint);
uint32_t recoverd = 0;
HTree *cur_tree = optimizeDataFile(bc->tree, i, datapath, hintpath, limit, &recoverd);
if (NULL == cur_tree) continue;
pthread_mutex_lock(&bc->buffer_lock);
bc->bytes -= recoverd;
pthread_mutex_unlock(&bc->buffer_lock);
pthread_mutex_lock(&bc->write_lock);
ht_visit(cur_tree, update_items, bc->tree);
pthread_mutex_unlock(&bc->write_lock);
build_hint(cur_tree, hintpath);
}
}
void bc_flush(Bitcask *bc, int limit, int flush_period)
{
if (bc->curr >= MAX_BUCKET_COUNT) {
fprintf(stderr, "reach max bucket count\n");
exit(1);
}
pthread_mutex_lock(&bc->flush_lock);
time_t now = time(NULL);
if (bc->wbuf_curr_pos > limit * 1024 ||
now > bc->last_flush_time + flush_period && bc->wbuf_curr_pos > 0) {
char name[20], buf[255];
sprintf(name, DATA_FILE, bc->curr);
sprintf(buf, "%s/%s", mgr_alloc(bc->mgr, name), name);
FILE *f = fopen(buf, "ab");
if (f == NULL) {
fprintf(stderr, "open file %s for flushing failed.\n", buf);
exit(1);
}
// check file size
uint64_t last_pos = ftell(f);
if (last_pos > 0 && last_pos != bc->wbuf_start_pos) {
fprintf(stderr, "last pos not match: %llu != %d\n", last_pos, bc->wbuf_start_pos);
exit(1);
}
pthread_mutex_lock(&bc->buffer_lock);
int size = bc->wbuf_curr_pos;
char * tmp = (char*) malloc(size);
memcpy(tmp, bc->write_buffer, size);
pthread_mutex_unlock(&bc->buffer_lock);
int n = fwrite(tmp, 1, size, f);
if (n <= 0) {
fprintf(stderr, "write failed: return %d\n", n);
exit(1);
}
free(tmp);
fclose(f);
bc->last_flush_time = now;
pthread_mutex_lock(&bc->buffer_lock);
bc->bytes += n;
if (n < bc->wbuf_curr_pos) {
memmove(bc->write_buffer, bc->write_buffer + n, bc->wbuf_curr_pos - n);
}
bc->wbuf_start_pos += n;
bc->wbuf_curr_pos -= n;
if (bc->wbuf_curr_pos == 0) {
if (bc->wbuf_size < WRITE_BUFFER_SIZE) {
bc->wbuf_size *= 2;
free(bc->write_buffer);
bc->write_buffer = malloc(bc->wbuf_size);
} else if (bc->wbuf_size > WRITE_BUFFER_SIZE * 2) {
bc->wbuf_size = WRITE_BUFFER_SIZE;
free(bc->write_buffer);
bc->write_buffer = malloc(bc->wbuf_size);
}
}
if (bc->wbuf_start_pos + bc->wbuf_size > MAX_BUCKET_SIZE) {
// build in new thread
char hname[20], hintpath[255];
sprintf(hname, HINT_FILE, bc->curr);
sprintf(hintpath, "%s/%s", mgr_alloc(bc->mgr, hname), hname);
struct build_thread_args *args = (struct build_thread_args*)malloc(
sizeof(struct build_thread_args));
args->tree = bc->curr_tree;
args->path = strdup(hintpath);
pthread_t build_ptid;
pthread_create(&build_ptid, NULL, build_thread, args);
// next bucket
bc->curr ++;
bc->curr_tree = ht_new(bc->depth, bc->pos);
bc->wbuf_start_pos = 0;
}
pthread_mutex_unlock(&bc->buffer_lock);
}
pthread_mutex_unlock(&bc->flush_lock);
}