static int load_latest_inode(struct hlfs_ctrl *ctrl) { //HLOG_DEBUG("enter func %s", __func__); if (NULL == ctrl) { HLOG_ERROR("Params Error"); return -1; } int ret = 0; const char segfile_name[SEGMENT_FILE_NAME_MAX]; build_segfile_name(ctrl->last_segno,segfile_name); bs_file_t file = ctrl->storage->bs_file_open \ (ctrl->storage,segfile_name, BS_READONLY); if (file == NULL) { HLOG_ERROR("can not open segment file %s", segfile_name); goto out; } uint64_t inode_pos = ctrl->last_offset - \ sizeof(struct inode_map_entry) -sizeof(struct inode); //HLOG_DEBUG("inode pos %llu",inode_pos); if(sizeof(struct inode) != ctrl->storage-> \ bs_file_pread(ctrl->storage, file, (char *)&ctrl->inode, \ sizeof(struct inode), inode_pos)){ HLOG_ERROR("can not read inode from %s", segfile_name); ret = -1; } ctrl->storage->bs_file_close(ctrl->storage, file); out: //HLOG_DEBUG("leave func %s", __func__); return ret; }
int prev_open_rsegfile(struct hlfs_ctrl *ctrl, uint32_t segno){ //HLOG_DEBUG("enter func %s", __func__); struct back_storage *storage = NULL; if (NULL == ctrl->last_rsegfile_handler) { char segfile_name[SEGMENT_FILE_NAME_MAX]; memset((void *)segfile_name, 0, SEGMENT_FILE_NAME_MAX); build_segfile_name(segno, segfile_name); bs_file_t file; if (segno >= ctrl->start_segno) { storage = ctrl->storage; }else{ HLOG_DEBUG("get parent storage for segno:%d",segno); if (NULL == (storage = get_parent_storage(ctrl->family, segno))){ g_assert(0); return -1; } } file = storage->bs_file_open(storage, segfile_name, BS_READONLY); if (file == NULL) { HLOG_ERROR("can not open segment file %s", segfile_name); g_assert(0); return -1; } ctrl->last_rsegfile_handler = file; ctrl->last_rsegfile_offset = ctrl->last_offset; ctrl->last_read_segno =segno; }else if (ctrl->last_read_segno != segno || (ctrl->last_read_segno == \ segno && ctrl->last_rsegfile_offset != ctrl->last_offset)){ HLOG_DEBUG("cur segno:%d is, last segno no:%d, \ last rsegfile offset:%d, last offset:%d,start_segno:%d - \ need close old and open new segfile", segno, \ ctrl->last_read_segno,ctrl->last_rsegfile_offset, \ ctrl->last_offset, ctrl->start_segno); /* close last seg file handler... */ if (ctrl->last_read_segno >= ctrl->start_segno) { storage = ctrl->storage; }else{ HLOG_DEBUG("get parent storage for segno:%d",segno); if (NULL == (storage = get_parent_storage(ctrl->family, \ ctrl->last_read_segno))){ g_assert(0); return -1; } } if (0 != storage->bs_file_close(storage, ctrl->last_rsegfile_handler)) { g_assert(0); return -1; } /* open cur seg file handler... */ const char segfile_name[SEGMENT_FILE_NAME_MAX]; build_segfile_name(segno, segfile_name); if (segno >= ctrl->start_segno) { storage = ctrl->storage; }else{ HLOG_DEBUG("get parent storage for segno:%d", segno); if (NULL == (storage = get_parent_storage(ctrl->family, segno))) { g_assert(0); return -1; } } bs_file_t file = storage->bs_file_open(storage, segfile_name,\ BS_READONLY); if (file == NULL) { HLOG_ERROR("can not open segment file %s", segfile_name); g_assert(0); return -1; } ctrl->last_rsegfile_handler = file; ctrl->last_rsegfile_offset = ctrl->last_offset; ctrl->last_read_segno = segno; }else{
int seg_usage_calc(struct back_storage* storage,uint32_t block_size,uint32_t segno,struct inode *refer_inode,SEG_USAGE_T *seg_usage) { //HLOG_DEBUG("enter func %s",__func__); if(storage == NULL || refer_inode == NULL || seg_usage == NULL){ HLOG_ERROR("input params failed"); return -1; } #if 0 if(seg_usage->bitmap!=0 && segno != seg_usage->segno){ HLOG_ERROR("segno not match"); return -1; } #endif int ret = 0; int log_idx=0; int idx; uint32_t offset = 0; struct log_header *lh; //gchar **v = g_strsplit(segfile,".",2); // uint64_t db_mine_storage_addr_segno = atol(v[0]); //g_strfreev(v); GArray *tmp_bit_array; seg_usage->segno = segno; seg_usage->timestamp = get_current_time(); HLOG_DEBUG("seg usage's segno:%llu,timestamp:%llu",seg_usage->segno,seg_usage->timestamp); char segfile[SEGMENT_FILE_NAME_MAX]; build_segfile_name(segno,segfile); char *content = NULL; uint32_t size; if(0!= (ret = file_get_contents(storage,segfile,&content,&size))){ HLOG_ERROR("read segfile:%s failed",segfile); return -1; } tmp_bit_array = g_array_new(FALSE,FALSE,sizeof(gint)); while(offset < size){ #if 0 ret=storage->bs_file_pread(storage,file, (char*)&lh, LOG_HEADER_LENGTH, offset) ;//TODO read 64M once g_message("read content len:%d\n",ret); if(ret<0){ ret = -1; goto out; }else if (ret == 0){ g_message("read over?\n"); ret = 0; break; }else if (ret == LOG_HEADER_LENGTH){ g_message("read log header over\n"); ; }else{ g_message("read log header failed\n"); ret = -1; goto out; } #endif lh = (struct log_header*)(content + offset); if(seg_usage->log_num !=0){ //HLOG_DEBUG("this segfile:%s has calc",segfile); idx = log_idx/sizeof(gint); if(!(seg_usage->bitmap[idx] & (1<<(log_idx%sizeof(gint))))){ log_idx++; int x=0; g_array_append_val(tmp_bit_array,x); offset += lh->log_size; continue; } } uint32_t orgine_alive_block_num = seg_usage->alive_block_num; //HLOG_DEBUG("start db no:%llu,db num:%d",lh->start_db_no,lh->db_num); int i; #if 1 /* check refer inode whether still refer to given db in seg */ for(i=0;i<lh->db_num;i++){ //HLOG_DEBUG("for db:%llu",lh->start_db_no+i); uint64_t db_mine_storage_addr = 0; uint32_t db_mine_storage_addr_offset = offset + LOG_HEADER_LENGTH+i*block_size; set_offset(&db_mine_storage_addr,db_mine_storage_addr_offset); set_segno (&db_mine_storage_addr,segno); uint64_t db_cur_storage_addr = get_db_storage_addr_in_inode(storage,refer_inode, lh->start_db_no+i,block_size); HLOG_DEBUG("db:%llu's mine storage addr:%llu,cur storage addr:%llu", lh->start_db_no+i,db_mine_storage_addr,db_cur_storage_addr); if(db_mine_storage_addr != db_cur_storage_addr){ HLOG_DEBUG("this is overwrite data block"); }else{ seg_usage->alive_block_num++; HLOG_DEBUG("this is used data block :%llu",seg_usage->alive_block_num); } seg_usage->block_num++; } #endif //uint32_t alive_blocks = log_usage_calc(storage,latest_inode,&lh,db_mine_storage_addr_segno,offset,block_size); if(orgine_alive_block_num == seg_usage->alive_block_num){ HLOG_DEBUG("log:%d has not any datablock",log_idx); //uint32_t bitmap_idx = log_idx / ALIVE_LOG_BITMAP ; //seg_usage->alive_log_bitmap[bitmap_idx] &= ~(1 << (log_idx % sizeof(uint64_t))); int x=0; g_array_append_val(tmp_bit_array,x); }else{ HLOG_DEBUG("log:%d has any datablock",log_idx); int x=1; g_array_append_val(tmp_bit_array,x); } offset += lh->log_size; log_idx++; } int i; seg_usage->log_num = tmp_bit_array->len; g_free(seg_usage->bitmap); seg_usage->bitmap = (char*)g_malloc0((seg_usage->log_num-1)/sizeof(gint)+1); //HLOG_DEBUG("size of bitmap:%d",tmp_bit_array->len); for(i=0;i<tmp_bit_array->len;i++){ gint value = g_array_index(tmp_bit_array,gint,i); idx = i/sizeof(gint); if(value==1){ //g_message("bitmap idx bit:%d = 1\n",i); seg_usage->bitmap[idx] |= 1<<i%sizeof(gint); //g_message("bitmap idx %x\n",seg_usage->bitmap[idx]); } } g_array_free(tmp_bit_array,TRUE); //HLOG_DEBUG("leave func %s",__func__); return 0; }
/* seg.clean -u uri -s segno -w copy_water -v verbose */ int main(int argc, char *argv[]) { GError *error = NULL; GOptionContext *context; context = g_option_context_new("- segment clearn..."); g_option_context_add_main_entries(context, entries, NULL); g_option_context_set_help_enabled(context, TRUE); g_option_group_set_error_hook(g_option_context_get_main_group(context), (GOptionErrorFunc)error_func); if (!g_option_context_parse(context, &argc, &argv, &error)) { g_message("option parsing failed: %s", error->message); exit(EXIT_FAILURE); } g_message("uri is :%s",uri); g_message("start segno:%d,end segno:%d",start_segno,end_segno); g_message("copy_waterlevel is: %d",copy_waterlevel); g_option_context_free(context); HLFS_CTRL * ctrl = init_hlfs(uri); g_assert(ctrl != NULL); g_message("ctrl init over"); int ret = 0; ret = hlfs_open(ctrl,1); g_message("ctrl open over"); g_assert(ret == 0); if(0!=ctrl->storage->bs_file_is_exist(ctrl->storage,SEGMENTS_USAGE_FILE)) { g_message("seg usage file not exit"); goto OUT; } GHashTable *seg_usage_hashtable = g_hash_table_new_full(g_direct_hash,g_direct_equal,NULL,NULL);//TODO GList* seg_usage_list = NULL; ret = load_all_seg_usage(ctrl->storage,SEGMENTS_USAGE_FILE,seg_usage_hashtable); g_assert(ret == 0); ret = sort_all_seg_usage(seg_usage_hashtable,&seg_usage_list); g_assert(ret == 0); int i; for(i=start_segno; i<=end_segno; i++) { SEG_USAGE_T *seg_usage = g_hash_table_lookup(seg_usage_hashtable,GINT_TO_POINTER((uint32_t)i)); char segfile[128]; build_segfile_name(i, segfile); if(seg_usage != NULL) { g_message("seg no:%d ...",seg_usage->segno); if (seg_usage->alive_block_num == 0) { g_message("seg no:%d no alive block now,delete it",i); if(0 == ctrl->storage->bs_file_is_exist(ctrl->storage,segfile)) { ret = ctrl->storage->bs_file_delete(ctrl->storage,segfile); g_assert(ret == 0); } else { g_message("seg no:%d has delete",i); } continue; } if(0 == strcmp(seg_usage->up_sname,EMPTY_UP_SNAPSHOT)) { g_message("seg no:%d is maybe need to migrate",seg_usage->segno); if (seg_usage->alive_block_num > copy_waterlevel) { g_message("seg no:%d is need to migrate",seg_usage->segno); ret = migrate_alive_blocks(ctrl,seg_usage); g_assert(ret == 0); seg_usage->alive_block_num =0; seg_usage->timestamp = get_current_time(); memset(seg_usage->bitmap,0,(seg_usage->log_num-1)/sizeof(gint)+1); ret = dump_seg_usage(ctrl->storage,SEGMENTS_USAGE_FILE,seg_usage); g_free(seg_usage->bitmap); g_free(seg_usage); ret = ctrl->storage->bs_file_delete(ctrl->storage,segfile); } } } else { g_message("seg no:%d has not yet do seg usage calc",i); } } OUT: ret = hlfs_close(ctrl); g_assert(ret == 0); ret = deinit_hlfs(ctrl); g_assert(ret ==0); return 0; }