Esempio n. 1
0
bs_file_info_t *hdfs_list_dir(struct back_storage *storage, \
		const char *dir_path, uint32_t *num_entries){
	HLOG_DEBUG("hdfs -- enter func %s", __func__);
	char full_path[256];
	build_hdfs_path(full_path, storage->dir, storage->fs_name, dir_path);
	int num;
	hdfsFileInfo *hinfos = \
			       hdfsListDirectory((hdfsFS)storage->fs_handler, full_path, &num);
	if (NULL == hinfos) {
		//HLOG_ERROR("hdfsListDirectory error");
		return NULL; 
	}
	hdfsFileInfo *hinfo = hinfos;
	bs_file_info_t *infos = \
				(bs_file_info_t*)g_malloc0(sizeof(bs_file_info_t)*8192);
	if (NULL == infos) {
		//HLOG_ERROR("Allocate Error!");
		return NULL;
	}
	bs_file_info_t *info = infos;
	int i;
	for (i = 0;i < num;i++) {
		strcpy((char *)info->name, \
				(const char *)g_path_get_basename(hinfo->mName));
		info->is_dir = 0;
		info->size = hinfo->mSize;
		info->lmtime = hinfo->mLastMod;
		info++;
		hinfo++;
	}
    hdfsFreeFileInfo(hinfos, num);
	*num_entries = num;
	HLOG_DEBUG("hdfs -- leave func %s", __func__);
	return infos;
}
Esempio n. 2
0
int hdfs_connect(struct back_storage *storage, const char *uri){
	HLOG_DEBUG("hdfs -- enter func %s,hostname:%s,port:%d", __func__,storage->hostname,storage->port);
	hdfsFS fs = hdfsConnect(storage->hostname, storage->port);
	// for local test
	if (NULL == fs) {
		HLOG_ERROR("fs is null, hdfsConnect error!");
		return -1;
	}
	//storage->uri = uri; 
	storage->fs_handler = (bs_fs_t)fs;
	HLOG_DEBUG("hdfs -- leave func %s", __func__);
	return 0;
}
Esempio n. 3
0
static void build_hdfs_path(char *full_path, const char *dir, \
		const char *fs_name, const char *path){
	HLOG_DEBUG("local -- enter func %s", __func__);
	memset(full_path, 0, 256);
	if (NULL != path) {
		sprintf(full_path, "%s/%s/%s", dir, fs_name, path);
	}else{
		sprintf(full_path, "%s/%s", dir, fs_name);
	}
	HLOG_DEBUG("path:%s,full path:%s",path,full_path);
	HLOG_DEBUG("local -- leave func %s", __func__);
	return;
}
Esempio n. 4
0
int __read_layer_iblock(struct hlfs_ctrl *hctrl,uint32_t dbno,int layerno,char *iblock_buf){
    //return -1;
    //HLOG_DEBUG("enter func %s", __func__);
    int ret;
    int ibno ;
    if(NULL == hctrl->icache){
        return -1;
    }
    if(layerno == 1){
        ibno = get_layer1_ibno(dbno);
    }else if(layerno == 2){
        ibno = get_layer2_ibno(dbno);
    }else if(layerno == 3){
        ibno = get_layer3_ibno(dbno);
    }else{
        g_assert(0);
    }
    //HLOG_DEBUG("dbno:%d,ibno:%d",dbno,ibno);
    g_assert(ibno >= 0);
    if(TRUE == icache_iblock_exist(hctrl->icache,ibno)){
        //*iblock_buf = g_malloc0(hctrl->icache->iblock_size);	
        ret =  icache_query_iblock(hctrl->icache,ibno,iblock_buf);
        if(0 != ret){
            HLOG_ERROR(" can not find iblock in icache ");
            return -1; 
        }
    }else{
        HLOG_DEBUG(" can not find iblock in icache ");
        return -1;   
    }
    return 0;	
}
Esempio n. 5
0
int hlfs_take_snapshot(struct hlfs_ctrl *ctrl, const char *ssname) 
{
	//HLOG_DEBUG("enter func %s", __func__);
	//HLOG_DEBUG("create ssname is %s", ssname);
	if(ctrl == NULL || ssname ==NULL){
		HLOG_ERROR("parameter error!");
		return -1;
	}
	int ret = 0;
	if ((strlen(ssname) + 1) > HLFS_FILE_NAME_MAX) {
		HLOG_ERROR("error, snapshot name beyond max length!");
		return -1;
	}

	g_mutex_lock(ctrl->hlfs_access_mutex);
	if (ctrl->rw_inode_flag == 0) {
		HLOG_ERROR("error, snapshot can not take when readonly");
		g_mutex_unlock (ctrl->hlfs_access_mutex);
		return -1;
	}
	g_mutex_unlock (ctrl->hlfs_access_mutex);

	struct snapshot *_ss = NULL;
	if (0 == (ret=load_snapshot_by_name (ctrl->storage, \
					SNAPSHOT_FILE, &_ss, ssname))){
		HLOG_ERROR("snapshot %s is exist, use another snapshot name", \
				ssname);
		return -1;
	}else{
		HLOG_DEBUG("snapshot %s is not exist , create it ", ssname);
	}

	struct snapshot ss;
	memset(&ss, 0, sizeof(struct snapshot));
	ss.timestamp = get_current_time();
	g_strlcpy(ss.sname, ssname, strlen(ssname) + 1);
	g_mutex_lock(ctrl->hlfs_access_mutex);
	sprintf(ss.up_sname, "%s", ctrl->alive_ss_name);
	ss.inode_addr = ctrl->imap_entry.inode_addr;
	memset(ctrl->alive_ss_name, 0, MAX_FILE_NAME_LEN);
	sprintf(ctrl->alive_ss_name, "%s", ss.sname);
	g_mutex_unlock (ctrl->hlfs_access_mutex);

	ret = dump_alive_snapshot(ctrl->storage, ALIVE_SNAPSHOT_FILE, &ss);
	if (ret != 0) {
		HLOG_ERROR("dump snapshot alive error!");
		return -1;
	}
	ret = dump_snapshot(ctrl->storage, SNAPSHOT_FILE, &ss);
	if (ret != 0) {
		HLOG_ERROR("dump snapshot error!");
		return -1;
	}
	HLOG_INFO("Take Snapshot Succ- snapshot_name:%s,last_segno:%d, \
			last_offset:%d", ssname, ctrl->last_segno, ctrl->last_offset);
	return ret;
}
Esempio n. 6
0
int faimly_init(FAMILY_CTRL *fctrl,char* furi,uint64_t fbase_inode,uint32_t fsegno){
	int ret = 0;
	
	struct back_storage *storage =NULL;
	char *uri = furi;
	char * father_uri = NULL;
    uint64_t base_father_inode,max_fs_size;
    uint32_t from_segno,seg_size,block_size;
	STORAGE_ITEM *storage_item = NULL;
    from_segno = fsegno;
    do{
	   father_uri=NULL;
	   if(NULL == (storage = init_storage_handler(uri))){
	   	  HLOG_ERROR("fail to init storage for uri:%s",uri);
		  ret = -1;
	  	  goto out;;
	   }	
	   //g_tree_insert(fctrl->seg_storage_map,GINT_TO_POINTER((uint32_t)(segno-1),storage);
	   storage_item = g_malloc0(sizeof(STORAGE_ITEM));
	   storage_item->storage = storage;
	   storage_item->segno = from_segno -1;
	   HLOG_DEBUG("from_segno:%d\n",from_segno);
	   fctrl->seg_storage_list = g_list_append(fctrl->seg_storage_list,storage_item);
       if(0 !=(ret = read_fs_meta_all(storage,&seg_size,&block_size,&max_fs_size,
		   			            &father_uri,&base_father_inode,&from_segno))){
		  HLOG_ERROR("fail to read fs meta info");
		  ret = -1;
		  goto out;
       }
       if(father_uri!=NULL){
	   	  HLOG_DEBUG("need read uri:%s 's father uri:%s",uri,father_uri);
		  HLOG_DEBUG("from_segno:%d",from_segno);
		  uri = father_uri;
       }else{
          HLOG_DEBUG("need read uri:%s is first uri",uri);
		  break;
       }
	}while(1);
    fctrl->from_segno = fsegno;
    fctrl->father_uri = g_strdup(furi);
    fctrl->base_father_inode = fbase_inode;
out:
	return ret;
}
Esempio n. 7
0
static gchar *build_hdfs_path(const char *uri, const char *path){
	HLOG_DEBUG("hdfs -- enter func %s", __func__);
	char *head = NULL;
	char *hostname = NULL;
	char *dir = NULL;
	char *fs_name = NULL;
	int port;
	int ret = parse_from_uri(uri, &head, &hostname, &dir, &fs_name, &port);
	if (ret != 0) {
		HLOG_ERROR("parse_from_uri error!");
		return NULL;
	}
	gchar *full_path = g_build_filename(dir, fs_name, path, NULL);
	HLOG_DEBUG("full path is %s", full_path);
	g_free(head);
	g_free(hostname);
	//g_free(dir);
	HLOG_DEBUG("hdfs -- leave func %s", __func__);
	return full_path;
}
Esempio n. 8
0
struct back_storage * get_parent_storage(FAMILY_CTRL *fctrl, uint32_t segno){
        struct  back_storage * storage = NULL;
	 int i = 0;
	 for(i = g_list_length(fctrl->seg_storage_list)-1; i >= 0; i--){
        STORAGE_ITEM *storage_item = g_list_nth_data(fctrl->seg_storage_list,i);
        HLOG_DEBUG("storage_item's segno:%d,search segno:%d",storage_item->segno,segno);
        if(storage_item->segno >= segno){
		   storage = storage_item->storage;
           break; 
        }
    }
    return storage;
}
Esempio n. 9
0
int hlfs_rmfs(struct back_storage *storage){
        HLOG_DEBUG("enter func %s", __func__);
        int ret = 0;
        uint32_t num_entries;
        bs_file_info_t *infos = storage->bs_file_list_dir(storage, NULL, &num_entries);
        if (infos == NULL) {
                HLOG_ERROR("can not get fs:%s seg entries", storage->uri);
                return -1;
        }
        printf("how much file :%d\n", num_entries);
        int i = 0;
        bs_file_info_t *info = infos;
        for (i = 0;i < num_entries;i++) {
                printf("999 file:%s, size:%llu, time:%llu\n", info->name, \
                                info->size, info->lmtime);  
				storage->bs_file_delete(storage, info->name);
                info++;
		}
		storage->bs_file_delete(storage, NULL);
        printf("leave func %s", __func__);
        free(infos);
        return ret;  
}
Esempio n. 10
0
int hlfs_read(struct hlfs_ctrl *ctrl, char* read_buf, uint32_t read_len, uint64_t pos)
{
	
    if((NULL == read_buf) || (NULL == ctrl) || (0 == read_len)){
	    HLOG_ERROR("Params Error");
           return -1;
    }

    if(ctrl->sb.max_fs_size *1024 *1024< pos+read_len){
          HLOG_ERROR("your config only allow write beyond :%llu",ctrl->sb.max_fs_size);
          //g_mutex_unlock (ctrl->hlfs_access_mutex);
          return -1;
    }		
    //g_mutex_lock (ctrl->hlfs_access_mutex);
    HLOG_INFO("Hlfs Read Req pos:%llu,read_len:%d,last_segno:%d,last_offset:%d,cur_file_len:%llu",
    						      pos,
    						      read_len,
    						      ctrl->last_segno,
    						      ctrl->last_offset,
    						      ctrl->inode.length);
    guint32 BLOCKSIZE = ctrl->sb.block_size;
    HLOG_DEBUG("read offset:%llu,read len:%d", pos,read_len);
    int ret = 0;
    int start_db = 0;
    if(pos/BLOCKSIZE == (pos+read_len-1)/BLOCKSIZE){
		HLOG_DEBUG("only need to read one block: %llu", pos / BLOCKSIZE);
        char *block = (char*)alloca(BLOCKSIZE);
        //g_mutex_lock (ctrl->hlfs_access_mutex);
        ret=load_block_by_addr_fast(ctrl,pos,block);
        //g_mutex_unlock (ctrl->hlfs_access_mutex);
        if(-1 == ret ){
	     HLOG_ERROR("fail to load block for addr %llu", pos);
            //g_mutex_unlock (ctrl->hlfs_access_mutex);
            return -1;
        }else if(1==ret){
            //HLOG_DEBUG("fail to load block for not write yet");
            memset(block,0,BLOCKSIZE);
        }
        memcpy(read_buf,block + pos%BLOCKSIZE,read_len);
        //g_free(block);
        //g_mutex_unlock (ctrl->hlfs_access_mutex);
        HLOG_DEBUG("read len %u", read_len);
        return read_len;
    }

    HLOG_DEBUG("need to read muti block", __func__);
    uint32_t offset=0; 
    if( pos % BLOCKSIZE != 0 ){
	    HLOG_DEBUG("need to read first block", __func__);
        char *first_block = (char*)alloca(BLOCKSIZE);
        //g_mutex_lock (ctrl->hlfs_access_mutex);
        ret=load_block_by_addr_fast(ctrl,pos,first_block);
        //g_mutex_unlock (ctrl->hlfs_access_mutex);
        if(-1 == ret){
            HLOG_ERROR("fail to load block for addr %llu", pos);
            //g_mutex_unlock (ctrl->hlfs_access_mutex);
            return -1;
        }else if(1 == ret){
            //HLOG_DEBUG("fail to load block for not write yet");
            memset(first_block,0,BLOCKSIZE);
        }
        memcpy(read_buf,first_block + pos%BLOCKSIZE, BLOCKSIZE - pos%BLOCKSIZE);
        offset += BLOCKSIZE - pos%BLOCKSIZE;
        HLOG_DEBUG("fist offset:%u", offset);
        //g_free(block);
        start_db = (pos + BLOCKSIZE)/BLOCKSIZE;
    }else{
        start_db = pos/BLOCKSIZE;
    }
    int end_db = (pos+read_len)/BLOCKSIZE;
    HLOG_DEBUG("start db: %d end db: %d", start_db, end_db);
    int i;
	//char *block = (char*)alloca(BLOCKSIZE);
    for(i = start_db; i < end_db;i++){
        //g_mutex_lock (ctrl->hlfs_access_mutex);
        char *block = read_buf+offset;
        ret=load_block_by_no_fast(ctrl,i,block);
        //g_mutex_unlock (ctrl->hlfs_access_mutex);
        if(-1 == ret){
            HLOG_ERROR("fail to load block for no %d", i);
            //g_mutex_unlock (ctrl->hlfs_access_mutex);
            return -1;
        }else if(1==ret){
            //HLOG_DEBUG("fail to load block for not write yet");
            memset(block,0,BLOCKSIZE);
        }
        //memcpy(read_buf+offset,block,BLOCKSIZE);
        offset +=BLOCKSIZE;
        HLOG_DEBUG("offset: %u", offset);
        //g_free(block);
    }
    if((pos + read_len)% BLOCKSIZE != 0 ){
        HLOG_DEBUG("need to read last block", __func__);
        char *last_block = (char*)alloca(BLOCKSIZE);
        //g_mutex_lock (ctrl->hlfs_access_mutex);
        ret=load_block_by_addr_fast(ctrl,pos+read_len,last_block);
        //g_mutex_unlock (ctrl->hlfs_access_mutex);
        if(-1 == ret){
            HLOG_ERROR("fail to load block for addr %llu", pos + read_len);
            //g_mutex_unlock (ctrl->hlfs_access_mutex);
            return -1;
        }else if (1==ret){
            //HLOG_DEBUG("fail to load block for not write yet");
            memset(last_block,0,BLOCKSIZE);
        }
        memcpy(read_buf + offset , last_block , (pos + read_len)%BLOCKSIZE );
        offset +=(pos+read_len)%BLOCKSIZE;
        //g_free(block);
    }
    //g_mutex_unlock (ctrl->hlfs_access_mutex);
    //ctrl->last_access_timestamp = get_current_time();
    ctrl->last_read_timestamp = get_current_time();
    HLOG_DEBUG("leave func %s", __func__);
    return offset;
}
Esempio n. 11
0
static int __load_block_by_no(struct hlfs_ctrl *ctrl,uint32_t no,READ_BLOCK_FUN RB_FUN,char *block){
	//HLOG_DEBUG("enter func %s,no:%d", __func__,no);
    int ret =0;
    if(ctrl->cctrl!=NULL){
	      //HLOG_DEBUG("read from cache first");
          //*block = g_malloc0(ctrl->cctrl->block_size);
         ret = cache_query_block(ctrl->cctrl,no,block);
         if(ret == 0 ){
	      HLOG_DEBUG("succ to get block from cache!");
          return 0;
      }
      //g_free(*block);
	  HLOG_DEBUG("not find in cache!");
    }
    
    uint64_t storage_address ;
    guint32  BLOCKSIZE = ctrl->sb.block_size;
    uint32_t db_no = no;
    uint32_t IB_ENTRY_NUM = BLOCKSIZE/sizeof(uint64_t);
    if( (db_no+1)*BLOCKSIZE >ctrl->inode.length){
        HLOG_DEBUG("beyond current inode's length:%llu",ctrl->inode.length);
        return 1;
    }
    if(is_db_in_level1_index_range(db_no)){
        int _idx = db_no % 12;
        storage_address = ctrl->inode.blocks[_idx];
    }else if (is_db_in_level2_index_range(db_no)){
        //HLOG_DEBUG("inode.iblock :%llu",ctrl->inode.iblock);	
        if(ctrl->inode.iblock == 0){
		  //HLOG_DEBUG("inode.iblock zero");	
          return 1;
        }	
	 char *_ib= (char *)alloca(BLOCKSIZE);
	 memset(_ib,0,sizeof(BLOCKSIZE));
	 if(0>read_layer1_iblock(ctrl,db_no,(char*)_ib)){
        	if(0 != RB_FUN(ctrl,ctrl->inode.iblock,(char*)_ib)){
			   HLOG_ERROR("read_block error for iblock_addr:%llu",ctrl->inode.iblock);
			   return -1;
	 	    }
		    write_layer1_iblock(ctrl,db_no,_ib);	
	    }
        int  _idx = (db_no-12)%IB_ENTRY_NUM;
        storage_address = *(_ib+_idx);
        //g_free(_ib);
    }else if (is_db_in_level3_index_range(db_no)){
        //HLOG_DEBUG("inode.double_iblock :%llu",ctrl->inode.doubly_iblock);	
        if(ctrl->inode.doubly_iblock ==0){
			HLOG_DEBUG("inode.doubly_iblock zero");	
            return 1;
        }
        char *_ib= (char *)alloca(BLOCKSIZE);
	 memset(_ib,0,sizeof(BLOCKSIZE));
	 if(0>read_layer1_iblock(ctrl,db_no,(char*)_ib)){
	 	if(0 != RB_FUN(ctrl,ctrl->inode.doubly_iblock,(char*)_ib)){
				HLOG_ERROR("read_block error for doubly_iblock_addr:%llu",ctrl->inode.doubly_iblock);
				return -1;
		 	}
			write_layer1_iblock(ctrl,db_no,_ib);	
	 }	
        int _idx   = ( db_no - 12 - IB_ENTRY_NUM)/IB_ENTRY_NUM;
        //HLOG_DEBUG("ib1 address:%llu",*(_ib+_idx));
        if(*(_ib+_idx) == 0 ){
            return 1;
        }
        char *_ib2=(char *)alloca(BLOCKSIZE);
	    memset(_ib2,0,sizeof(BLOCKSIZE));
	 if( 0> read_layer2_iblock(ctrl,db_no,_ib2)){
		if( 0!= RB_FUN(ctrl,*(_ib+_idx),(char*)_ib2)){	
			//HLOG_ERROR("read_block error");
			return -1;
		}
		write_layer2_iblock(ctrl,db_no,_ib2);
	}
        int _idx2  = (db_no - 12 - IB_ENTRY_NUM)%IB_ENTRY_NUM;
        storage_address = *(_ib2 + _idx2);
        //g_free(_ib);
        //g_free(_ib2);
    }else if (is_db_in_level4_index_range(db_no)){
        if(ctrl->inode.triply_iblock == 0){
			//HLOG_DEBUG("inode.triply_iblock zero");	
            return 1;
        }
        char *_ib =(char *)alloca(BLOCKSIZE);
	 memset(_ib,0,sizeof(BLOCKSIZE));
	 if(0>read_layer1_iblock(ctrl,db_no,(char*)_ib)){	
	 	 if(0 != RB_FUN(ctrl,ctrl->inode.triply_iblock,(char*)_ib)){
			//HLOG_ERROR("read_block error for triply_iblock_addr:%llu",ctrl->inode.triply_iblock);
			return -1;
		}
			write_layer1_iblock(ctrl,db_no,_ib);
	 }
        int _idx   = (db_no -12 - IB_ENTRY_NUM - IB_ENTRY_NUM*IB_ENTRY_NUM) / (IB_ENTRY_NUM*IB_ENTRY_NUM);
        if(*(_ib + _idx) == 0){
            return 1;
        }
        char *_ib2 = (char *)alloca(BLOCKSIZE);
	 memset(_ib2,0,sizeof(BLOCKSIZE));
	 if(0>read_layer2_iblock(ctrl,db_no,_ib2)){
	 	 if(0 != RB_FUN(ctrl,*(_ib + _idx),(char*)_ib2)){
			//HLOG_ERROR("read_block error");
			return -1;
		}
		write_layer2_iblock(ctrl,db_no,_ib2);	
	}	
        int _idx2  = (db_no-12 - IB_ENTRY_NUM - IB_ENTRY_NUM*IB_ENTRY_NUM)/IB_ENTRY_NUM % IB_ENTRY_NUM;
        if(*(_ib2 + _idx2) == 0){
            return 1;
        }
        char *_ib3 = (char *)alloca(BLOCKSIZE);
	 memset(_ib3,0,sizeof(BLOCKSIZE));
	 if(0>read_layer3_iblock(ctrl,db_no,(char*)_ib3)){
	 	if(0 != RB_FUN(ctrl,*(_ib2 + _idx2),(char*)_ib3)){
			//HLOG_ERROR("read_block error");
			return -1;
		}
		write_layer3_iblock(ctrl,db_no,_ib3);	
	 }
        int _idx3  = (db_no-12 - IB_ENTRY_NUM - IB_ENTRY_NUM*IB_ENTRY_NUM) % IB_ENTRY_NUM; 
        storage_address = *(_ib3 + _idx3);
        //g_free(_ib);
        //g_free(_ib2);
        //g_free(_ib3);
    }
	HLOG_DEBUG("storage address:%llu",storage_address);
    if(storage_address == 0){
        return 1;
    }
    ret = RB_FUN(ctrl,storage_address,block);
  
    //HLOG_DEBUG("leave func %s", __func__);
    return ret;
}
Esempio n. 12
0
int prev_open_rsegfile(struct hlfs_ctrl *ctrl, uint32_t segno){
	//HLOG_DEBUG("enter func %s", __func__);
	struct back_storage *storage = NULL;
	if (NULL == ctrl->last_rsegfile_handler) {
		char segfile_name[SEGMENT_FILE_NAME_MAX];
		memset((void *)segfile_name, 0, SEGMENT_FILE_NAME_MAX);
		build_segfile_name(segno, segfile_name);
		bs_file_t file;   
		if (segno >= ctrl->start_segno) {
			storage = ctrl->storage;
		}else{
			HLOG_DEBUG("get parent storage for segno:%d",segno);
			if (NULL == (storage = get_parent_storage(ctrl->family, segno))){
				g_assert(0);
				return -1;
			}
		}
		file = storage->bs_file_open(storage, segfile_name, BS_READONLY); 
		if (file == NULL) {
			HLOG_ERROR("can not open segment file %s", segfile_name);
			g_assert(0);
			return -1;
		}
		ctrl->last_rsegfile_handler = file;
		ctrl->last_rsegfile_offset = ctrl->last_offset;
		ctrl->last_read_segno =segno;
	}else if (ctrl->last_read_segno != segno || (ctrl->last_read_segno == \
				segno && ctrl->last_rsegfile_offset != ctrl->last_offset)){
		HLOG_DEBUG("cur segno:%d is, last segno no:%d, \
				last rsegfile offset:%d, last offset:%d,start_segno:%d - \
				need close old and open new segfile", segno, \
				ctrl->last_read_segno,ctrl->last_rsegfile_offset, \
				ctrl->last_offset, ctrl->start_segno);
		/* close last seg file handler...  */
		if (ctrl->last_read_segno >= ctrl->start_segno) {
			storage = ctrl->storage;
		}else{
			HLOG_DEBUG("get parent storage for segno:%d",segno);
			if (NULL == (storage = get_parent_storage(ctrl->family, \
							ctrl->last_read_segno))){
				g_assert(0);
				return -1;
			}
		}
		if (0 != storage->bs_file_close(storage, ctrl->last_rsegfile_handler)) {
			g_assert(0);
			return -1;
		}	
		/* open cur seg file handler...  */
		const char segfile_name[SEGMENT_FILE_NAME_MAX];
		build_segfile_name(segno, segfile_name);
		if (segno >= ctrl->start_segno) {
			storage = ctrl->storage;
		}else{
			HLOG_DEBUG("get parent storage for segno:%d", segno);	
			if (NULL == (storage = get_parent_storage(ctrl->family, segno))) {
				g_assert(0);
				return -1;
			}
		}
		bs_file_t file = storage->bs_file_open(storage, segfile_name,\
				BS_READONLY); 
		if (file == NULL) {
			HLOG_ERROR("can not open segment file %s", segfile_name);
			g_assert(0);
			return -1;
		}
		ctrl->last_rsegfile_handler = file;
		ctrl->last_rsegfile_offset  = ctrl->last_offset;
		ctrl->last_read_segno       = segno;
	}else{
Esempio n. 13
0
int seg_usage_calc(struct back_storage* storage,uint32_t block_size,uint32_t segno,struct inode *refer_inode,SEG_USAGE_T *seg_usage)
{
    //HLOG_DEBUG("enter func %s",__func__);
    if(storage == NULL || refer_inode == NULL || seg_usage == NULL){
		HLOG_ERROR("input params failed");
		return -1;
    }
#if 0
	if(seg_usage->bitmap!=0 && segno != seg_usage->segno){
		HLOG_ERROR("segno not match");
		return -1;
	}	
#endif	
    int ret = 0;
    int log_idx=0;
    int idx;
    uint32_t offset = 0; 
    struct log_header *lh;
    //gchar **v = g_strsplit(segfile,".",2);
    // uint64_t db_mine_storage_addr_segno = atol(v[0]);
    //g_strfreev(v);

    GArray *tmp_bit_array;
    seg_usage->segno = segno; 
    seg_usage->timestamp = get_current_time();
    HLOG_DEBUG("seg usage's segno:%llu,timestamp:%llu",seg_usage->segno,seg_usage->timestamp);

    char segfile[SEGMENT_FILE_NAME_MAX];
	build_segfile_name(segno,segfile);
	char *content = NULL;
	uint32_t size;
	if(0!= (ret = file_get_contents(storage,segfile,&content,&size))){
	    HLOG_ERROR("read segfile:%s failed",segfile);
		return -1;
	}		

    tmp_bit_array = g_array_new(FALSE,FALSE,sizeof(gint));
	

    while(offset < size){
#if 0
        ret=storage->bs_file_pread(storage,file, (char*)&lh, LOG_HEADER_LENGTH, offset) ;//TODO read 64M once
        g_message("read content len:%d\n",ret);
        if(ret<0){
            ret = -1;
            goto out;
        }else if (ret == 0){
            g_message("read over?\n");
            ret = 0;
            break;
        }else if (ret == LOG_HEADER_LENGTH){
            g_message("read log header over\n");
            ;
        }else{
            g_message("read log header failed\n");
            ret = -1;
            goto out;
        }
#endif 
        lh = (struct log_header*)(content + offset);
        if(seg_usage->log_num !=0){
            //HLOG_DEBUG("this segfile:%s has calc",segfile);
            idx = log_idx/sizeof(gint);
            if(!(seg_usage->bitmap[idx] & (1<<(log_idx%sizeof(gint))))){
                log_idx++;
                int x=0;
                g_array_append_val(tmp_bit_array,x);
                offset += lh->log_size;
                continue;
            }
        }
        uint32_t orgine_alive_block_num = seg_usage->alive_block_num;
        //HLOG_DEBUG("start db no:%llu,db num:%d",lh->start_db_no,lh->db_num);
        int i;
#if 1 /* check refer inode whether still refer to given db in seg */
        for(i=0;i<lh->db_num;i++){
            //HLOG_DEBUG("for db:%llu",lh->start_db_no+i);
            uint64_t db_mine_storage_addr = 0;
            uint32_t db_mine_storage_addr_offset = offset + LOG_HEADER_LENGTH+i*block_size;
            set_offset(&db_mine_storage_addr,db_mine_storage_addr_offset);
            set_segno (&db_mine_storage_addr,segno);
            uint64_t db_cur_storage_addr = get_db_storage_addr_in_inode(storage,refer_inode,
			    						                                lh->start_db_no+i,block_size);
            HLOG_DEBUG("db:%llu's mine storage addr:%llu,cur storage addr:%llu",
			    	lh->start_db_no+i,db_mine_storage_addr,db_cur_storage_addr);
            if(db_mine_storage_addr != db_cur_storage_addr){
                HLOG_DEBUG("this is overwrite data block");
            }else{
                seg_usage->alive_block_num++;
                HLOG_DEBUG("this is used data block :%llu",seg_usage->alive_block_num);
            }
			seg_usage->block_num++;
        }
#endif
        //uint32_t alive_blocks = log_usage_calc(storage,latest_inode,&lh,db_mine_storage_addr_segno,offset,block_size);
        if(orgine_alive_block_num == seg_usage->alive_block_num){
            HLOG_DEBUG("log:%d has not any datablock",log_idx);
            //uint32_t bitmap_idx = log_idx / ALIVE_LOG_BITMAP ;
            //seg_usage->alive_log_bitmap[bitmap_idx] &= ~(1 << (log_idx % sizeof(uint64_t)));
            int x=0;
            g_array_append_val(tmp_bit_array,x);
        }else{
            HLOG_DEBUG("log:%d has any datablock",log_idx);
            int x=1;
            g_array_append_val(tmp_bit_array,x);
        }
        offset += lh->log_size;
        log_idx++;    
    }

    int i;
    seg_usage->log_num = tmp_bit_array->len;
    g_free(seg_usage->bitmap);
    seg_usage->bitmap = (char*)g_malloc0((seg_usage->log_num-1)/sizeof(gint)+1);
    //HLOG_DEBUG("size of bitmap:%d",tmp_bit_array->len);
    for(i=0;i<tmp_bit_array->len;i++){
        gint value = g_array_index(tmp_bit_array,gint,i);
        idx = i/sizeof(gint);
        if(value==1){
           //g_message("bitmap idx bit:%d = 1\n",i);
           seg_usage->bitmap[idx] |= 1<<i%sizeof(gint);
           //g_message("bitmap idx %x\n",seg_usage->bitmap[idx]);
       }
    }
    g_array_free(tmp_bit_array,TRUE);
	//HLOG_DEBUG("leave func %s",__func__);
    return 0;
}
Esempio n. 14
0
/*
 * hlfs_open: open a file.
 * @param ctrl: the global control.
 * @param flag: the flag for open operation, flag == 0
 *        readonly and flag == 1 writable.
 * @return: if successful return 0, else return -1.
 */
int hlfs_open(struct hlfs_ctrl *ctrl, int flag)
{
	HLOG_DEBUG("enter func %s", __func__);
	if (ctrl == NULL ||(flag != 0 && flag != 1)) { /* check the parameters */
		HLOG_ERROR("error params :flag %d", flag);
		return -1;
	}

	if (1 == flag) {
		ctrl->rw_inode_flag = 1;
	} else if (0 == flag) {
		ctrl->rw_inode_flag = 0;
	} else {
		HLOG_ERROR("the bad flag for hlfs open by inode");
        return -1;
    }

	if (ctrl->usage_ref > 0) {
		HLOG_DEBUG("This fs has opened by other,can not use it"); 
        return -1;
	}
    int ret = 0;
    HLOG_DEBUG("inode no %llu , inode address %llu", ctrl->imap_entry.inode_no, ctrl->imap_entry.inode_addr);
    if (ctrl->imap_entry.inode_no == 0 && ctrl->imap_entry.inode_addr == 0) { /* no inode condition */
	 HLOG_DEBUG("empty filesystem %s", ctrl->sb.fsname);
        if (flag == 0) {
			HLOG_ERROR("must create it with writeable flag");
			return -1;
	 }
        HLOG_DEBUG("create new fs inode !");
        ctrl->inode.length = 0;
        ctrl->inode.mtime = get_current_time();
        //ctrl->inode.ctime = get_current_time();
        //ctrl->inode.atime = get_current_time();
	} else { /* exist inode */
	      HLOG_DEBUG("this is not empty filesystem:%s", ctrl->sb.fsname);
	      struct back_storage *storage = NULL;
	      uint32_t segno = get_segno(ctrl->imap_entry.inode_addr);
             if (segno >= ctrl->start_segno) {
            		storage = ctrl->storage;
            }else{
            		HLOG_DEBUG("get parent storage for segno:%d", segno);
                     if (NULL == (storage = \
			get_parent_storage(ctrl->family, segno))){
                               g_assert(0);
                               return -1;
                     }
            } 
         struct inode *my_inode = NULL;
	    
	     my_inode = load_inode(storage, ctrl->imap_entry.inode_addr);    
	     if (my_inode == NULL) {
		     HLOG_ERROR("load_inode error!");
             return -1;
	     }
	    
	     HLOG_DEBUG("inode'length:%llu, ctrl->inode length:%llu, \
		sizeof inode:%d",my_inode->length,ctrl->inode.length, \
		sizeof(struct inode));	 
            memcpy(&(ctrl->inode), my_inode, sizeof(struct inode));
            g_free(my_inode);
	}
	HLOG_DEBUG("ctrl->rw_inode_flag:%d", ctrl->rw_inode_flag);
       struct snapshot *ss;
	if (0 == ctrl->storage->bs_file_is_exist(ctrl->storage, \
		SNAPSHOT_FILE)) {
        ret = find_latest_alive_snapshot(ctrl->storage, \
		ALIVE_SNAPSHOT_FILE, SNAPSHOT_FILE, &ss);
        if (ret != 0) {
		    HLOG_DEBUG("can not read alive snapshot, \
			there must be some error");
            return -1; 
        }
Esempio n. 15
0
int flush_work(gpointer data){
	int ret = 0;
	CACHE_CTRL *cctrl = (CACHE_CTRL*)data;
	GTimeVal expired;
	char *tmp_buf = (char *)g_malloc0(cctrl->block_size \
			*cctrl->flush_once_size);
	g_assert(tmp_buf);
	while (!cctrl->flush_worker_should_exit) {
		HLOG_DEBUG("-- flush worker doing --");
		g_get_current_time(&expired);
		g_time_val_add(&expired, cctrl->flush_interval * 1000 * 1000);
		g_mutex_lock(cctrl->cache_mutex);
		gboolean res = g_cond_timed_wait(cctrl->flush_waken_cond, \
				cctrl->cache_mutex, &expired);
		g_mutex_unlock(cctrl->cache_mutex); 
		HLOG_DEBUG(" time wait res for cond is :%d !",res);
		if (cctrl->flush_worker_should_exit) {
			HLOG_INFO("-- flush worker should exit --");
			break;
		}

		do {
			GSList *continue_blocks = NULL;
			ret = get_continues_blocks(cctrl, &continue_blocks);
			g_assert(ret==0);
			uint32_t blocks_count = g_slist_length(continue_blocks);
			uint32_t buff_len = blocks_count *cctrl->block_size;
			HLOG_DEBUG("--blocks_count:%d, buff_len:%d--", \
					blocks_count, buff_len);
			if (res == TRUE && buff_len == 0) {
				HLOG_ERROR("Never reach here");
				g_assert(0);
			}


			if (buff_len == 0) {
				HLOG_DEBUG("do not need flush now");
				break;
			}
			if (NULL == cctrl->write_callback_func) {
				HLOG_WARN("--not given flush callback func--");
				break;
			} 
			//char* tmp_buf = g_malloc0(buff_len);
			//g_assert(tmp_buf!=NULL);
			uint32_t start_no;
			uint32_t end_no; 
			int i = 0;
			for (i = 0; i < blocks_count; i++) {
				block_t *block = g_slist_nth_data(continue_blocks, i);
				if (i == 0) {
					start_no = block->block_no;
				}
				if (i == blocks_count-1) {
					end_no = block->block_no;
				}
				memcpy(tmp_buf + i * cctrl->block_size, \
						block->block, cctrl->block_size);
			}
			//HLOG_DEBUG("--tmp_buf:%p",tmp_buf);
			ret = cctrl->write_callback_func(cctrl->write_callback_user_param, \
					tmp_buf, start_no,end_no);
			g_assert(ret >= 0);
			//g_free(tmp_buf);
			if (ret >= 0 ) {
				HLOG_DEBUG("--signal write thread--");
				g_mutex_lock(cctrl->cache_mutex);
				__free_from_cache(cctrl, continue_blocks);
				//g_cond_broadcast(cctrl->writer_waken_cond);
				g_cond_signal(cctrl->writer_waken_cond);
				g_mutex_unlock(cctrl->cache_mutex);
				g_slist_free(continue_blocks);
				//HLOG_DEBUG("--return blocks to cache over--");
			}
		} while (get_cache_free_size(cctrl) < cctrl->flush_trigger_level \
				*cctrl->cache_size / 100 || (res == 0 && get_cache_free_size(cctrl) != 0));
	}
	g_free(tmp_buf);
	HLOG_INFO("--flush worker exit--");
	return 0;
}