int
hlfs_get_inode_info(const char *uri,
uint64_t inode_addr,
uint64_t *mtime,
uint64_t *length) {
int ret = 0;
struct back_storage *storage = init_storage_handler(uri);
if (NULL == uri) {
HLOG_ERROR("init storage handler error!");
ret = -1;
goto out;
}
struct inode *inode = load_inode(storage, inode_addr);
if (NULL == inode) {
HLOG_ERROR("load inode error!");
ret = -1;
goto out;
}
*mtime = inode->mtime;
*length = inode->length;
out:
g_free(storage);
g_free(inode);
return ret;
}
static int load_latest_inode(struct hlfs_ctrl *ctrl)
{
//HLOG_DEBUG("enter func %s", __func__);
if (NULL == ctrl) {
HLOG_ERROR("Params Error");
return -1;
}
int ret = 0;
const char segfile_name[SEGMENT_FILE_NAME_MAX];
build_segfile_name(ctrl->last_segno,segfile_name);
bs_file_t file = ctrl->storage->bs_file_open \
(ctrl->storage,segfile_name, BS_READONLY);
if (file == NULL) {
HLOG_ERROR("can not open segment file %s", segfile_name);
goto out;
}
uint64_t inode_pos = ctrl->last_offset - \
sizeof(struct inode_map_entry) -sizeof(struct inode);
//HLOG_DEBUG("inode pos %llu",inode_pos);
if(sizeof(struct inode) != ctrl->storage-> \
bs_file_pread(ctrl->storage, file, (char *)&ctrl->inode, \
sizeof(struct inode), inode_pos)){
HLOG_ERROR("can not read inode from %s", segfile_name);
ret = -1;
}
ctrl->storage->bs_file_close(ctrl->storage, file);
out:
//HLOG_DEBUG("leave func %s", __func__);
return ret;
}
static int load_snapshot_from_text(struct snapshot **ss, \
const char *buf, int *flag)
{
//HLOG_DEBUG("enter func %s", __func__);
const char *sep = SS_ITEM_SEP;
gchar **v = g_strsplit(buf, sep, 2);
if ('+' == v[0][0]) {
*flag = 0;
gchar **_v = g_strsplit(v[1], sep, 3);
gchar *_ss_name = v[0] + 1;
gchar *_version = _v[0];
gchar *_ime_inode_addr = _v[1];
gchar *_up_ss_name = _v[2];
char *endptr = NULL;
(*ss) = (struct snapshot*)g_malloc0(sizeof(struct snapshot));
if ((*ss) == NULL) {
HLOG_ERROR("Allocate error!");
return EHLFS_MEM;
}
(*ss)->timestamp = strtoull(_version, &endptr, 0);
sprintf((*ss)->sname, "%s", _ss_name);
sprintf((*ss)->up_sname, "%s", _up_ss_name);
(*ss)->inode_addr = strtoull(_ime_inode_addr, &endptr, 0);
HLOG_DEBUG("ss->timestamp:%llu ss->sname:%s ss->up_sname:%s, \
ss->inode_addr: %llu", (*ss)->timestamp, (*ss)->sname, \
(*ss)->up_sname, (*ss)->inode_addr);
g_strfreev(_v);
} else if ('-' == v[0][0]) {
int __read_layer_iblock(struct hlfs_ctrl *hctrl,uint32_t dbno,int layerno,char *iblock_buf){
//return -1;
//HLOG_DEBUG("enter func %s", __func__);
int ret;
int ibno ;
if(NULL == hctrl->icache){
return -1;
}
if(layerno == 1){
ibno = get_layer1_ibno(dbno);
}else if(layerno == 2){
ibno = get_layer2_ibno(dbno);
}else if(layerno == 3){
ibno = get_layer3_ibno(dbno);
}else{
g_assert(0);
}
//HLOG_DEBUG("dbno:%d,ibno:%d",dbno,ibno);
g_assert(ibno >= 0);
if(TRUE == icache_iblock_exist(hctrl->icache,ibno)){
//*iblock_buf = g_malloc0(hctrl->icache->iblock_size);
ret = icache_query_iblock(hctrl->icache,ibno,iblock_buf);
if(0 != ret){
HLOG_ERROR(" can not find iblock in icache ");
return -1;
}
}else{
HLOG_DEBUG(" can not find iblock in icache ");
return -1;
}
return 0;
}
int faimly_init(FAMILY_CTRL *fctrl,char* furi,uint64_t fbase_inode,uint32_t fsegno){
int ret = 0;
struct back_storage *storage =NULL;
char *uri = furi;
char * father_uri = NULL;
uint64_t base_father_inode,max_fs_size;
uint32_t from_segno,seg_size,block_size;
STORAGE_ITEM *storage_item = NULL;
from_segno = fsegno;
do{
father_uri=NULL;
if(NULL == (storage = init_storage_handler(uri))){
HLOG_ERROR("fail to init storage for uri:%s",uri);
ret = -1;
goto out;;
}
//g_tree_insert(fctrl->seg_storage_map,GINT_TO_POINTER((uint32_t)(segno-1),storage);
storage_item = g_malloc0(sizeof(STORAGE_ITEM));
storage_item->storage = storage;
storage_item->segno = from_segno -1;
HLOG_DEBUG("from_segno:%d\n",from_segno);
fctrl->seg_storage_list = g_list_append(fctrl->seg_storage_list,storage_item);
if(0 !=(ret = read_fs_meta_all(storage,&seg_size,&block_size,&max_fs_size,
&father_uri,&base_father_inode,&from_segno))){
HLOG_ERROR("fail to read fs meta info");
ret = -1;
goto out;
}
if(father_uri!=NULL){
HLOG_DEBUG("need read uri:%s 's father uri:%s",uri,father_uri);
HLOG_DEBUG("from_segno:%d",from_segno);
uri = father_uri;
}else{
HLOG_DEBUG("need read uri:%s is first uri",uri);
break;
}
}while(1);
fctrl->from_segno = fsegno;
fctrl->father_uri = g_strdup(furi);
fctrl->base_father_inode = fbase_inode;
out:
return ret;
}
int hdfs_connect(struct back_storage *storage,const char* uri){
//HLOG_DEBUG("hdfs -- enter func %s", __func__);
hdfsFS fs = hdfsConnect(storage->hostname,storage->port); // for local test
if(NULL==fs){
HLOG_ERROR("fs is null, hdfsConnect error!");
return -1;
}
//storage->uri = uri;
storage->fs_handler = (bs_fs_t)fs;
//HLOG_DEBUG("hdfs -- leave func %s", __func__);
return 0;
}
int hlfs_lstat(const char*uri,HLFS_STAT_T *stat){
//HLOG_DEBUG("enter func %s", __func__);
int ret = 0;
if(NULL == uri|| NULL == stat){
HLOG_ERROR("param error");
return -1;
}
struct back_storage *storage = init_storage_handler(uri);
if(storage == NULL){
ret = -1;
goto out;
}
uint32_t seg_size,block_size,last_segno,last_offset;
uint64_t max_fs_size;
ret = read_fs_meta(storage,&seg_size,&block_size,&max_fs_size);
if(ret != 0){
HLOG_ERROR("can not read fs meta data");
goto out;
}
ret =get_cur_latest_segment_info(storage,&last_segno,&last_offset);
if(ret !=0){
HLOG_ERROR("can not read fs meta data");
goto out;
}
stat->seg_size = seg_size;
stat->block_size = block_size;
stat->max_fs_size = max_fs_size;
g_strlcpy(stat->fsname,g_basename(uri),MAX_FILE_NAME_LEN);
stat->last_segno = last_segno;
stat->last_offset = last_offset;
out:
if(storage!=NULL){
deinit_storage_handler(storage);
}
//HLOG_DEBUG("leave func %s", __func__);
return ret;
}
int hlfs_take_snapshot(struct hlfs_ctrl *ctrl, const char *ssname)
{
//HLOG_DEBUG("enter func %s", __func__);
//HLOG_DEBUG("create ssname is %s", ssname);
if(ctrl == NULL || ssname ==NULL){
HLOG_ERROR("parameter error!");
return -1;
}
int ret = 0;
if ((strlen(ssname) + 1) > HLFS_FILE_NAME_MAX) {
HLOG_ERROR("error, snapshot name beyond max length!");
return -1;
}
g_mutex_lock(ctrl->hlfs_access_mutex);
if (ctrl->rw_inode_flag == 0) {
HLOG_ERROR("error, snapshot can not take when readonly");
g_mutex_unlock (ctrl->hlfs_access_mutex);
return -1;
}
g_mutex_unlock (ctrl->hlfs_access_mutex);
struct snapshot *_ss = NULL;
if (0 == (ret=load_snapshot_by_name (ctrl->storage, \
SNAPSHOT_FILE, &_ss, ssname))){
HLOG_ERROR("snapshot %s is exist, use another snapshot name", \
ssname);
return -1;
}else{
HLOG_DEBUG("snapshot %s is not exist , create it ", ssname);
}
struct snapshot ss;
memset(&ss, 0, sizeof(struct snapshot));
ss.timestamp = get_current_time();
g_strlcpy(ss.sname, ssname, strlen(ssname) + 1);
g_mutex_lock(ctrl->hlfs_access_mutex);
sprintf(ss.up_sname, "%s", ctrl->alive_ss_name);
ss.inode_addr = ctrl->imap_entry.inode_addr;
memset(ctrl->alive_ss_name, 0, MAX_FILE_NAME_LEN);
sprintf(ctrl->alive_ss_name, "%s", ss.sname);
g_mutex_unlock (ctrl->hlfs_access_mutex);
ret = dump_alive_snapshot(ctrl->storage, ALIVE_SNAPSHOT_FILE, &ss);
if (ret != 0) {
HLOG_ERROR("dump snapshot alive error!");
return -1;
}
ret = dump_snapshot(ctrl->storage, SNAPSHOT_FILE, &ss);
if (ret != 0) {
HLOG_ERROR("dump snapshot error!");
return -1;
}
HLOG_INFO("Take Snapshot Succ- snapshot_name:%s,last_segno:%d, \
last_offset:%d", ssname, ctrl->last_segno, ctrl->last_offset);
return ret;
}
int dump_snapshot(struct back_storage *storage,
const char *snapshot_file,
struct snapshot *snapshot) {
//HLOG_DEBUG("enter func %s", __func__);
if (snapshot_file == NULL || snapshot == NULL || storage == NULL) {
HLOG_ERROR("Parameter error!");
return -1;
}
int ret = 0;
char snapshot_text[1024];
memset(snapshot_text, 0, 1024);
int len = snapshot2text(snapshot, snapshot_text);
//HLOG_DEBUG("ss text is %s", snapshot_text);
ret = file_append_contents(storage, snapshot_file, snapshot_text, len);
return ret;
}
int hlfs_stat(struct hlfs_ctrl* ctrl,HLFS_STAT_T *stat){
//HLOG_DEBUG("enter func %s", __func__);
int ret = 0;
if(NULL == ctrl || NULL == stat){
HLOG_ERROR("param error");
return -1;
}
stat->seg_size = ctrl->sb.seg_size;
stat->block_size = ctrl->sb.block_size;
stat->max_fs_size = ctrl->sb.max_fs_size;
g_strlcpy(stat->fsname,ctrl->sb.fsname,MAX_FILE_NAME_LEN);
stat->last_segno = ctrl->last_segno;
stat->last_offset = ctrl->last_offset;
//HLOG_DEBUG("leave func %s", __func__);
return ret;
}
int hlfs_open_by_snapshot(struct hlfs_ctrl *ctrl,
const char* snapshot,
int flag) {
//HLOG_DEBUG("enter func %s", __func__);
if (NULL == ctrl || NULL == snapshot) {
HLOG_ERROR("Parameter Error!");
return -1;
}
if ((strlen(snapshot) + 1) > HLFS_FILE_NAME_MAX) {
HLOG_ERROR("snapshot name beyond max length");
return -1;
}
int ret = 0;
struct snapshot *ss = NULL;
if (0 == ctrl->storage->bs_file_is_exist(ctrl->storage, SNAPSHOT_FILE)) {
if (0 > (ret = load_snapshot_by_name(ctrl->storage, SNAPSHOT_FILE, &ss, snapshot))) {
HLOG_ERROR("load ss by name error");
g_free(ss);
ret = -1;
goto out;
}
} else {
HLOG_ERROR("We have no snapshot at the moment");
ret = -1;
goto out;
}
struct inode *inode = load_inode(ctrl->storage,ss->inode_addr);
if (inode == NULL) {
HLOG_ERROR("load_inode error!");
ret = -1;
goto out;
}
memcpy(&(ctrl->inode), inode, sizeof(struct inode));
g_free(inode);
ctrl->imap_entry.inode_no = HLFS_INODE_NO;
ctrl->imap_entry.inode_addr = ss->inode_addr;
if (0 == flag) {
ctrl->rw_inode_flag = 0;
} else if (1 == flag) {
ctrl->rw_inode_flag = 1;
} else {
HLOG_ERROR("the bad flag for hlfs open by inode");
ret = -1;
}
memset(ctrl->alive_ss_name, 0, MAX_FILE_NAME_LEN);
sprintf(ctrl->alive_ss_name, "%s", ss->sname);
g_free(ss);
ctrl->usage_ref += 1;
out:
//HLOG_DEBUG("leave func %s", __func__);
return ret;
}
int dump_snapshot_delmark(struct back_storage *storage,
const char *snapshot_file,
const char *ssname){
//HLOG_DEBUG("enter func %s", __func__);
if (snapshot_file == NULL || ssname == NULL || storage == NULL) {
HLOG_ERROR("Parameter Error!");
return -1;
}
int ret = 0;
char snapshot_delmark_text[128];
memset(snapshot_delmark_text, 0, 128);
int len = snapshot_delmark2text(ssname, snapshot_delmark_text);
//HLOG_DEBUG("cp text is %s", snapshot_delmark_text);
ret = file_append_contents(storage, snapshot_file, \
snapshot_delmark_text, len);
return ret;
}
static gchar *build_hdfs_path(const char *uri, const char *path){
HLOG_DEBUG("hdfs -- enter func %s", __func__);
char *head = NULL;
char *hostname = NULL;
char *dir = NULL;
char *fs_name = NULL;
int port;
int ret = parse_from_uri(uri, &head, &hostname, &dir, &fs_name, &port);
if (ret != 0) {
HLOG_ERROR("parse_from_uri error!");
return NULL;
}
gchar *full_path = g_build_filename(dir, fs_name, path, NULL);
HLOG_DEBUG("full path is %s", full_path);
g_free(head);
g_free(hostname);
//g_free(dir);
HLOG_DEBUG("hdfs -- leave func %s", __func__);
return full_path;
}
bs_file_info_t*
local_list_dir(struct back_storage * storage,const char * dir_path,uint32_t* num_entries){
//HLOG_DEBUG("local -- enter func %s", __func__);
char full_path[256];
build_local_path(full_path,storage->dir,storage->fs_name,dir_path);
GDir * dir = g_dir_open(full_path,0,NULL);
if(dir==NULL){
//HLOG_ERROR("g_dir_open error");
return NULL;
}
struct stat buf;
bs_file_info_t * infos = (bs_file_info_t*)g_malloc0(sizeof(bs_file_info_t)*8192);
if (NULL == infos) {
//HLOG_ERROR("build local path error!");
return NULL;
}
const gchar *filename;
bs_file_info_t *info = infos;
int idx=0;
while((filename = g_dir_read_name(dir)) != NULL) {
gchar *file_path = g_build_filename(full_path,filename,NULL);
int res=lstat(file_path,&buf);
if (0 != res) {
g_free(file_path);
g_free(infos);
//g_dir_close(dir);
HLOG_ERROR("lstat error");
return NULL;
}
strcpy((char *) info->name,filename);
info->is_dir = g_file_test(file_path, G_FILE_TEST_IS_DIR);
info->size = buf.st_size;
info->lmtime = buf.st_mtime;
info++;
idx++;
}
*num_entries = idx;
g_dir_close(dir);
//HLOG_DEBUG("local -- leave func %s", __func__);
return infos;
}
int hlfs_rmfs(struct back_storage *storage){
HLOG_DEBUG("enter func %s", __func__);
int ret = 0;
uint32_t num_entries;
bs_file_info_t *infos = storage->bs_file_list_dir(storage, NULL, &num_entries);
if (infos == NULL) {
HLOG_ERROR("can not get fs:%s seg entries", storage->uri);
return -1;
}
printf("how much file :%d\n", num_entries);
int i = 0;
bs_file_info_t *info = infos;
for (i = 0;i < num_entries;i++) {
printf("999 file:%s, size:%llu, time:%llu\n", info->name, \
info->size, info->lmtime);
storage->bs_file_delete(storage, info->name);
info++;
}
storage->bs_file_delete(storage, NULL);
printf("leave func %s", __func__);
free(infos);
return ret;
}
int hlfs_read(struct hlfs_ctrl *ctrl, char* read_buf, uint32_t read_len, uint64_t pos)
{
if((NULL == read_buf) || (NULL == ctrl) || (0 == read_len)){
HLOG_ERROR("Params Error");
return -1;
}
if(ctrl->sb.max_fs_size *1024 *1024< pos+read_len){
HLOG_ERROR("your config only allow write beyond :%llu",ctrl->sb.max_fs_size);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
return -1;
}
//g_mutex_lock (ctrl->hlfs_access_mutex);
HLOG_INFO("Hlfs Read Req pos:%llu,read_len:%d,last_segno:%d,last_offset:%d,cur_file_len:%llu",
pos,
read_len,
ctrl->last_segno,
ctrl->last_offset,
ctrl->inode.length);
guint32 BLOCKSIZE = ctrl->sb.block_size;
HLOG_DEBUG("read offset:%llu,read len:%d", pos,read_len);
int ret = 0;
int start_db = 0;
if(pos/BLOCKSIZE == (pos+read_len-1)/BLOCKSIZE){
HLOG_DEBUG("only need to read one block: %llu", pos / BLOCKSIZE);
char *block = (char*)alloca(BLOCKSIZE);
//g_mutex_lock (ctrl->hlfs_access_mutex);
ret=load_block_by_addr_fast(ctrl,pos,block);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
if(-1 == ret ){
HLOG_ERROR("fail to load block for addr %llu", pos);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
return -1;
}else if(1==ret){
//HLOG_DEBUG("fail to load block for not write yet");
memset(block,0,BLOCKSIZE);
}
memcpy(read_buf,block + pos%BLOCKSIZE,read_len);
//g_free(block);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
HLOG_DEBUG("read len %u", read_len);
return read_len;
}
HLOG_DEBUG("need to read muti block", __func__);
uint32_t offset=0;
if( pos % BLOCKSIZE != 0 ){
HLOG_DEBUG("need to read first block", __func__);
char *first_block = (char*)alloca(BLOCKSIZE);
//g_mutex_lock (ctrl->hlfs_access_mutex);
ret=load_block_by_addr_fast(ctrl,pos,first_block);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
if(-1 == ret){
HLOG_ERROR("fail to load block for addr %llu", pos);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
return -1;
}else if(1 == ret){
//HLOG_DEBUG("fail to load block for not write yet");
memset(first_block,0,BLOCKSIZE);
}
memcpy(read_buf,first_block + pos%BLOCKSIZE, BLOCKSIZE - pos%BLOCKSIZE);
offset += BLOCKSIZE - pos%BLOCKSIZE;
HLOG_DEBUG("fist offset:%u", offset);
//g_free(block);
start_db = (pos + BLOCKSIZE)/BLOCKSIZE;
}else{
start_db = pos/BLOCKSIZE;
}
int end_db = (pos+read_len)/BLOCKSIZE;
HLOG_DEBUG("start db: %d end db: %d", start_db, end_db);
int i;
//char *block = (char*)alloca(BLOCKSIZE);
for(i = start_db; i < end_db;i++){
//g_mutex_lock (ctrl->hlfs_access_mutex);
char *block = read_buf+offset;
ret=load_block_by_no_fast(ctrl,i,block);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
if(-1 == ret){
HLOG_ERROR("fail to load block for no %d", i);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
return -1;
}else if(1==ret){
//HLOG_DEBUG("fail to load block for not write yet");
memset(block,0,BLOCKSIZE);
}
//memcpy(read_buf+offset,block,BLOCKSIZE);
offset +=BLOCKSIZE;
HLOG_DEBUG("offset: %u", offset);
//g_free(block);
}
if((pos + read_len)% BLOCKSIZE != 0 ){
HLOG_DEBUG("need to read last block", __func__);
char *last_block = (char*)alloca(BLOCKSIZE);
//g_mutex_lock (ctrl->hlfs_access_mutex);
ret=load_block_by_addr_fast(ctrl,pos+read_len,last_block);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
if(-1 == ret){
HLOG_ERROR("fail to load block for addr %llu", pos + read_len);
//g_mutex_unlock (ctrl->hlfs_access_mutex);
return -1;
}else if (1==ret){
//HLOG_DEBUG("fail to load block for not write yet");
memset(last_block,0,BLOCKSIZE);
}
memcpy(read_buf + offset , last_block , (pos + read_len)%BLOCKSIZE );
offset +=(pos+read_len)%BLOCKSIZE;
//g_free(block);
}
//g_mutex_unlock (ctrl->hlfs_access_mutex);
//ctrl->last_access_timestamp = get_current_time();
ctrl->last_read_timestamp = get_current_time();
HLOG_DEBUG("leave func %s", __func__);
return offset;
}
static int __load_block_by_no(struct hlfs_ctrl *ctrl,uint32_t no,READ_BLOCK_FUN RB_FUN,char *block){
//HLOG_DEBUG("enter func %s,no:%d", __func__,no);
int ret =0;
if(ctrl->cctrl!=NULL){
//HLOG_DEBUG("read from cache first");
//*block = g_malloc0(ctrl->cctrl->block_size);
ret = cache_query_block(ctrl->cctrl,no,block);
if(ret == 0 ){
HLOG_DEBUG("succ to get block from cache!");
return 0;
}
//g_free(*block);
HLOG_DEBUG("not find in cache!");
}
uint64_t storage_address ;
guint32 BLOCKSIZE = ctrl->sb.block_size;
uint32_t db_no = no;
uint32_t IB_ENTRY_NUM = BLOCKSIZE/sizeof(uint64_t);
if( (db_no+1)*BLOCKSIZE >ctrl->inode.length){
HLOG_DEBUG("beyond current inode's length:%llu",ctrl->inode.length);
return 1;
}
if(is_db_in_level1_index_range(db_no)){
int _idx = db_no % 12;
storage_address = ctrl->inode.blocks[_idx];
}else if (is_db_in_level2_index_range(db_no)){
//HLOG_DEBUG("inode.iblock :%llu",ctrl->inode.iblock);
if(ctrl->inode.iblock == 0){
//HLOG_DEBUG("inode.iblock zero");
return 1;
}
char *_ib= (char *)alloca(BLOCKSIZE);
memset(_ib,0,sizeof(BLOCKSIZE));
if(0>read_layer1_iblock(ctrl,db_no,(char*)_ib)){
if(0 != RB_FUN(ctrl,ctrl->inode.iblock,(char*)_ib)){
HLOG_ERROR("read_block error for iblock_addr:%llu",ctrl->inode.iblock);
return -1;
}
write_layer1_iblock(ctrl,db_no,_ib);
}
int _idx = (db_no-12)%IB_ENTRY_NUM;
storage_address = *(_ib+_idx);
//g_free(_ib);
}else if (is_db_in_level3_index_range(db_no)){
//HLOG_DEBUG("inode.double_iblock :%llu",ctrl->inode.doubly_iblock);
if(ctrl->inode.doubly_iblock ==0){
HLOG_DEBUG("inode.doubly_iblock zero");
return 1;
}
char *_ib= (char *)alloca(BLOCKSIZE);
memset(_ib,0,sizeof(BLOCKSIZE));
if(0>read_layer1_iblock(ctrl,db_no,(char*)_ib)){
if(0 != RB_FUN(ctrl,ctrl->inode.doubly_iblock,(char*)_ib)){
HLOG_ERROR("read_block error for doubly_iblock_addr:%llu",ctrl->inode.doubly_iblock);
return -1;
}
write_layer1_iblock(ctrl,db_no,_ib);
}
int _idx = ( db_no - 12 - IB_ENTRY_NUM)/IB_ENTRY_NUM;
//HLOG_DEBUG("ib1 address:%llu",*(_ib+_idx));
if(*(_ib+_idx) == 0 ){
return 1;
}
char *_ib2=(char *)alloca(BLOCKSIZE);
memset(_ib2,0,sizeof(BLOCKSIZE));
if( 0> read_layer2_iblock(ctrl,db_no,_ib2)){
if( 0!= RB_FUN(ctrl,*(_ib+_idx),(char*)_ib2)){
//HLOG_ERROR("read_block error");
return -1;
}
write_layer2_iblock(ctrl,db_no,_ib2);
}
int _idx2 = (db_no - 12 - IB_ENTRY_NUM)%IB_ENTRY_NUM;
storage_address = *(_ib2 + _idx2);
//g_free(_ib);
//g_free(_ib2);
}else if (is_db_in_level4_index_range(db_no)){
if(ctrl->inode.triply_iblock == 0){
//HLOG_DEBUG("inode.triply_iblock zero");
return 1;
}
char *_ib =(char *)alloca(BLOCKSIZE);
memset(_ib,0,sizeof(BLOCKSIZE));
if(0>read_layer1_iblock(ctrl,db_no,(char*)_ib)){
if(0 != RB_FUN(ctrl,ctrl->inode.triply_iblock,(char*)_ib)){
//HLOG_ERROR("read_block error for triply_iblock_addr:%llu",ctrl->inode.triply_iblock);
return -1;
}
write_layer1_iblock(ctrl,db_no,_ib);
}
int _idx = (db_no -12 - IB_ENTRY_NUM - IB_ENTRY_NUM*IB_ENTRY_NUM) / (IB_ENTRY_NUM*IB_ENTRY_NUM);
if(*(_ib + _idx) == 0){
return 1;
}
char *_ib2 = (char *)alloca(BLOCKSIZE);
memset(_ib2,0,sizeof(BLOCKSIZE));
if(0>read_layer2_iblock(ctrl,db_no,_ib2)){
if(0 != RB_FUN(ctrl,*(_ib + _idx),(char*)_ib2)){
//HLOG_ERROR("read_block error");
return -1;
}
write_layer2_iblock(ctrl,db_no,_ib2);
}
int _idx2 = (db_no-12 - IB_ENTRY_NUM - IB_ENTRY_NUM*IB_ENTRY_NUM)/IB_ENTRY_NUM % IB_ENTRY_NUM;
if(*(_ib2 + _idx2) == 0){
return 1;
}
char *_ib3 = (char *)alloca(BLOCKSIZE);
memset(_ib3,0,sizeof(BLOCKSIZE));
if(0>read_layer3_iblock(ctrl,db_no,(char*)_ib3)){
if(0 != RB_FUN(ctrl,*(_ib2 + _idx2),(char*)_ib3)){
//HLOG_ERROR("read_block error");
return -1;
}
write_layer3_iblock(ctrl,db_no,_ib3);
}
int _idx3 = (db_no-12 - IB_ENTRY_NUM - IB_ENTRY_NUM*IB_ENTRY_NUM) % IB_ENTRY_NUM;
storage_address = *(_ib3 + _idx3);
//g_free(_ib);
//g_free(_ib2);
//g_free(_ib3);
}
HLOG_DEBUG("storage address:%llu",storage_address);
if(storage_address == 0){
return 1;
}
ret = RB_FUN(ctrl,storage_address,block);
//HLOG_DEBUG("leave func %s", __func__);
return ret;
}
int prev_open_rsegfile(struct hlfs_ctrl *ctrl, uint32_t segno){
//HLOG_DEBUG("enter func %s", __func__);
struct back_storage *storage = NULL;
if (NULL == ctrl->last_rsegfile_handler) {
char segfile_name[SEGMENT_FILE_NAME_MAX];
memset((void *)segfile_name, 0, SEGMENT_FILE_NAME_MAX);
build_segfile_name(segno, segfile_name);
bs_file_t file;
if (segno >= ctrl->start_segno) {
storage = ctrl->storage;
}else{
HLOG_DEBUG("get parent storage for segno:%d",segno);
if (NULL == (storage = get_parent_storage(ctrl->family, segno))){
g_assert(0);
return -1;
}
}
file = storage->bs_file_open(storage, segfile_name, BS_READONLY);
if (file == NULL) {
HLOG_ERROR("can not open segment file %s", segfile_name);
g_assert(0);
return -1;
}
ctrl->last_rsegfile_handler = file;
ctrl->last_rsegfile_offset = ctrl->last_offset;
ctrl->last_read_segno =segno;
}else if (ctrl->last_read_segno != segno || (ctrl->last_read_segno == \
segno && ctrl->last_rsegfile_offset != ctrl->last_offset)){
HLOG_DEBUG("cur segno:%d is, last segno no:%d, \
last rsegfile offset:%d, last offset:%d,start_segno:%d - \
need close old and open new segfile", segno, \
ctrl->last_read_segno,ctrl->last_rsegfile_offset, \
ctrl->last_offset, ctrl->start_segno);
/* close last seg file handler... */
if (ctrl->last_read_segno >= ctrl->start_segno) {
storage = ctrl->storage;
}else{
HLOG_DEBUG("get parent storage for segno:%d",segno);
if (NULL == (storage = get_parent_storage(ctrl->family, \
ctrl->last_read_segno))){
g_assert(0);
return -1;
}
}
if (0 != storage->bs_file_close(storage, ctrl->last_rsegfile_handler)) {
g_assert(0);
return -1;
}
/* open cur seg file handler... */
const char segfile_name[SEGMENT_FILE_NAME_MAX];
build_segfile_name(segno, segfile_name);
if (segno >= ctrl->start_segno) {
storage = ctrl->storage;
}else{
HLOG_DEBUG("get parent storage for segno:%d", segno);
if (NULL == (storage = get_parent_storage(ctrl->family, segno))) {
g_assert(0);
return -1;
}
}
bs_file_t file = storage->bs_file_open(storage, segfile_name,\
BS_READONLY);
if (file == NULL) {
HLOG_ERROR("can not open segment file %s", segfile_name);
g_assert(0);
return -1;
}
ctrl->last_rsegfile_handler = file;
ctrl->last_rsegfile_offset = ctrl->last_offset;
ctrl->last_read_segno = segno;
}else{
int seg_usage_calc(struct back_storage* storage,uint32_t block_size,uint32_t segno,struct inode *refer_inode,SEG_USAGE_T *seg_usage)
{
//HLOG_DEBUG("enter func %s",__func__);
if(storage == NULL || refer_inode == NULL || seg_usage == NULL){
HLOG_ERROR("input params failed");
return -1;
}
#if 0
if(seg_usage->bitmap!=0 && segno != seg_usage->segno){
HLOG_ERROR("segno not match");
return -1;
}
#endif
int ret = 0;
int log_idx=0;
int idx;
uint32_t offset = 0;
struct log_header *lh;
//gchar **v = g_strsplit(segfile,".",2);
// uint64_t db_mine_storage_addr_segno = atol(v[0]);
//g_strfreev(v);
GArray *tmp_bit_array;
seg_usage->segno = segno;
seg_usage->timestamp = get_current_time();
HLOG_DEBUG("seg usage's segno:%llu,timestamp:%llu",seg_usage->segno,seg_usage->timestamp);
char segfile[SEGMENT_FILE_NAME_MAX];
build_segfile_name(segno,segfile);
char *content = NULL;
uint32_t size;
if(0!= (ret = file_get_contents(storage,segfile,&content,&size))){
HLOG_ERROR("read segfile:%s failed",segfile);
return -1;
}
tmp_bit_array = g_array_new(FALSE,FALSE,sizeof(gint));
while(offset < size){
#if 0
ret=storage->bs_file_pread(storage,file, (char*)&lh, LOG_HEADER_LENGTH, offset) ;//TODO read 64M once
g_message("read content len:%d\n",ret);
if(ret<0){
ret = -1;
goto out;
}else if (ret == 0){
g_message("read over?\n");
ret = 0;
break;
}else if (ret == LOG_HEADER_LENGTH){
g_message("read log header over\n");
;
}else{
g_message("read log header failed\n");
ret = -1;
goto out;
}
#endif
lh = (struct log_header*)(content + offset);
if(seg_usage->log_num !=0){
//HLOG_DEBUG("this segfile:%s has calc",segfile);
idx = log_idx/sizeof(gint);
if(!(seg_usage->bitmap[idx] & (1<<(log_idx%sizeof(gint))))){
log_idx++;
int x=0;
g_array_append_val(tmp_bit_array,x);
offset += lh->log_size;
continue;
}
}
uint32_t orgine_alive_block_num = seg_usage->alive_block_num;
//HLOG_DEBUG("start db no:%llu,db num:%d",lh->start_db_no,lh->db_num);
int i;
#if 1 /* check refer inode whether still refer to given db in seg */
for(i=0;i<lh->db_num;i++){
//HLOG_DEBUG("for db:%llu",lh->start_db_no+i);
uint64_t db_mine_storage_addr = 0;
uint32_t db_mine_storage_addr_offset = offset + LOG_HEADER_LENGTH+i*block_size;
set_offset(&db_mine_storage_addr,db_mine_storage_addr_offset);
set_segno (&db_mine_storage_addr,segno);
uint64_t db_cur_storage_addr = get_db_storage_addr_in_inode(storage,refer_inode,
lh->start_db_no+i,block_size);
HLOG_DEBUG("db:%llu's mine storage addr:%llu,cur storage addr:%llu",
lh->start_db_no+i,db_mine_storage_addr,db_cur_storage_addr);
if(db_mine_storage_addr != db_cur_storage_addr){
HLOG_DEBUG("this is overwrite data block");
}else{
seg_usage->alive_block_num++;
HLOG_DEBUG("this is used data block :%llu",seg_usage->alive_block_num);
}
seg_usage->block_num++;
}
#endif
//uint32_t alive_blocks = log_usage_calc(storage,latest_inode,&lh,db_mine_storage_addr_segno,offset,block_size);
if(orgine_alive_block_num == seg_usage->alive_block_num){
HLOG_DEBUG("log:%d has not any datablock",log_idx);
//uint32_t bitmap_idx = log_idx / ALIVE_LOG_BITMAP ;
//seg_usage->alive_log_bitmap[bitmap_idx] &= ~(1 << (log_idx % sizeof(uint64_t)));
int x=0;
g_array_append_val(tmp_bit_array,x);
}else{
HLOG_DEBUG("log:%d has any datablock",log_idx);
int x=1;
g_array_append_val(tmp_bit_array,x);
}
offset += lh->log_size;
log_idx++;
}
int i;
seg_usage->log_num = tmp_bit_array->len;
g_free(seg_usage->bitmap);
seg_usage->bitmap = (char*)g_malloc0((seg_usage->log_num-1)/sizeof(gint)+1);
//HLOG_DEBUG("size of bitmap:%d",tmp_bit_array->len);
for(i=0;i<tmp_bit_array->len;i++){
gint value = g_array_index(tmp_bit_array,gint,i);
idx = i/sizeof(gint);
if(value==1){
//g_message("bitmap idx bit:%d = 1\n",i);
seg_usage->bitmap[idx] |= 1<<i%sizeof(gint);
//g_message("bitmap idx %x\n",seg_usage->bitmap[idx]);
}
}
g_array_free(tmp_bit_array,TRUE);
//HLOG_DEBUG("leave func %s",__func__);
return 0;
}
/*
* hlfs_open: open a file.
* @param ctrl: the global control.
* @param flag: the flag for open operation, flag == 0
* readonly and flag == 1 writable.
* @return: if successful return 0, else return -1.
*/
int hlfs_open(struct hlfs_ctrl *ctrl, int flag)
{
HLOG_DEBUG("enter func %s", __func__);
if (ctrl == NULL ||(flag != 0 && flag != 1)) { /* check the parameters */
HLOG_ERROR("error params :flag %d", flag);
return -1;
}
if (1 == flag) {
ctrl->rw_inode_flag = 1;
} else if (0 == flag) {
ctrl->rw_inode_flag = 0;
} else {
HLOG_ERROR("the bad flag for hlfs open by inode");
return -1;
}
if (ctrl->usage_ref > 0) {
HLOG_DEBUG("This fs has opened by other,can not use it");
return -1;
}
int ret = 0;
HLOG_DEBUG("inode no %llu , inode address %llu", ctrl->imap_entry.inode_no, ctrl->imap_entry.inode_addr);
if (ctrl->imap_entry.inode_no == 0 && ctrl->imap_entry.inode_addr == 0) { /* no inode condition */
HLOG_DEBUG("empty filesystem %s", ctrl->sb.fsname);
if (flag == 0) {
HLOG_ERROR("must create it with writeable flag");
return -1;
}
HLOG_DEBUG("create new fs inode !");
ctrl->inode.length = 0;
ctrl->inode.mtime = get_current_time();
//ctrl->inode.ctime = get_current_time();
//ctrl->inode.atime = get_current_time();
} else { /* exist inode */
HLOG_DEBUG("this is not empty filesystem:%s", ctrl->sb.fsname);
struct back_storage *storage = NULL;
uint32_t segno = get_segno(ctrl->imap_entry.inode_addr);
if (segno >= ctrl->start_segno) {
storage = ctrl->storage;
}else{
HLOG_DEBUG("get parent storage for segno:%d", segno);
if (NULL == (storage = \
get_parent_storage(ctrl->family, segno))){
g_assert(0);
return -1;
}
}
struct inode *my_inode = NULL;
my_inode = load_inode(storage, ctrl->imap_entry.inode_addr);
if (my_inode == NULL) {
HLOG_ERROR("load_inode error!");
return -1;
}
HLOG_DEBUG("inode'length:%llu, ctrl->inode length:%llu, \
sizeof inode:%d",my_inode->length,ctrl->inode.length, \
sizeof(struct inode));
memcpy(&(ctrl->inode), my_inode, sizeof(struct inode));
g_free(my_inode);
}
HLOG_DEBUG("ctrl->rw_inode_flag:%d", ctrl->rw_inode_flag);
struct snapshot *ss;
if (0 == ctrl->storage->bs_file_is_exist(ctrl->storage, \
SNAPSHOT_FILE)) {
ret = find_latest_alive_snapshot(ctrl->storage, \
ALIVE_SNAPSHOT_FILE, SNAPSHOT_FILE, &ss);
if (ret != 0) {
HLOG_DEBUG("can not read alive snapshot, \
there must be some error");
return -1;
}
int flush_work(gpointer data){
int ret = 0;
CACHE_CTRL *cctrl = (CACHE_CTRL*)data;
GTimeVal expired;
char *tmp_buf = (char *)g_malloc0(cctrl->block_size \
*cctrl->flush_once_size);
g_assert(tmp_buf);
while (!cctrl->flush_worker_should_exit) {
HLOG_DEBUG("-- flush worker doing --");
g_get_current_time(&expired);
g_time_val_add(&expired, cctrl->flush_interval * 1000 * 1000);
g_mutex_lock(cctrl->cache_mutex);
gboolean res = g_cond_timed_wait(cctrl->flush_waken_cond, \
cctrl->cache_mutex, &expired);
g_mutex_unlock(cctrl->cache_mutex);
HLOG_DEBUG(" time wait res for cond is :%d !",res);
if (cctrl->flush_worker_should_exit) {
HLOG_INFO("-- flush worker should exit --");
break;
}
do {
GSList *continue_blocks = NULL;
ret = get_continues_blocks(cctrl, &continue_blocks);
g_assert(ret==0);
uint32_t blocks_count = g_slist_length(continue_blocks);
uint32_t buff_len = blocks_count *cctrl->block_size;
HLOG_DEBUG("--blocks_count:%d, buff_len:%d--", \
blocks_count, buff_len);
if (res == TRUE && buff_len == 0) {
HLOG_ERROR("Never reach here");
g_assert(0);
}
if (buff_len == 0) {
HLOG_DEBUG("do not need flush now");
break;
}
if (NULL == cctrl->write_callback_func) {
HLOG_WARN("--not given flush callback func--");
break;
}
//char* tmp_buf = g_malloc0(buff_len);
//g_assert(tmp_buf!=NULL);
uint32_t start_no;
uint32_t end_no;
int i = 0;
for (i = 0; i < blocks_count; i++) {
block_t *block = g_slist_nth_data(continue_blocks, i);
if (i == 0) {
start_no = block->block_no;
}
if (i == blocks_count-1) {
end_no = block->block_no;
}
memcpy(tmp_buf + i * cctrl->block_size, \
block->block, cctrl->block_size);
}
//HLOG_DEBUG("--tmp_buf:%p",tmp_buf);
ret = cctrl->write_callback_func(cctrl->write_callback_user_param, \
tmp_buf, start_no,end_no);
g_assert(ret >= 0);
//g_free(tmp_buf);
if (ret >= 0 ) {
HLOG_DEBUG("--signal write thread--");
g_mutex_lock(cctrl->cache_mutex);
__free_from_cache(cctrl, continue_blocks);
//g_cond_broadcast(cctrl->writer_waken_cond);
g_cond_signal(cctrl->writer_waken_cond);
g_mutex_unlock(cctrl->cache_mutex);
g_slist_free(continue_blocks);
//HLOG_DEBUG("--return blocks to cache over--");
}
} while (get_cache_free_size(cctrl) < cctrl->flush_trigger_level \
*cctrl->cache_size / 100 || (res == 0 && get_cache_free_size(cctrl) != 0));
}
g_free(tmp_buf);
HLOG_INFO("--flush worker exit--");
return 0;
}
