int get_file_list(const string& log_file, SYNC_FILE_MAP& sync_file_map)
{
UNUSED(sync_file_map);
int ret = TFS_ERROR;
FILE* fp = fopen (log_file.c_str(), "r");
if (fp == NULL)
{
TBSYS_LOG(ERROR, "open file(%s) fail,errors(%s)", log_file.c_str(), strerror(errno));
}
else
{
char info[MAX_READ_LEN];
while(!feof(fp))
{
memset(info, 0, MAX_READ_LEN);
fgets (info, MAX_READ_LEN, fp);
if (strlen(info) > 0)
{
char* block = str_match(info, "blockid: ");
if (block == NULL)
{
continue;
}
uint32_t block_id = static_cast<uint32_t>(atoi(block));
char* file = str_match(block + strlen(block) + 1, "fileid: ");
if (file == NULL)
{
continue;
}
uint64_t file_id = strtoull(file, NULL, 10);
FSName fs;
fs.set_block_id(block_id);
fs.set_file_id(file_id);
string file_name = string(fs.get_name());
SYNC_FILE_MAP_ITER iter = sync_file_map.find(block_id);
if (iter != sync_file_map.end())
{
iter->second.insert(file_name);
}
else
{
FILE_SET name_set;
name_set.insert(file_name);
sync_file_map.insert(make_pair(block_id, name_set));
}
TBSYS_LOG(INFO, "block_id: %u, file_id: %"PRI64_PREFIX"u, name: %s\n", block_id, file_id, fs.get_name());
}
}
fclose (fp);
ret = TFS_SUCCESS;
}
return ret;
}
int64_t TfsMetaManager::write_data(const char* ns_addr, const void* buffer, const int64_t offset, const int64_t length,
common::FragMeta& frag_meta)
{
//TODO get tfs_file from file_pool
int ret = TFS_SUCCESS;
int64_t cur_pos = 0;
int64_t cur_length, left_length = length;
char tfsname[MAX_FILE_NAME_LEN];
int fd = TfsClientImplV2::Instance()->open(static_cast<const char*>(NULL), static_cast<const char*>(NULL), ns_addr, T_WRITE);
if (fd < 0)
{
TBSYS_LOG(ERROR, "open write file error, fd: %d", fd);
}
else
{
int64_t write_length = 0;
do
{
cur_length = min(left_length, MAX_WRITE_DATA_IO);
write_length = TfsClientImplV2::Instance()->write(fd, reinterpret_cast<const char*>(buffer) + cur_pos, cur_length);
if (write_length < 0)
{
TBSYS_LOG(ERROR, "tfs write data error, ret: %"PRI64_PREFIX"d", write_length);
ret = TFS_ERROR;
break;
}
cur_pos += write_length;
left_length -= write_length;
}
while(left_length > 0);
TfsClientImplV2::Instance()->close(fd, tfsname, MAX_FILE_NAME_LEN);
TBSYS_LOG(DEBUG, "tfs write success, tfsname: %s", tfsname);
FSName fsname;
fsname.set_name(tfsname);
frag_meta.block_id_ = fsname.get_block_id();
frag_meta.file_id_ = fsname.get_file_id();
frag_meta.offset_ = offset;
frag_meta.size_ = length - left_length;
}
return (length - left_length);
}
int64_t TfsMetaManager::read_data(const char* ns_addr, const uint32_t block_id, const uint64_t file_id,
void* buffer, const int32_t offset, const int64_t length)
{
FSName fsname;
fsname.set_block_id(block_id);
fsname.set_file_id(file_id);
int64_t ret_length = -1;
int fd = TfsClientImplV2::Instance()->open(fsname.get_name(), NULL, ns_addr, T_READ);
if (fd < 0)
{
TBSYS_LOG(ERROR, "open read file error, file_name: %s, fd: %d", fsname.get_name(), fd);
}
else
{
ret_length = TfsClientImplV2::Instance()->pread(fd, buffer, length, offset);
TfsClientImplV2::Instance()->close(fd);
}
return ret_length;
}
int get_crc_from_filename(const string& old_tfs_client, const string& new_tfs_client,
const char* tfs_file_name, const string& modify_time)
{
int ret = TFS_SUCCESS;
uint32_t old_real_crc = 0;
uint32_t new_real_crc = 0;
TfsFileStat old_info, new_info;
memset(&old_info, 0, sizeof(TfsFileStat));
memset(&new_info, 0, sizeof(TfsFileStat));
bool skip_flag = false;
cmp_stat_.total_count_++;
// failed to get meta crc in old cluster
int src_ret = get_meta_crc(old_tfs_client, tfs_file_name, &old_info);
if (TFS_SUCCESS != src_ret && META_FLAG_ABNORMAL != src_ret) //no need to stat old file
{
TBSYS_LOG(ERROR, "%s failed in old cluster, ret: %d", tfs_file_name, src_ret);
ret = TFS_ERROR;
}
else
{
int dest_ret = get_meta_crc(new_tfs_client, tfs_file_name, &new_info);
//check src modify time
if (old_info.modify_time_ >= tbsys::CTimeUtil::strToTime(const_cast<char*>(modify_time.c_str())))
{
//skip
skip_flag = true;
cmp_stat_.skip_count_++;
}
else
{
if (META_FLAG_ABNORMAL == src_ret) // flag modify in old cluster, check new flag
{
if (TFS_ERROR == dest_ret) //dest file is not exist
{
if (old_info.flag_ & 1) //src is delete
{
}
else
{
ret = TFS_ERROR;
TBSYS_LOG(ERROR, "%s not exist in new cluster, src flag: %d",
tfs_file_name, old_info.flag_);
}
}
else if (META_FLAG_ABNORMAL == dest_ret) //dest file flag is abnormal
{
if (old_info.flag_ & 1 && !(new_info.flag_ & 1)) //fail
{
ret = TFS_ERROR;
}
else if (!(old_info.flag_ & 1) && (old_info.flag_ & 4) && !(new_info.flag_ & 4))
{
ret = TFS_ERROR;
}
if (TFS_ERROR == ret)
{
TBSYS_LOG(ERROR, "%s flag is conflict, src flag: %d, dest flag: %d",
tfs_file_name, old_info.flag_, new_info.flag_);
}
}
else //dest file is normal
{
ret = TFS_ERROR;
TBSYS_LOG(ERROR, "%s flag is conflict, src flag: %d, dest flag: %d",
tfs_file_name, old_info.flag_, new_info.flag_);
}
}
else // success
{
if (TFS_SUCCESS != dest_ret)
{
ret = TFS_ERROR;
TBSYS_LOG(ERROR, "%s fail, src flag: %d, dest flag: %d, dest ret: %d",
tfs_file_name, old_info.flag_, new_info.flag_, dest_ret);
}
// compare real crc
}
}
}
if (TFS_ERROR == ret)
{
if (TFS_SUCCESS != src_ret && META_FLAG_ABNORMAL != src_ret)
{
fprintf(g_error_file, "%s\n", tfs_file_name);
cmp_stat_.error_count_++;
}
else
{
fprintf(g_fail_file, "%s\n", tfs_file_name);
cmp_stat_.fail_count_++;
}
}
else if (META_FLAG_ABNORMAL == src_ret)
{
TBSYS_LOG(ERROR, "meta flag abnormal");
fprintf(g_error_file, "%s\n", tfs_file_name);
cmp_stat_.error_count_++;
}
else
{
if (!skip_flag)
{
if (get_real_crc(new_tfs_client, tfs_file_name, &new_real_crc) != TFS_SUCCESS)
{
TBSYS_LOG(ERROR, "get dest file fail: %s",tfs_file_name);
fprintf(g_fail_file, "%s\n", tfs_file_name);
cmp_stat_.error_count_++;
}
else
{
if ((old_info.crc_ == new_info.crc_) && (old_info.crc_ == new_real_crc))
{
TBSYS_LOG(DEBUG, "%s success",tfs_file_name);
fprintf(g_succ_file, "%s\n", tfs_file_name);
cmp_stat_.succ_count_++;
}
else
{
if (get_real_crc(old_tfs_client, tfs_file_name, &old_real_crc) != TFS_SUCCESS)
{
TBSYS_LOG(ERROR, "get src file fail: %s",tfs_file_name);
fprintf(g_error_file, "%s\n", tfs_file_name);
cmp_stat_.error_count_++;
}
else
{
if (old_info.crc_ != old_real_crc)
{
TBSYS_LOG(ERROR, "%s failed as : old cluster crc not same",tfs_file_name);
fprintf(g_error_file, "%s\n", tfs_file_name);
cmp_stat_.error_count_++;
ret = TFS_ERROR;
}
else
{
int tfs_fd_old = 0;
tfs_fd_old = g_tfs_client->open(tfs_file_name, NULL, old_tfs_client.c_str(), T_READ);
TfsFileStat finfo;
g_tfs_client->fstat(tfs_fd_old, &finfo);
FSName fsname;
fsname.set_name(tfs_file_name);
if(finfo.modify_time_ <= tbsys::CTimeUtil::strToTime(const_cast<char*>(modify_time.c_str())))
{
TBSYS_LOG(ERROR, "%s failed with Other reason, at block : %u : %s: %u %u %u %u",tfs_file_name,
fsname.get_block_id(),
Func::time_to_str(finfo.modify_time_).c_str(),
old_info.crc_, old_real_crc,
new_info.crc_, new_real_crc);
fprintf(g_fail_file, "%s\n", tfs_file_name);
cmp_stat_.fail_count_++;
}
else
{
TBSYS_LOG(ERROR, "%s failed as modified at %s",tfs_file_name,Func::time_to_str(finfo.modify_time_).c_str());
fprintf(g_unsync_file, "%s\n", tfs_file_name);
cmp_stat_.unsync_count_++;
}
g_tfs_client->close(tfs_fd_old);
}
}
}
}
}
}
return ret;
}
