void recover_block_from_slave_cluster(const char* ns_addr, const char* ns_slave_addr,
const VUINT64& tmp_fail_block_list, VUINT64& success_block_list,
VUINT64& fail_block_list, multimap<uint64_t, uint64_t>& fail_block_file_list)
{
int ret = TFS_SUCCESS;
VUINT64::const_iterator vit = tmp_fail_block_list.begin();
multimap<uint64_t, uint64_t> tmp_fail_block_file_list;
vector<FileInfoV2> finfos;
for (; vit != tmp_fail_block_list.end(); vit++)
{
uint64_t block_id = (*vit);
finfos.clear();
ret = ToolUtil::read_file_infos_v2(Func::get_host_ip(ns_slave_addr), block_id, finfos);
if (ret == TFS_SUCCESS)
{
bool all_success = true;
int32_t copy_file_succ_count = 0;
tmp_fail_block_file_list.clear();
vector<FileInfoV2>::const_iterator v_file_iter = finfos.begin();
for (; v_file_iter != finfos.end(); v_file_iter++)
{
if ((v_file_iter->status_ & FILE_STATUS_DELETE) != 0)
continue;
uint64_t file_id = v_file_iter->id_;
ret = copy_file_from_slave_cluster(ns_slave_addr, ns_addr, block_id, (*v_file_iter));
if (TFS_SUCCESS == ret)
{
++copy_file_succ_count;
TBSYS_LOG(DEBUG, "recover block_id: %" PRI64_PREFIX "u, file_id: %" PRI64_PREFIX "u successful from slave cluster!", block_id, file_id);
}
else
{
TBSYS_LOG(WARN, "recover block_id: %" PRI64_PREFIX "u, file_id: %" PRI64_PREFIX "u failed from slave cluster, ret: %d", block_id, file_id, ret);
all_success = false;
tmp_fail_block_file_list.insert(pair<uint64_t, uint64_t>(block_id, file_id));
}
}
if (all_success)
{
success_block_list.push_back(block_id);
}
else if (0 == copy_file_succ_count)
{
fail_block_list.push_back(block_id);
}
else
{
fail_block_file_list.insert(tmp_fail_block_file_list.begin(), tmp_fail_block_file_list.end());
}
}
else
{
fail_block_list.push_back(block_id);
}
}
}
int32_t elect_ds_normal(const DS_WEIGHT& weights, const int32_t elect_count, int64_t& elect_seq,
VUINT64& elect_ds_list)
{
if (elect_count == 0)
return 0;
int32_t need_elect_count = elect_count;
DS_WEIGHT::const_iterator iter = weights.begin();
while (iter != weights.end() && need_elect_count > 0)
{
elect_ds_list.push_back(iter->second->get_ds()->id_);
if (elect_seq > 0)
iter->second->elect(++elect_seq);
--need_elect_count;
++iter;
}
return elect_count - need_elect_count;
}
int32_t elect_ds_exclude_group(const DS_WEIGHT& weights, const int32_t elect_count, int64_t& elect_seq,
VUINT64& elect_ds_list)
{
if (elect_count == 0)
{
TBSYS_LOG(DEBUG, "current elect count(%d) <= 0, must be return", elect_count);
return 0;
}
std::set < uint32_t > existlan;
for (uint32_t i = 0; i < elect_ds_list.size(); ++i)
{
uint32_t lan = Func::get_lan(elect_ds_list[i], SYSPARAM_NAMESERVER.group_mask_);
existlan.insert(lan);
}
//dump_weigths(weights);
DS_WEIGHT::const_iterator iter = weights.begin();
int32_t need_elect_count = elect_count;
TBSYS_LOG(DEBUG, "weights.size(%u), need_elect_count(%d)", weights.size(), need_elect_count);
DataServerStatInfo* ds_stat_info = NULL;
while (iter != weights.end() && need_elect_count > 0)
{
ds_stat_info = iter->second->get_ds();
uint32_t dlan = Func::get_lan(ds_stat_info->id_, SYSPARAM_NAMESERVER.group_mask_);
if (existlan.find(dlan) == existlan.end())
{
existlan.insert(dlan);
elect_ds_list.push_back(ds_stat_info->id_);
if (elect_seq > 0)
iter->second->elect(++elect_seq);
--need_elect_count;
}
++iter;
}
TBSYS_LOG(DEBUG, "current elect_count(%d)", elect_count - need_elect_count);
return elect_count - need_elect_count;
}
// if parameter: ds_id != INVALID_SERVER_ID, check from current cluster, need exclude repliates in this ds;
// if parameter: ds_id == INVALID_SERVER_ID, check from slave cluster
int get_and_check_all_blocks_copy(const uint64_t ns_id, const VUINT64& blocks, VUINT64& no_copy_blocks, const uint64_t ds_id = INVALID_SERVER_ID)
{
no_copy_blocks.clear();
vector<BlockMeta> blocks_meta;
int ret = ToolUtil::get_all_blocks_meta(ns_id, blocks, blocks_meta, true);
if (TFS_SUCCESS == ret)
{
VUINT64 lost_in_family;
for (uint32_t block_index = 0; block_index < blocks_meta.size(); ++block_index)
{
// blk_meta.result_ just be used in trunk(stable2.7)
BlockMeta blk_meta = blocks_meta.at(block_index);
uint64_t block_id = blk_meta.block_id_;
FamilyInfoExt family_info = blk_meta.family_info_;
if (INVALID_FAMILY_ID == family_info.family_id_)
{
// if family can't reinstall, family_id here is also invalid
if (INVALID_BLOCK_ID == block_id || IS_VERFIFY_BLOCK(block_id))
continue;
// else family data block regard as normal block
if (0 == blk_meta.size_)
{
// maybe not exist the block
no_copy_blocks.push_back(block_id);
}
else if (INVALID_SERVER_ID != ds_id && 1 == blk_meta.size_ && ds_id == blk_meta.ds_[0])
{
// only exist one copy and be positioned in this ds
no_copy_blocks.push_back(block_id);
}
}
else // block is marshalling
{
int data_num = GET_DATA_MEMBER_NUM(family_info.family_aid_info_);
int member_num = data_num + GET_CHECK_MEMBER_NUM(family_info.family_aid_info_);
int32_t alive_num = 0;
lost_in_family.clear();
for (int i = 0; i < member_num; ++i)
{
// pair is <blockid, ds_id>
pair<uint64_t, uint64_t>& item = family_info.members_[i];
if (INVALID_SERVER_ID != item.second && item.second != ds_id) // exclude current ds
{
++alive_num;
}
else
{
// keep all lost(and will lost) data blockid
if (!IS_VERFIFY_BLOCK(item.first) && INVALID_BLOCK_ID != item.first)
{
lost_in_family.push_back(item.first);
}
}
}
if (alive_num < data_num) // can't reinstate
{
no_copy_blocks.insert(no_copy_blocks.end(), lost_in_family.begin(), lost_in_family.end());
}
}
}
}
else
{
// this error need script-calller care
TBSYS_LOG(ERROR, "get blockis ds_list fail from ns: %s, ret:%d",
tbsys::CNetUtil::addrToString(ns_id).c_str(), ret);
}
return ret;
}
int recover_block_from_disk_data(const char* ns_addr, const char* ns_slave_addr, BlockManager& block_manager,
VUINT64& tmp_fail_block_list, VUINT64& no_need_recover_block_list, VUINT64& success_block_list,
multimap<uint64_t, uint64_t>& fail_block_file_list, VUINT64& fail_block_list)
{
int ret = TFS_SUCCESS;
VUINT64 blocks;
vector<FileInfoV2> finfos;
multimap<uint64_t, uint64_t> tmp_fail_block_file_list;
uint64_t ns_id = Func::get_host_ip(ns_addr);
//get all blocks from ds disk
block_manager.get_all_block_ids(blocks);//肯定返回成功
vector<BlockMeta> blocks_meta;
ret = ToolUtil::get_all_blocks_meta(ns_id, blocks, blocks_meta, false);// no need get check block
TBSYS_LOG(DEBUG , "all logic blocks count: %zd, data block count: %zd", blocks.size(), blocks_meta.size());
if (TFS_SUCCESS == ret)
{
int32_t bret = TFS_SUCCESS;
for (uint32_t block_index = 0; block_index < blocks_meta.size(); ++block_index)
{
BlockMeta blk_meta = blocks_meta.at(block_index);
uint64_t block_id = blk_meta.block_id_;
if (blk_meta.size_ > 0 || INVALID_FAMILY_ID != blk_meta.family_info_.family_id_)
{
TBSYS_LOG(DEBUG , "blockid: %" PRI64_PREFIX "u no need recover, ds_size:%d, family_id: %" PRI64_PREFIX "u",
block_id, blk_meta.size_, blk_meta.family_info_.family_id_);
no_need_recover_block_list.push_back(block_id);//只要还有副本或者副本丢失但有编组(不考虑退化读恢复)都不用本工具恢复
}
else
{
bret = rm_no_replicate_block_from_ns(ns_addr, block_id);//now T_NEWBLK will can not remove empty ds_list's block
if (TFS_SUCCESS != bret)
{
fail_block_list.push_back(block_id);
TBSYS_LOG(WARN , "remove block %" PRI64_PREFIX "u from ns: %s failed, ret: %d", block_id, ns_addr, bret);
}
else
{
IndexHeaderV2 header;
finfos.clear();
bret = block_manager.traverse(header, finfos, block_id, block_id);
if (TFS_SUCCESS != bret)
{
TBSYS_LOG(WARN , "block %" PRI64_PREFIX "u get local file infos failed, ret: %d", block_id, bret);
tmp_fail_block_list.push_back(block_id);//只有本磁盘读取block的文件index错误，才需要整个block尝试从辅集群恢复
}
else
{
bool all_success = true;
int32_t copy_file_succ_count = 0;
tmp_fail_block_file_list.clear();
for (uint32_t file_index = 0; file_index < finfos.size(); ++file_index)
{
// skip deleted file
if ((finfos.at(file_index).status_ & FILE_STATUS_DELETE) != 0)
continue;
uint64_t file_id = finfos.at(file_index).id_;
bret = copy_file(block_manager, block_id, finfos.at(file_index));
if (TFS_SUCCESS == bret)
{
TBSYS_LOG(DEBUG, "recover block_id: %" PRI64_PREFIX "u, file_id: %" PRI64_PREFIX "u successful!", block_id, file_id);
}
else
{// 如果磁盘中该文件数据已经损坏(如crc出错)，则从对等集群(ns_slave_addr)拷贝数据复制
bret = copy_file_from_slave_cluster(ns_slave_addr, ns_addr, block_id, finfos.at(file_index));
if (TFS_SUCCESS == bret)
{
TBSYS_LOG(DEBUG, "recover block_id: %" PRI64_PREFIX "u, file_id: %" PRI64_PREFIX "u successful from slave cluster!", block_id, file_id);
}
else
{
TBSYS_LOG(WARN, "recover block_id: %" PRI64_PREFIX "u, file_id: %" PRI64_PREFIX "u failed from slave cluster, ret: %d!", block_id, file_id, bret);
all_success = false;
tmp_fail_block_file_list.insert(pair<uint64_t, uint64_t>(block_id, file_id));
}
}
if (TFS_SUCCESS == bret)
{
++copy_file_succ_count;
}
}
if (all_success)
{
success_block_list.push_back(block_id);
if (0 == copy_file_succ_count)
{
TBSYS_LOG(DEBUG, "recover block_id: %" PRI64_PREFIX "u need to do nothing,"
" because the count of files who need to recover is ZERO except DELETE status files!", block_id);
}
}
else if (0 == copy_file_succ_count)// all file(exclude DELETE) fail
{
TBSYS_LOG(WARN, "recover block_id: %" PRI64_PREFIX "u's files failed, copy_file_succ_count is Zero !", block_id);
fail_block_list.push_back(block_id);// for print fail block to out log file at end
}
else
{
fail_block_file_list.insert(tmp_fail_block_file_list.begin(), tmp_fail_block_file_list.end());
}
}
}
}
}//end for blocks loop
}
else
{
TBSYS_LOG(WARN, "get blockis ds_list error, ret:%d", ret);
}
TBSYS_LOG(INFO, "success_block_list size: %zd, tmp_fail_block_list size: %zd, fail_block_list size: %zd",
success_block_list.size(), tmp_fail_block_list.size(), fail_block_list.size());
return ret;
}