//将增量备份历史转换为完整备份
//暂时只能将最新的历史转换为完整备份
bool incremental_to_full(const std::string &prj, size_t index)
{
std::list<file_info> patch_list;
std::list<file_info> addition_list;
std::string history_path;
std::list<file_info> full_list;
std::list<file_info>::iterator iter;
std::string dest_dir;
std::string dest_path;
int fd;
//step 1: copy files from "current/" to "history/#/full/"
history_path = prj + "/history/" + size2string(index);
dest_path = history_path + "/full";
dest_dir = dest_path + "/";
rm_recursive(dest_path);
mkdir(dest_path.c_str(), 0755);
ffstorage::scan(prj.c_str(), &full_list);
for(iter = full_list.begin(); iter != full_list.end(); ++iter)
{
bool ok;
if(iter->type == 'f')
ok = link_or_copy(prj + "/current/" + iter->path, dest_dir + iter->path);
else if(iter->type == 'd')
ok = (0 == mkdir((dest_dir + iter->path).c_str(), 0775));
else
ok = false;
if(!ok)
goto fail;
}
fd = creat((history_path + "/full.done").c_str(), 0664);
if(fd == -1)
goto fail;
close(fd);
//step 2: remove incremental files (including patches)
if(!_read_list(history_path + "/patch_list", &patch_list))
goto fail;
if(!_read_list(history_path + "/addition_list", &addition_list))
goto fail;
for(size_t i = 0; i < patch_list.size(); ++i)
rm_recursive(history_path + "/patch." + size2string(i));
for(size_t i = 0; i < addition_list.size(); ++i)
//files of 'd' type are not exist
rm_recursive(history_path + "/" + size2string(i));
rm_recursive(history_path + "/addition_list");
rm_recursive(history_path + "/deletion_list");
rm_recursive(history_path + "/patch_list");
return true;
fail:
rm_recursive(dest_path);
return false;
}
Status ClientImpl::qrange(const std::string &name, int64_t begin, int64_t limit, std::vector<std::string> *ret){
std::string s_begin = str(begin);
std::string s_limit = str(limit);
const std::vector<std::string> *resp;
resp = this->request("qrange", name, s_begin, s_limit);
return _read_list(resp, ret);
}
Status ClientImpl::hgetall(const std::string &name,
std::vector<std::string> *ret)
{
const std::vector<std::string> *resp;
resp = this->request("hgetall", name );
return _read_list(resp, ret);
}
Status ClientImpl::rscan(const std::string &key_start, const std::string &key_end,
uint64_t limit, std::vector<std::string> *ret)
{
std::string s_limit = str(limit);
const std::vector<std::string> *resp;
resp = this->request("rscan", key_start, key_end, s_limit);
return _read_list(resp, ret);
}
Status ClientImpl::hkeys(const std::string &name,
const std::string &key_start, const std::string &key_end,
uint64_t limit, std::vector<std::string> *ret)
{
std::string s_limit = uint64_to_str(limit);
const std::vector<std::string> *resp;
resp = this->request("hkeys", name, key_start, key_end, s_limit);
return _read_list(resp, ret);
}
Status ClientImpl::qslice(const std::string &name,
int64_t begin, int64_t end,
std::vector<std::string> *ret)
{
std::string s_begin = str(begin);
std::string s_end = str(end);
const std::vector<std::string> *resp;
resp = this->request("qslice", name, s_begin, s_end);
return _read_list(resp, ret);
}
Status ClientImpl::zrrange(const std::string &name,
uint64_t offset, uint64_t limit,
std::vector<std::string> *ret)
{
std::string s_offset = str(offset);
std::string s_limit = str(limit);
const std::vector<std::string> *resp;
resp = this->request("zrrange", name, s_offset, s_limit);
return _read_list(resp, ret);
}
Status ClientImpl::zrscan(const std::string &name, const std::string &key_start,
int64_t *score_start, int64_t *score_end,
uint64_t limit, std::vector<std::string> *ret)
{
std::string s_score_start = score_start? str(*score_start) : "";
std::string s_score_end = score_end? str(*score_end) : "";
std::string s_limit = str(limit);
const std::vector<std::string> *resp;
resp = this->request("zrscan", name, key_start, s_score_start, s_score_end, s_limit);
return _read_list(resp, ret);
}
Status ClientImpl::qpop(const std::string &name, int64_t limit, std::vector<std::string> *ret){
const std::vector<std::string> *resp;
resp = this->request("qpop", name, str(limit));
return _read_list(resp, ret);
}
Status ClientImpl::multi_zget(const std::string &name, const std::vector<std::string> &keys,
std::vector<std::string> *ret){
const std::vector<std::string> *resp;
resp = this->request("multi_zget", name, keys);
return _read_list(resp, ret);
}
/**
*
* 检查、修复 corruption
*
* (1) /history/#/info 完整：如果 /cache 目录存在，删除掉即可
* (2) /history/#/info 不完整：此时并未发生 corruption，重写 info 文件即可
* (3) /history/#/info 不存在：此时可能发生 corruption，必须重新备份一次
*
*/
void storage_check()
{
size_t id;
std::string history_path;
std::list<std::string> prj_list;
std::list<std::string>::iterator prj;
size_t index;
std::list<file_info>::iterator iter;
std::list<file_info> patch_list;
std::list<file_info> deletion_list;
std::list<file_info> addition_list;
fprintf(stderr, "storage checking started\n");
prj_list = ffstorage::get_project_list();
for(prj = prj_list.begin(); prj != prj_list.end(); ++prj)
{
struct stat buf;
//删除残留的锁文件
rm_recursive(*prj + "/lock.0");
rm_recursive(*prj + "/lock.1");
id = ffstorage::get_history_qty(*prj);
if(id == 0)
continue;
--id;
fprintf(stderr, "checking %s : ", prj->c_str());
fflush(stderr);
history_path = *prj + "/history/" + size2string(id);
if(lstat((history_path + "/info").c_str(), &buf) == 0)
{
if(S_ISREG(buf.st_mode) && buf.st_size == 4)
{
//info 完整，对应于(1)
if(lstat((*prj + "/cache").c_str(), &buf) == 0)
rm_recursive(*prj + "/cache");
fprintf(stderr, "OK\n");
}
else
{
write_info(*prj, id); //info 存在但不完整，对应于(2)
fprintf(stderr, "history information repaired\n");
}
continue;
}
//info 不存在，对应于(3)
fprintf(stderr, "corruption detected");
fflush(stderr);
if(lstat((history_path + "/full").c_str(), &buf) == 0)
{
if(lstat((history_path + "/full.done").c_str(), &buf) == 0)
{
//此时，已经转换为完整备份，可以删掉多余的增量文件
}
else
{
//full.done 不存在，直接删除 full
//TODO: 也可以重新转换为完整备份
rm_recursive(history_path + "/full");
}
goto final;
}
_read_list(history_path + "/patch_list", &patch_list);
_read_list(history_path + "/deletion_list", &deletion_list);
_read_list(history_path + "/addition_list", &addition_list);
//将 patch 后的文件移动到 current 目录中
index = 0;
for(iter = patch_list.begin(); iter != patch_list.end(); ++iter)
{
rename((history_path + "/rc/" + size2string(index)).c_str(),
(*prj + "/current/" + iter->path).c_str());
++index;
}
rmdir((history_path + "/rc").c_str());
//递归删除列表中的文件
for(iter = deletion_list.begin(); iter != deletion_list.end(); ++iter)
rm_recursive(*prj + "/current/" + iter->path);
//将新增的文件复制到相应目录下
index = 0;
for(iter = addition_list.begin(); iter != addition_list.end(); ++iter)
{
if(iter->type == 'f')
link_or_copy(history_path + "/" + size2string(index),
*prj + "/current/" + iter->path);
else if(iter->type == 'd')
mkdir((*prj + "/current/" + iter->path).c_str(), 0775);
++index;
}
final:
ffstorage::write_info(*prj, id);
fprintf(stderr, ", repaired\n");
}
bool _restore(const std::string &project_name,
const std::string &storage_path, const std::string &history_path)
{
std::list<file_info> patch_list;
std::list<file_info> deletion_list;
std::list<file_info> addition_list;
std::list<file_info>::iterator iter;
size_t index;
if(!_read_list(history_path + "/patch_list", &patch_list))
return false;
if(!_read_list(history_path + "/deletion_list", &deletion_list))
return false;
if(!_read_list(history_path + "/addition_list", &addition_list))
return false;
//rsync patch
index = 0;
for(iter = patch_list.begin(); iter != patch_list.end(); ++iter)
{
std::string basis;
std::string patch;
std::string output;
basis = storage_path + "/" + iter->path;
patch = history_path + "/patch." + size2string(index);
output = project_name + "/tmp_ffbackup";
if(rsync_patch(basis, patch, output) == false)
{
rm_recursive(output);
return false;
}
if(rename(output.c_str(), basis.c_str()) < 0)
{
rm_recursive(output);
return false;
}
++index;
}
//process deletion list
for(iter = deletion_list.begin(); iter != deletion_list.end(); ++iter)
{
rm_recursive(storage_path + "/" + iter->path);
}
//process addition list
index = 0;
for(iter = addition_list.begin(); iter != addition_list.end(); ++iter)
{
std::string path(storage_path + "/" + iter->path);
bool ok;
if(iter->type == 'f')
ok = link_or_copy(history_path + "/" + size2string(index), path);
else if(iter->type == 'd')
ok = (0 == mkdir(path.c_str(), 0775));
else
ok = false;
if(!ok)
return false;
++index;
}
return true;
}