Example #1
0
//将增量备份历史转换为完整备份
//暂时只能将最新的历史转换为完整备份
bool incremental_to_full(const std::string &prj, size_t index)
{
    std::list<file_info> patch_list;
    std::list<file_info> addition_list;
    std::string history_path;
    std::list<file_info> full_list;
    std::list<file_info>::iterator iter;
    std::string dest_dir;
    std::string dest_path;
    int fd;

    //step 1: copy files from "current/" to "history/#/full/"

    history_path = prj + "/history/" + size2string(index);
    dest_path = history_path + "/full";
    dest_dir = dest_path + "/";
    rm_recursive(dest_path);
    mkdir(dest_path.c_str(), 0755);
    ffstorage::scan(prj.c_str(), &full_list);
    for(iter = full_list.begin(); iter != full_list.end(); ++iter)
    {
        bool ok;

        if(iter->type == 'f')
            ok = link_or_copy(prj + "/current/" + iter->path, dest_dir + iter->path);
        else if(iter->type == 'd')
            ok = (0 == mkdir((dest_dir + iter->path).c_str(), 0775));
        else
            ok = false;
        if(!ok)
            goto fail;
    }
    fd = creat((history_path + "/full.done").c_str(), 0664);
    if(fd == -1)
        goto fail;
    close(fd);

    //step 2: remove incremental files (including patches)
    if(!_read_list(history_path + "/patch_list", &patch_list))
        goto fail;
    if(!_read_list(history_path + "/addition_list", &addition_list))
        goto fail;
    for(size_t i = 0; i < patch_list.size(); ++i)
        rm_recursive(history_path + "/patch." + size2string(i));
    for(size_t i = 0; i < addition_list.size(); ++i)
        //files of 'd' type are not exist
        rm_recursive(history_path + "/" + size2string(i));
    rm_recursive(history_path + "/addition_list");
    rm_recursive(history_path + "/deletion_list");
    rm_recursive(history_path + "/patch_list");

    return true;
fail:
    rm_recursive(dest_path);
    return false;
}
Example #2
0
Status ClientImpl::qrange(const std::string &name, int64_t begin, int64_t limit, std::vector<std::string> *ret){
	std::string s_begin = str(begin);
	std::string s_limit = str(limit);
	const std::vector<std::string> *resp;
	resp = this->request("qrange", name, s_begin, s_limit);
	return _read_list(resp, ret);
}
Example #3
0
Status ClientImpl::hgetall(const std::string &name,
	 std::vector<std::string> *ret)
{
	const std::vector<std::string> *resp;
	resp = this->request("hgetall", name );
	return _read_list(resp, ret);
}
Example #4
0
Status ClientImpl::rscan(const std::string &key_start, const std::string &key_end,
	uint64_t limit, std::vector<std::string> *ret)
{
	std::string s_limit = str(limit);
	const std::vector<std::string> *resp;
	resp = this->request("rscan", key_start, key_end, s_limit);
	return _read_list(resp, ret);
}
Example #5
0
Status ClientImpl::hkeys(const std::string &name,
	const std::string &key_start, const std::string &key_end,
	uint64_t limit, std::vector<std::string> *ret)
{
	std::string s_limit = uint64_to_str(limit);
	const std::vector<std::string> *resp;
	resp = this->request("hkeys", name, key_start, key_end, s_limit);
	return _read_list(resp, ret);
}
Example #6
0
Status ClientImpl::qslice(const std::string &name,
		int64_t begin, int64_t end,
		std::vector<std::string> *ret)
{
	std::string s_begin = str(begin);
	std::string s_end = str(end);
	const std::vector<std::string> *resp;
	resp = this->request("qslice", name, s_begin, s_end);
	return _read_list(resp, ret);
}
Example #7
0
Status ClientImpl::zrrange(const std::string &name,
		uint64_t offset, uint64_t limit,
		std::vector<std::string> *ret)
{
	std::string s_offset = str(offset);
	std::string s_limit = str(limit);
	const std::vector<std::string> *resp;
	resp = this->request("zrrange", name, s_offset, s_limit);
	return _read_list(resp, ret);
}
Example #8
0
Status ClientImpl::zrscan(const std::string &name, const std::string &key_start,
	int64_t *score_start, int64_t *score_end,
	uint64_t limit, std::vector<std::string> *ret)
{
	std::string s_score_start = score_start? str(*score_start) : "";
	std::string s_score_end = score_end? str(*score_end) : "";
	std::string s_limit = str(limit);
	const std::vector<std::string> *resp;
	resp = this->request("zrscan", name, key_start, s_score_start, s_score_end, s_limit);
	return _read_list(resp, ret);
}
Example #9
0
Status ClientImpl::qpop(const std::string &name, int64_t limit, std::vector<std::string> *ret){
	const std::vector<std::string> *resp;
	resp = this->request("qpop", name, str(limit));
	return _read_list(resp, ret);
}
Example #10
0
Status ClientImpl::multi_zget(const std::string &name, const std::vector<std::string> &keys,
	std::vector<std::string> *ret){
	const std::vector<std::string> *resp;
	resp = this->request("multi_zget", name, keys);
	return _read_list(resp, ret);
}
Example #11
0
/**
 *
 * 检查、修复 corruption
 *
 * (1) /history/#/info 完整:如果 /cache 目录存在,删除掉即可
 * (2) /history/#/info 不完整:此时并未发生 corruption,重写 info 文件即可
 * (3) /history/#/info 不存在:此时可能发生 corruption,必须重新备份一次
 *
 */
void storage_check()
{
    size_t id;
    std::string history_path;
    std::list<std::string> prj_list;
    std::list<std::string>::iterator prj;
    size_t index;
    std::list<file_info>::iterator iter;
    std::list<file_info> patch_list;
    std::list<file_info> deletion_list;
    std::list<file_info> addition_list;

    fprintf(stderr, "storage checking started\n");
    prj_list = ffstorage::get_project_list();
    for(prj = prj_list.begin(); prj != prj_list.end(); ++prj)
    {
        struct stat buf;

        //删除残留的锁文件
        rm_recursive(*prj + "/lock.0");
        rm_recursive(*prj + "/lock.1");

        id = ffstorage::get_history_qty(*prj);
        if(id == 0)
            continue;
        --id;

        fprintf(stderr, "checking %s : ", prj->c_str());
        fflush(stderr);

        history_path = *prj + "/history/" + size2string(id);
        if(lstat((history_path + "/info").c_str(), &buf) == 0)
        {
            if(S_ISREG(buf.st_mode) && buf.st_size == 4)
            {
                //info 完整,对应于(1)
                if(lstat((*prj + "/cache").c_str(), &buf) == 0)
                    rm_recursive(*prj + "/cache");
                fprintf(stderr, "OK\n");
            }
            else
            {
                write_info(*prj, id); //info 存在但不完整,对应于(2)
                fprintf(stderr, "history information repaired\n");
            }
            continue;
        }

        //info 不存在,对应于(3)
        fprintf(stderr, "corruption detected");
        fflush(stderr);

        if(lstat((history_path + "/full").c_str(), &buf) == 0)
        {
            if(lstat((history_path + "/full.done").c_str(), &buf) == 0)
            {
                //此时,已经转换为完整备份,可以删掉多余的增量文件
            }
            else
            {
                //full.done 不存在,直接删除 full
                //TODO: 也可以重新转换为完整备份
                rm_recursive(history_path + "/full");
            }
            goto final;
        }

        _read_list(history_path + "/patch_list", &patch_list);
        _read_list(history_path + "/deletion_list", &deletion_list);
        _read_list(history_path + "/addition_list", &addition_list);

        //将 patch 后的文件移动到 current 目录中
        index = 0;
        for(iter = patch_list.begin(); iter != patch_list.end(); ++iter)
        {
            rename((history_path + "/rc/" + size2string(index)).c_str(),
                   (*prj + "/current/" + iter->path).c_str());
            ++index;
        }
        rmdir((history_path + "/rc").c_str());
        //递归删除列表中的文件
        for(iter = deletion_list.begin(); iter != deletion_list.end(); ++iter)
            rm_recursive(*prj + "/current/" + iter->path);
        //将新增的文件复制到相应目录下
        index = 0;
        for(iter = addition_list.begin(); iter != addition_list.end(); ++iter)
        {
            if(iter->type == 'f')
                link_or_copy(history_path + "/" + size2string(index),
                          *prj + "/current/" + iter->path);
            else if(iter->type == 'd')
                mkdir((*prj + "/current/" + iter->path).c_str(), 0775);
            ++index;
        }

final:
        ffstorage::write_info(*prj, id);
        fprintf(stderr, ", repaired\n");
    }
Example #12
0
bool _restore(const std::string &project_name,
              const std::string &storage_path, const std::string &history_path)
{
    std::list<file_info> patch_list;
    std::list<file_info> deletion_list;
    std::list<file_info> addition_list;
    std::list<file_info>::iterator iter;
    size_t index;

    if(!_read_list(history_path + "/patch_list", &patch_list))
        return false;
    if(!_read_list(history_path + "/deletion_list", &deletion_list))
        return false;
    if(!_read_list(history_path + "/addition_list", &addition_list))
        return false;

    //rsync patch
    index = 0;
    for(iter = patch_list.begin(); iter != patch_list.end(); ++iter)
    {
        std::string basis;
        std::string patch;
        std::string output;

        basis = storage_path + "/" + iter->path;
        patch = history_path + "/patch." + size2string(index);
        output = project_name + "/tmp_ffbackup";
        if(rsync_patch(basis, patch, output) == false)
        {
            rm_recursive(output);
            return false;
        }
        if(rename(output.c_str(), basis.c_str()) < 0)
        {
            rm_recursive(output);
            return false;
        }
        ++index;
    }

    //process deletion list
    for(iter = deletion_list.begin(); iter != deletion_list.end(); ++iter)
    {
        rm_recursive(storage_path + "/" + iter->path);
    }

    //process addition list
    index = 0;
    for(iter = addition_list.begin(); iter != addition_list.end(); ++iter)
    {
        std::string path(storage_path + "/" + iter->path);
        bool ok;

        if(iter->type == 'f')
            ok = link_or_copy(history_path + "/" + size2string(index), path);
        else if(iter->type == 'd')
            ok = (0 == mkdir(path.c_str(), 0775));
        else
            ok = false;
        if(!ok)
            return false;
        ++index;
    }
    return true;
}