int main(int argc, char* argv[])
{
int ret = TAIR_RETURN_SUCCESS;
char* db_path = NULL;
char* local_cluster_addr = NULL;
char* remote_cluster_addr = NULL;
char* manifest_file = NULL;
char* fail_logger_file = NULL;
char* buckets = NULL;
char* yes_areas = NULL;
char* no_areas = NULL;
bool mtime_care = true;
int i = 0;
while ((i = getopt(argc, argv, "p:f:l:r:e:b:a:A:n")) != EOF)
{
switch (i)
{
case 'p':
db_path = optarg;
break;
case 'f':
manifest_file = optarg;
break;
case 'l':
local_cluster_addr = optarg;
break;
case 'r':
remote_cluster_addr = optarg;
break;
case 'e':
fail_logger_file = optarg;
break;
case 'b':
buckets = optarg;
break;
case 'a':
yes_areas = optarg;
break;
case 'A':
no_areas = optarg;
break;
case 'n':
mtime_care = false;
break;
default:
print_help(argv[0]);
return 1;
}
}
if (db_path == NULL || manifest_file == NULL || remote_cluster_addr == NULL || fail_logger_file == NULL || buckets == NULL)
{
print_help(argv[0]);
return 1;
}
// init signals
signal(SIGINT, sign_handler);
signal(SIGTERM, sign_handler);
TBSYS_LOGGER.setLogLevel("warn");
// init local cluster handler(optional)
ClusterHandler* local_handler = NULL;
if (local_cluster_addr != NULL)
{
local_handler = new ClusterHandler();
ret = init_cluster_handler(local_cluster_addr, *local_handler);
if (ret != TAIR_RETURN_SUCCESS)
{
log_error("init local client fail, addr: %d, ret: %d", local_cluster_addr, ret);
delete local_handler;
return 1;
}
}
// init remote cluster handler(must)
ClusterHandler* remote_handler = new ClusterHandler();
ret = init_cluster_handler(remote_cluster_addr, *remote_handler);
if (ret != TAIR_RETURN_SUCCESS)
{
log_error("init remote client fail, addr: %s, ret: %d", remote_cluster_addr, ret);
delete remote_handler;
return 1;
}
// init buckets
std::vector<int32_t> bucket_container;
std::vector<std::string> bucket_strs;
tair::util::string_util::split_str(buckets, ", ", bucket_strs);
for (size_t i = 0; i < bucket_strs.size(); ++i)
{
bucket_container.push_back(atoi(bucket_strs[i].c_str()));
}
// init fail logger
RecordLogger* fail_logger = new SequentialFileRecordLogger(fail_logger_file, 30<<20/*30M*/, true/*rotate*/);
if (fail_logger->init() != TAIR_RETURN_SUCCESS)
{
log_error("init fail logger fail, ret: %d", ret);
}
else
{
// init data filter
DataFilter filter(yes_areas, no_areas);
// init data stat
DataStat stat;
// do data rsync
uint32_t start_time = time(NULL);
ret = do_rsync(db_path, manifest_file, bucket_container, local_handler, remote_handler, mtime_care, filter, stat, fail_logger);
log_warn("rsync data over, stopped: %s, cost: %u(s), stat:", g_stop ? "yes" : "no", time(NULL) - start_time);
stat.dump_all();
}
// cleanup
delete fail_logger;
if (local_handler != NULL)
{
delete local_handler;
}
if (remote_handler != NULL)
{
delete remote_handler;
}
return ret == TAIR_RETURN_SUCCESS ? 0 : 1;
}
int do_dump(const char* db_path, const char* manifest, const char* cmp_desc,
const std::vector<int32_t>& buckets, DataFilter& filter, DataStat& stat,
const char* dump_file, int64_t dump_file_max_size)
{
// open db
leveldb::Options open_options;
leveldb::DB* db = NULL;
leveldb::Status s = open_db_readonly(db_path, manifest, cmp_desc, open_options, db);
if (!s.ok())
{
fprintf(stderr, "open db fail: %s\n", s.ToString().c_str());
return 1;
}
// get db iterator
leveldb::ReadOptions scan_options;
scan_options.verify_checksums = false;
scan_options.fill_cache = false;
leveldb::Iterator* db_it = db->NewIterator(scan_options);
char scan_key[LDB_KEY_META_SIZE];
int32_t bucket = 0;
int32_t area = 0;
LdbKey ldb_key;
LdbItem ldb_item;
int32_t size = 0;
bool skip_in_bucket = false;
bool skip_in_area = false;
int dump_fd = -1;
int32_t dump_file_index = 1;
int64_t dump_file_size = 0;
static const int32_t BUF_SIZE = 2 << 20; // 2M
char* buf = new char[BUF_SIZE];
int32_t buf_remain = BUF_SIZE;
int ret = 0;
for (size_t i = 0; !g_stop && i < buckets.size(); ++i)
{
area = -1;
bucket = buckets[i];
// seek to bucket
LdbKey::build_key_meta(scan_key, bucket);
for (db_it->Seek(leveldb::Slice(scan_key, sizeof(scan_key))); !g_stop && db_it->Valid() && ret == 0; db_it->Next())
{
skip_in_bucket = false;
skip_in_area = false;
ldb_key.assign(const_cast<char*>(db_it->key().data()), db_it->key().size());
ldb_item.assign(const_cast<char*>(db_it->value().data()), db_it->value().size());
area = LdbKey::decode_area(ldb_key.key());
// current bucket iterate over
if (ldb_key.get_bucket_number() != bucket)
{
break;
}
// skip this data
if (!filter.ok(area))
{
skip_in_bucket = true;
}
else
{
// open new dump file
if (dump_file_size >= dump_file_max_size || dump_fd < 0)
{
if (dump_fd > 0)
{
close(dump_fd);
}
char name[TAIR_MAX_PATH_LEN];
snprintf(name, sizeof(name), "%s.%d", dump_file, dump_file_index);
// open dump file
dump_fd = open(name, O_RDWR|O_CREAT|O_TRUNC, 0444);
if (dump_fd <= 0)
{
fprintf(stderr, "open dump file fail, file: %s, error: %s\n", name, strerror(errno));
ret = 1;
break;
}
dump_file_size = 0;
dump_file_index++;
}
// appropriate size
size = ldb_key.key_size() + ldb_item.value_size() + 3*sizeof(int32_t);
if (size < BUF_SIZE)
{
if (size > buf_remain)
{
if (write(dump_fd, buf, BUF_SIZE - buf_remain) != (BUF_SIZE - buf_remain))
{
fprintf(stderr, "write file fail: %s\n", strerror(errno));
ret = 1;
}
dump_file_size += (BUF_SIZE - buf_remain);
buf_remain = BUF_SIZE;
}
size = encode_ldb_kv(buf + (BUF_SIZE - buf_remain), ldb_key, ldb_item);
buf_remain -= size;
}
else // big data
{
char* tmp_buf = new char[size];
size = encode_ldb_kv(tmp_buf, ldb_key, ldb_item);
if (write(dump_fd, tmp_buf, size) != size)
{
fprintf(stderr, "write file fail: %s\n", strerror(errno));
ret = 1;
}
delete [] tmp_buf;
dump_file_size += size;
}
}
// update stat
stat.update(bucket, skip_in_bucket ? -1 : area, // skip in bucket, then no area to update
ldb_key.key_size() + ldb_item.value_size(), (skip_in_bucket || skip_in_area), ret == 0);
}
if (ret != 0)
{
break;
}
// only dump bucket stat
stat.dump(bucket, -1);
}
// last data
if (ret == 0 && buf_remain != BUF_SIZE)
{
if (write(dump_fd, buf, BUF_SIZE - buf_remain) != (BUF_SIZE - buf_remain))
{
fprintf(stderr, "write file fail: %s\n", strerror(errno));
ret = 1;
}
}
if (dump_fd > 0)
{
close(dump_fd);
}
// cleanup
delete [] buf;
if (db_it != NULL)
{
delete db_it;
}
if (db != NULL)
{
delete db;
delete open_options.comparator;
delete open_options.env;
delete open_options.info_log;
}
stat.dump_all();
return ret;
}
