int do_rsync(const char* db_path, const char* manifest_file, std::vector<int32_t>& buckets,
ClusterHandler* local_handler, ClusterHandler* remote_handler, bool mtime_care,
DataFilter& filter, DataStat& stat, RecordLogger* fail_logger)
{
// open db with specified manifest(read only)
leveldb::DB* db = NULL;
leveldb::Options open_options;
open_options.error_if_exists = false; // exist is ok
open_options.create_if_missing = true; // create if not exist
open_options.comparator = LdbComparator(NULL); // self-defined comparator
open_options.env = leveldb::Env::Instance();
leveldb::Status s = leveldb::DB::Open(open_options, db_path, manifest_file, &db);
if (!s.ok())
{
log_error("open db with mainfest fail: %s", s.ToString().c_str());
delete open_options.comparator;
delete open_options.env;
return TAIR_RETURN_FAILED;
}
// get db iterator
leveldb::ReadOptions scan_options;
scan_options.verify_checksums = false;
scan_options.fill_cache = false;
leveldb::Iterator* db_it = db->NewIterator(scan_options);
char scan_key[LDB_KEY_META_SIZE];
bool skip_in_bucket = false;
bool skip_in_area = false;
uint32_t start_time = 0;
int32_t bucket = -1;
int32_t area = -1;
LdbKey ldb_key;
LdbItem ldb_item;
data_entry* key = NULL;
data_entry* value = NULL;
int32_t mtime_care_flag = mtime_care ? TAIR_CLIENT_DATA_MTIME_CARE : 0;
int ret = TAIR_RETURN_SUCCESS;
if (db_it == NULL)
{
log_error("new db iterator fail.");
ret = TAIR_RETURN_FAILED;
}
else
{
for (size_t i = 0; !g_stop && i < buckets.size(); ++i)
{
start_time = time(NULL);
area = -1;
bucket = buckets[i];
// seek to bucket
LdbKey::build_key_meta(scan_key, bucket);
for (db_it->Seek(leveldb::Slice(scan_key, sizeof(scan_key))); !g_stop && db_it->Valid(); db_it->Next())
{
ret = TAIR_RETURN_SUCCESS;
skip_in_bucket = false;
skip_in_area = false;
ldb_key.assign(const_cast<char*>(db_it->key().data()), db_it->key().size());
ldb_item.assign(const_cast<char*>(db_it->value().data()), db_it->value().size());
area = LdbKey::decode_area(ldb_key.key());
// current bucket iterate over
if (ldb_key.get_bucket_number() != bucket)
{
break;
}
// skip this data
if (!filter.ok(area))
{
skip_in_bucket = true;
}
else
{
key = new data_entry(ldb_key.key(), ldb_key.key_size(), false);
value = NULL;
key->has_merged = true;
key->set_prefix_size(ldb_item.prefix_size());
// re-get from local
if (local_handler != NULL)
{
ret = get_from_local_cluster(*local_handler, *key, value, skip_in_area);
}
else
{
value = new data_entry(ldb_item.value(), ldb_item.value_size(), false);
key->data_meta.cdate = value->data_meta.cdate = ldb_item.cdate();
key->data_meta.edate = value->data_meta.edate = ldb_item.edate();
key->data_meta.mdate = value->data_meta.mdate = ldb_item.mdate();
key->data_meta.version = value->data_meta.version = ldb_item.version();
key->data_meta.keysize = value->data_meta.keysize = key->get_size();
key->data_meta.valsize = value->data_meta.valsize = ldb_item.value_size();
value->data_meta.flag = ldb_item.flag();
}
if (ret == TAIR_RETURN_SUCCESS)
{
log_debug("@@ k:%d %s %d %d %u %u %u.v:%s %d %d %u %u %u", key->get_area(), key->get_size() > 6 ? key->get_data()+6 : "", key->get_size(), key->get_prefix_size(),key->data_meta.cdate,key->data_meta.mdate,key->data_meta.edate, value->get_size() > 4 ? value->get_data()+4 : "", value->get_size(), value->data_meta.flag, value->data_meta.cdate, value->data_meta.mdate, value->data_meta.edate);
// mtime care / skip cache
key->data_meta.flag = mtime_care_flag | TAIR_CLIENT_PUT_SKIP_CACHE_FLAG;
key->server_flag = TAIR_SERVERFLAG_RSYNC;
// sync to remote cluster
ret = remote_handler->client()->put(key->get_area(), *key, *value, 0, 0, false/* not fill cache */);
if (ret == TAIR_RETURN_MTIME_EARLY)
{
ret = TAIR_RETURN_SUCCESS;
}
}
// log failed key
if (ret != TAIR_RETURN_SUCCESS)
{
log_error("fail one: %d", ret);
FailRecord record(key, remote_handler->info(), ret);
data_entry entry;
FailRecord::record_to_entry(record, entry);
int tmp_ret = fail_logger->add_record(0, TAIR_REMOTE_SYNC_TYPE_PUT, &entry, NULL);
if (tmp_ret != TAIR_RETURN_SUCCESS)
{
log_error("add fail record fail, ret: %d", tmp_ret);
}
}
}
// update stat
stat.update(bucket, skip_in_bucket ? -1 : area, // skip in bucket, then no area to update
ldb_key.key_size() + ldb_item.value_size(), (skip_in_bucket || skip_in_area), ret == TAIR_RETURN_SUCCESS);
// cleanup
if (key != NULL)
{
delete key;
key = NULL;
}
if (value != NULL)
{
delete value;
value = NULL;
}
}
log_warn("sync bucket %d over, cost: %d(s), stat:\n",
bucket, time(NULL) - start_time);
// only dump bucket stat
stat.dump(bucket, -1);
}
}
if (db_it != NULL)
{
delete db_it;
}
if (db != NULL)
{
delete db;
}
delete open_options.comparator;
delete open_options.env;
return ret;
}
int do_dump(const char* db_path, const char* manifest, const char* cmp_desc,
const std::vector<int32_t>& buckets, DataFilter& filter, DataStat& stat,
const char* dump_file, int64_t dump_file_max_size)
{
// open db
leveldb::Options open_options;
leveldb::DB* db = NULL;
leveldb::Status s = open_db_readonly(db_path, manifest, cmp_desc, open_options, db);
if (!s.ok())
{
fprintf(stderr, "open db fail: %s\n", s.ToString().c_str());
return 1;
}
// get db iterator
leveldb::ReadOptions scan_options;
scan_options.verify_checksums = false;
scan_options.fill_cache = false;
leveldb::Iterator* db_it = db->NewIterator(scan_options);
char scan_key[LDB_KEY_META_SIZE];
int32_t bucket = 0;
int32_t area = 0;
LdbKey ldb_key;
LdbItem ldb_item;
int32_t size = 0;
bool skip_in_bucket = false;
bool skip_in_area = false;
int dump_fd = -1;
int32_t dump_file_index = 1;
int64_t dump_file_size = 0;
static const int32_t BUF_SIZE = 2 << 20; // 2M
char* buf = new char[BUF_SIZE];
int32_t buf_remain = BUF_SIZE;
int ret = 0;
for (size_t i = 0; !g_stop && i < buckets.size(); ++i)
{
area = -1;
bucket = buckets[i];
// seek to bucket
LdbKey::build_key_meta(scan_key, bucket);
for (db_it->Seek(leveldb::Slice(scan_key, sizeof(scan_key))); !g_stop && db_it->Valid() && ret == 0; db_it->Next())
{
skip_in_bucket = false;
skip_in_area = false;
ldb_key.assign(const_cast<char*>(db_it->key().data()), db_it->key().size());
ldb_item.assign(const_cast<char*>(db_it->value().data()), db_it->value().size());
area = LdbKey::decode_area(ldb_key.key());
// current bucket iterate over
if (ldb_key.get_bucket_number() != bucket)
{
break;
}
// skip this data
if (!filter.ok(area))
{
skip_in_bucket = true;
}
else
{
// open new dump file
if (dump_file_size >= dump_file_max_size || dump_fd < 0)
{
if (dump_fd > 0)
{
close(dump_fd);
}
char name[TAIR_MAX_PATH_LEN];
snprintf(name, sizeof(name), "%s.%d", dump_file, dump_file_index);
// open dump file
dump_fd = open(name, O_RDWR|O_CREAT|O_TRUNC, 0444);
if (dump_fd <= 0)
{
fprintf(stderr, "open dump file fail, file: %s, error: %s\n", name, strerror(errno));
ret = 1;
break;
}
dump_file_size = 0;
dump_file_index++;
}
// appropriate size
size = ldb_key.key_size() + ldb_item.value_size() + 3*sizeof(int32_t);
if (size < BUF_SIZE)
{
if (size > buf_remain)
{
if (write(dump_fd, buf, BUF_SIZE - buf_remain) != (BUF_SIZE - buf_remain))
{
fprintf(stderr, "write file fail: %s\n", strerror(errno));
ret = 1;
}
dump_file_size += (BUF_SIZE - buf_remain);
buf_remain = BUF_SIZE;
}
size = encode_ldb_kv(buf + (BUF_SIZE - buf_remain), ldb_key, ldb_item);
buf_remain -= size;
}
else // big data
{
char* tmp_buf = new char[size];
size = encode_ldb_kv(tmp_buf, ldb_key, ldb_item);
if (write(dump_fd, tmp_buf, size) != size)
{
fprintf(stderr, "write file fail: %s\n", strerror(errno));
ret = 1;
}
delete [] tmp_buf;
dump_file_size += size;
}
}
// update stat
stat.update(bucket, skip_in_bucket ? -1 : area, // skip in bucket, then no area to update
ldb_key.key_size() + ldb_item.value_size(), (skip_in_bucket || skip_in_area), ret == 0);
}
if (ret != 0)
{
break;
}
// only dump bucket stat
stat.dump(bucket, -1);
}
// last data
if (ret == 0 && buf_remain != BUF_SIZE)
{
if (write(dump_fd, buf, BUF_SIZE - buf_remain) != (BUF_SIZE - buf_remain))
{
fprintf(stderr, "write file fail: %s\n", strerror(errno));
ret = 1;
}
}
if (dump_fd > 0)
{
close(dump_fd);
}
// cleanup
delete [] buf;
if (db_it != NULL)
{
delete db_it;
}
if (db != NULL)
{
delete db;
delete open_options.comparator;
delete open_options.env;
delete open_options.info_log;
}
stat.dump_all();
return ret;
}
