// returns the number of newly added items
static int hset_one(const SSDB *ssdb, const Bytes &name, const Bytes &key, const Bytes &val, char log_type){
if(name.size() > SSDB_KEY_LEN_MAX ){
log_error("name too long!");
return -1;
}
if(key.size() > SSDB_KEY_LEN_MAX){
log_error("key too long!");
return -1;
}
int ret = 0;
std::string dbval;
if(ssdb->hget(name, key, &dbval) == 0){ // not found
std::string hkey = encode_hash_key(name, key);
ssdb->binlogs->Put(hkey, val.Slice());
ssdb->binlogs->add(log_type, BinlogCommand::HSET, hkey);
ret = 1;
}else{
if(dbval != val){
std::string hkey = encode_hash_key(name, key);
ssdb->binlogs->Put(hkey, val.Slice());
ssdb->binlogs->add(log_type, BinlogCommand::HSET, hkey);
}
ret = 0;
}
return ret;
}
int SSDB::raw_set(const Bytes &key, const Bytes &val) const{
leveldb::Status s = db->Put(write_options, key.Slice(), val.Slice());
if(!s.ok()){
log_error("set error: %s", s.ToString().c_str());
return -1;
}
return 1;
}
static int qset_one(SSDB *ssdb, const Bytes &name, uint64_t seq, const Bytes &item){
std::string key = encode_qitem_key(name, seq);
leveldb::Status s;
ssdb->binlogs->Put(key, item.Slice());
return 0;
}
int SSDB::hset(const Bytes &name, const Bytes &key, const Bytes &val) const{
std::string hkey = encode_hash_key(name, key);
std::string dbval;
leveldb::Status s;
leveldb::WriteBatch batch;
if(this->hget(name, key, &dbval) == 0){
int64_t size = this->hsize(name);
if(size == -1){
return -1;
}
size ++;
std::string size_key = encode_hsize_key(name);
batch.Put(size_key, leveldb::Slice((char *)&size, sizeof(int64_t)));
}
if(dbval != val){
batch.Put(hkey, val.Slice());
}
s = db->Write(write_options, &batch);
if(!s.ok()){
return -1;
}
return 1;
}
int SSDB::raw_del(const Bytes &key) const{
leveldb::Status s = db->Delete(write_options, key.Slice());
if(!s.ok()){
log_error("del error: %s", s.ToString().c_str());
return -1;
}
return 1;
}
int SSDB::hset(const Bytes &name, const Bytes &key, const Bytes &val) const{
std::string buf = encode_hash_key(name, key);
leveldb::Status s = db->Put(write_options, buf, val.Slice());
if(!s.ok()){
return -1;
}
return 1;
}
int SSDB::raw_get(const Bytes &key, std::string *val) const{
leveldb::ReadOptions opts;
opts.fill_cache = false;
leveldb::Status s = db->Get(opts, key.Slice(), val);
if(s.IsNotFound()){
return 0;
}
if(!s.ok()){
log_error("get error: %s", s.ToString().c_str());
return -1;
}
return 1;
}
int SSDB::set(const Bytes &key, const Bytes &val, char log_type){
Transaction trans(binlogs);
std::string buf = encode_kv_key(key);
binlogs->Put(buf, val.Slice());
binlogs->add(log_type, BinlogCommand::KSET, buf);
leveldb::Status s = binlogs->commit();
if(!s.ok()){
log_error("set error: %s", s.ToString().c_str());
return -1;
}
return 1;
}
int BackendSync::Client::copy(){
if(this->iter == NULL){
log_debug("new iterator, last_key: '%s'", hexmem(last_key.data(), last_key.size()).c_str());
this->iter = backend->ssdb->iterator(this->last_key, "", -1);
}
for(int i=0; i<1000; i++){
if(!iter->next()){
log_info("fd: %d, copy end", link->fd());
this->status = Client::SYNC;
delete this->iter;
this->iter = NULL;
Binlog log(this->last_seq, BinlogType::COPY, BinlogCommand::END, "");
log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
link->send(log.repr(), "copy_end");
break;
}else{
Bytes key = iter->key();
Bytes val = iter->val();
this->last_key = key.String();
if(key.size() == 0){
continue;
}
char cmd = 0;
char data_type = key.data()[0];
if(data_type == DataType::KV){
cmd = BinlogCommand::KSET;
}else if(data_type == DataType::HASH){
cmd = BinlogCommand::HSET;
}else if(data_type == DataType::ZSET){
cmd = BinlogCommand::ZSET;
}else{
continue;
}
Binlog log(this->last_seq, BinlogType::COPY, cmd, key.Slice());
log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
link->send(log.repr(), val);
//if(link->output->size() > 1024 * 1024){
break;
//}
}
}
return 1;
}
int SSDB::getset(const Bytes &key, std::string *val, const Bytes &newval, char log_type){
if(key.empty()){
log_error("empty key!");
//return -1;
return 0;
}
Transaction trans(binlogs);
int found = this->get(key, val);
std::string buf = encode_kv_key(key);
binlogs->Put(buf, newval.Slice());
binlogs->add_log(log_type, BinlogCommand::KSET, buf);
leveldb::Status s = binlogs->commit();
if(!s.ok()){
log_error("set error: %s", s.ToString().c_str());
return -1;
}
return found;
}
int main(int argc, char **argv){
welcome();
set_log_level(Logger::LEVEL_MIN);
if(argc <= 3){
usage(argc, argv);
return 0;
}
char *ip = argv[1];
int port = atoi(argv[2]);
char *output_folder = argv[3];
if(file_exists(output_folder)){
printf("output_folder[%s] exists!\n", output_folder);
return 0;
}
if(mkdir(output_folder, 0777) == -1){
perror("error create backup directory!\n");
return 0;
}
std::string data_dir = "";
data_dir.append(output_folder);
data_dir.append("/data");
{
std::string meta_dir = "";
meta_dir.append(output_folder);
meta_dir.append("/meta");
int ret;
ret = mkdir(meta_dir.c_str(), 0755);
if(ret == -1){
fprintf(stderr, "error creating meta dir\n");
exit(0);
}
}
// connect to server
Link *link = Link::connect(ip, port);
if(link == NULL){
fprintf(stderr, "error connecting to server!\n");
return 0;
}
link->send("dump", "A", "", "-1");
link->flush();
leveldb::DB* db;
leveldb::Options options;
leveldb::Status status;
options.create_if_missing = true;
options.write_buffer_size = 32 * 1024 * 1024;
options.compression = leveldb::kSnappyCompression;
status = leveldb::DB::Open(options, data_dir.c_str(), &db);
if(!status.ok()){
printf("open leveldb: %s error!\n", output_folder);
return 0;
}
int dump_count = 0;
while(1){
const std::vector<Bytes> *req = link->recv();
if(req == NULL){
printf("recv error\n");
printf("ERROR: failed to dump data!\n");
exit(0);
}else if(req->empty()){
int len = link->read();
if(len <= 0){
printf("read error: %s\n", strerror(errno));
printf("ERROR: failed to dump data!\n");
exit(0);
}
}else{
Bytes cmd = req->at(0);
if(cmd == "begin"){
printf("recv begin...\n");
}else if(cmd == "end"){
printf("received %d entry(s)\n", dump_count);
printf("recv end\n\n");
break;
}else if(cmd == "set"){
/*
std::string s = serialize_req(*req);
printf("%s\n", s.c_str());
*/
if(req->size() != 3){
printf("invalid set params!\n");
printf("ERROR: failed to dump data!\n");
exit(0);
}
Bytes key = req->at(1);
Bytes val = req->at(2);
if(key.size() == 0 || key.data()[0] == DataType::SYNCLOG){
continue;
}
leveldb::Slice k = key.Slice();
leveldb::Slice v = val.Slice();
status = db->Put(leveldb::WriteOptions(), k, v);
if(!status.ok()){
printf("put leveldb error!\n");
printf("ERROR: failed to dump data!\n");
exit(0);
}
dump_count ++;
if((int)log10(dump_count - 1) != (int)log10(dump_count) || (dump_count > 0 && dump_count % 50000 == 0)){
printf("received %d entry(s)\n", dump_count);
}
}else{
printf("error: unknown command %s\n", std::string(cmd.data(), cmd.size()).c_str());
printf("ERROR: failed to dump data!\n");
exit(0);
}
}
}
printf("total dumped %d entry(s)\n", dump_count);
/*
printf("checking data...\n");
leveldb::Iterator *it;
it = db->NewIterator(leveldb::ReadOptions());
int save_count = 0;
for(it->SeekToFirst(); it->Valid(); it->Next()){
save_count ++;
//std::string k = hexmem(it->key().data(), it->key().size());
//std::string v = hexmem(it->value().data(), it->value().size());
//printf("%d %s : %s", save_count, k.c_str(), v.c_str());
}
if(dump_count != save_count){
printf("checking failed! dumped: %d, saved: %d\n", dump_count, save_count);
}else{
printf("checking OK.\n");
printf("\n");
}
*/
{
std::string val;
if(db->GetProperty("leveldb.stats", &val)){
fprintf(stderr, "%s\n", val.c_str());
}
}
fprintf(stderr, "compacting data...\n");
db->CompactRange(NULL, NULL);
{
std::string val;
if(db->GetProperty("leveldb.stats", &val)){
fprintf(stderr, "%s\n", val.c_str());
}
}
printf("backup has been made to folder: %s\n", output_folder);
delete link;
delete db;
return 0;
}
int BackendSync::Client::copy(){
if(this->iter == NULL){
log_debug("new iterator, last_key: '%s'", hexmem(last_key.data(), last_key.size()).c_str());
std::string key = this->last_key;
if(this->last_key.empty()){
key.push_back(DataType::MIN_PREFIX);
}
this->iter = backend->ssdb->iterator(key, "", -1);
}
int ret = 0;
int iterate_count = 0;
while(true){
// Prevent copy() from blocking too long
if(++iterate_count > 10000 || link->output->size() > 2 * 1024 * 1024){
break;
}
if(!iter->next()){
goto copy_end;
}
Bytes key = iter->key();
if(key.size() == 0){
continue;
}
// finish copying all valid data types
if(key.data()[0] > DataType::MAX_PREFIX){
goto copy_end;
}
Bytes val = iter->val();
this->last_key = key.String();
char cmd = 0;
char data_type = key.data()[0];
if(data_type == DataType::KV){
cmd = BinlogCommand::KSET;
}else if(data_type == DataType::HASH){
cmd = BinlogCommand::HSET;
}else if(data_type == DataType::ZSET){
cmd = BinlogCommand::ZSET;
}else{
continue;
}
ret = 1;
Binlog log(this->last_seq, BinlogType::COPY, cmd, key.Slice());
log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
link->send(log.repr(), val);
}
return ret;
copy_end:
log_info("%s:%d fd: %d, copy end", link->remote_ip, link->remote_port, link->fd());
this->status = Client::SYNC;
delete this->iter;
this->iter = NULL;
Binlog log(this->last_seq, BinlogType::COPY, BinlogCommand::END, "");
log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
link->send(log.repr(), "copy_end");
return 1;
}