int SSDBImpl::_qpop(const Bytes &name, std::string *item, uint64_t front_or_back_seq, char log_type){
Transaction trans(binlogs);
int ret;
uint64_t seq;
ret = qget_uint64(this->ldb, name, front_or_back_seq, &seq);
if(ret == -1){
return -1;
}
if(ret == 0){
return 0;
}
ret = qget_by_seq(this->ldb, name, seq, item);
if(ret == -1){
return -1;
}
if(ret == 0){
return 0;
}
// delete item
ret = qdel_one(this, name, seq);
if(ret == -1){
return -1;
}
if(front_or_back_seq == QFRONT_SEQ){
binlogs->add_log(log_type, BinlogCommand::QPOP_FRONT, name.String());
}else{
binlogs->add_log(log_type, BinlogCommand::QPOP_BACK, name.String());
}
// update size
int64_t size = incr_qsize(this, name, -1);
if(size == -1){
return -1;
}
// update front
if(size > 0){
seq += (front_or_back_seq == QFRONT_SEQ)? +1 : -1;
//log_debug("seq: %" PRIu64 ", ret: %d", seq, ret);
ret = qset_one(this, name, front_or_back_seq, Bytes(&seq, sizeof(seq)));
if(ret == -1){
return -1;
}
}
leveldb::Status s = binlogs->commit();
if(!s.ok()){
log_error("Write error! %s", s.ToString().c_str());
return -1;
}
return 1;
}
void BackendSync::Client::dump(){
Buffer *output = link->output;
if(!this->iter->next()){
delete this->iter;
this->iter = NULL;
this->status = Client::SYNC;
log_info("fd: %d, dump end", link->fd());
output->append_record("dump_end");
output->append('\n');
}else{
Bytes key = this->iter->key();
Bytes val = this->iter->val();
this->last_key = key.String();
log_trace("fd: %d, dump: %s", link->fd(), hexmem(key.data(), key.size()).c_str());
output->append_record("dump_set");
if(this->last_seq > 0){
char buf[20];
snprintf(buf, sizeof(buf), "%llu", this->last_seq);
output->append_record(buf);
}else{
output->append_record("0");
}
output->append_record(key);
output->append_record(val);
output->append('\n');
}
}
int ExpirationHandler::set_ttl(const Bytes &key, int64_t ttl){
int64_t expired = time_ms() + ttl * 1000;
char data[30];
int size = snprintf(data, sizeof(data), "%" PRId64, expired);
if(size <= 0){
log_error("snprintf return error!");
return -1;
}
int ret = ssdb->zset(this->list_name, key, Bytes(data, size));
if(ret == -1){
return -1;
}
if(!this->enable){
return 0;
}
if(expired < first_timeout){
first_timeout = expired;
}
std::string s_key = key.String();
fast_keys.del(s_key);
if(expired <= fast_keys.max_score()){
fast_keys.add(s_key, expired);
if(fast_keys.size() > BATCH_SIZE){
log_debug("pop_back");
fast_keys.pop_back();
}
}else{
//log_debug("don't put in fast_keys");
}
return 0;
}
int BackendSync::Client::copy(){
if(this->iter == NULL){
log_debug("new iterator, last_key: '%s'", hexmem(last_key.data(), last_key.size()).c_str());
this->iter = backend->ssdb->iterator(this->last_key, "", -1);
}
for(int i=0; i<1000; i++){
if(!iter->next()){
log_info("fd: %d, copy end", link->fd());
this->status = Client::SYNC;
delete this->iter;
this->iter = NULL;
Binlog log(this->last_seq, BinlogType::COPY, BinlogCommand::END, "");
log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
link->send(log.repr(), "copy_end");
break;
}else{
Bytes key = iter->key();
Bytes val = iter->val();
this->last_key = key.String();
if(key.size() == 0){
continue;
}
char cmd = 0;
char data_type = key.data()[0];
if(data_type == DataType::KV){
cmd = BinlogCommand::KSET;
}else if(data_type == DataType::HASH){
cmd = BinlogCommand::HSET;
}else if(data_type == DataType::ZSET){
cmd = BinlogCommand::ZSET;
}else{
continue;
}
Binlog log(this->last_seq, BinlogType::COPY, cmd, key.Slice());
log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
link->send(log.repr(), val);
//if(link->output->size() > 1024 * 1024){
break;
//}
}
}
return 1;
}
int proc_multi_exists(NetworkServer *net, Link *link, const Request &req, Response *resp){
SSDBServer *serv = (SSDBServer *)net->data;
CHECK_NUM_PARAMS(2);
resp->push_back("ok");
for(Request::const_iterator it=req.begin()+1; it!=req.end(); it++){
const Bytes key = *it;
std::string val;
int ret = serv->ssdb->get(key, &val);
resp->push_back(key.String());
if(ret == 1){
resp->push_back("1");
}else if(ret == 0){
resp->push_back("0");
}else{
resp->push_back("0");
}
}
return 0;
}
static int proc_multi_exists(Server *serv, Link *link, const Request &req, Response *resp){
if(req.size() < 2){
resp->push_back("client_error");
}else{
resp->push_back("ok");
for(Request::const_iterator it=req.begin()+1; it!=req.end(); it++){
const Bytes key = *it;
std::string val;
int ret = serv->ssdb->get(key, &val);
resp->push_back(key.String());
if(ret == 1){
resp->push_back("1");
}else if(ret == 0){
resp->push_back("0");
}else{
resp->push_back("0");
}
}
}
return 0;
}
int BackendSync::Client::copy() {
if(this->iter == NULL) {
log_info("new iterator, last_key: '%s'", hexmem(last_key.data(), last_key.size()).c_str());
std::string key = this->last_key;
if(this->last_key.empty()) {
key.push_back(DataType::MIN_PREFIX);
}
this->iter = backend->ssdb->iterator(key, "", -1);
log_info("iterator created, last_key: '%s'", hexmem(last_key.data(), last_key.size()).c_str());
}
int ret = 0;
int iterate_count = 0;
int64_t stime = time_ms();
while(true) {
// Prevent copy() from blocking too long
if(++iterate_count > 1000 || link->output->size() > 2 * 1024 * 1024) {
break;
}
if(!iter->next()) {
goto copy_end;
}
Bytes key = iter->key();
if(key.size() == 0) {
continue;
}
// finish copying all valid data types
if(key.data()[0] > DataType::MAX_PREFIX) {
goto copy_end;
}
Bytes val = iter->val();
this->last_key = key.String();
char cmd = 0;
char data_type = key.data()[0];
if(data_type == DataType::KV) {
cmd = BinlogCommand::KSET;
} else if(data_type == DataType::HASH) {
cmd = BinlogCommand::HSET;
} else if(data_type == DataType::ZSET) {
cmd = BinlogCommand::ZSET;
} else if(data_type == DataType::QUEUE) {
cmd = BinlogCommand::QPUSH_BACK;
} else {
continue;
}
ret++;
Binlog log(this->last_seq, BinlogType::COPY, cmd, slice(key));
log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
link->send(log.repr(), val);
if(time_ms() - stime > 3000) {
log_info("copy blocks too long, flush");
break;
}
}
return ret;
copy_end:
log_info("%s:%d fd: %d, copy end", link->remote_ip, link->remote_port, link->fd());
this->status = Client::SYNC;
delete this->iter;
this->iter = NULL;
Binlog log(this->last_seq, BinlogType::COPY, BinlogCommand::END, "");
log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
link->send(log.repr(), "copy_end");
return 1;
}
void* Slave::_run_thread(void *arg){
Slave *slave = (Slave *)arg;
const SSDB *ssdb = slave->ssdb;
const char *ip = slave->master_ip.c_str();
int port = slave->master_port;
Link *link = NULL;
int retry = 0;
const std::vector<Bytes> *req;
while(true){
if(link == NULL){
if(retry){
int t = retry > 15? 15 : retry;
usleep(t * 1000 * 1000);
log_info("[%d] connecting to master at %s:%d...", retry, ip, port);
}
link = connect(ip, port, slave->next_seq, slave->last_key);
if(link == NULL){
retry ++;
continue;
}else{
retry = 0;
}
}
req = link->recv();
if(req == NULL){
retry = 1;
delete link;
link = NULL;
log_info("recv error, reconnecting to master...");
continue;
}else if(req->empty()){
if(link->read() <= 0){
retry = 1;
delete link;
link = NULL;
log_info("network error, reconnecting to master...");
}
continue;
}
Bytes cmd = req->at(0);
if(cmd == "sync_set"){
log_trace("recv sync: %s", serialize_req(*req).c_str());
if(req->size() != 4){
log_warn("invalid set params!");
break;
}
uint64_t seq = req->at(1).Uint64();
Bytes key = req->at(2);
Bytes val = req->at(3);
if(seq == 0){
// dump
slave->last_key = key.String();
}else{
// sync
slave->next_seq = seq + 1;
}
int ret = ssdb->raw_set(key, val);
if(ret == -1){
log_error("ssdb.raw_set error!");
}
slave->save_status();
}else if(cmd == "sync_del"){
log_trace("recv sync: %s", serialize_req(*req).c_str());
if(req->size() != 3){
log_warn("invalid del params!");
break;
}
uint64_t seq = req->at(1).Uint64();
Bytes key = req->at(2);
if(seq == 0){
// dump
slave->last_key = key.String();
}else{
// sync
slave->next_seq = seq + 1;
}
int ret = ssdb->raw_del(key);
if(ret == -1){
log_error("ssdb.raw_del error!");
}
slave->save_status();
}else if(cmd == "dump_end"){
log_info("dump end, step in sync");
slave->last_key = "";
}else if(cmd == "noop"){
//
}else{
log_warn("unknow sync command: %s", serialize_req(*req).c_str());
}
} // end while
if(link){
delete link;
}
log_info("Slave thread quit");
return (void *)NULL;
}
int ExpirationHandler::del_ttl(const Bytes &key){
fast_keys.del(key.String());
ssdb->zdel(this->list_name, key);
return 0;
}