Пример #1
0
// returns the number of newly added items
static int hset_one(SSDBImpl *ssdb, const Bytes &name, const Bytes &key, const Bytes &val, char log_type){
	if(name.empty() || key.empty()){
		log_error("empty name or key!");
		return -1;
	}
	if(name.size() > SSDB_KEY_LEN_MAX ){
		log_error("name too long! %s", hexmem(name.data(), name.size()).c_str());
		return -1;
	}
	if(key.size() > SSDB_KEY_LEN_MAX){
		log_error("key too long! %s", hexmem(key.data(), key.size()).c_str());
		return -1;
	}
	int ret = 0;
	std::string dbval;
	if(ssdb->hget(name, key, &dbval) == 0){ // not found
		std::string hkey = encode_hash_key(name, key);
		ssdb->binlogs->Put(hkey, slice(val));
		ssdb->binlogs->add_log(log_type, BinlogCommand::HSET, hkey);
		ret = 1;
	}else{
		if(dbval != val){
			std::string hkey = encode_hash_key(name, key);
			ssdb->binlogs->Put(hkey, slice(val));
			ssdb->binlogs->add_log(log_type, BinlogCommand::HSET, hkey);
		}
		ret = 0;
	}
	return ret;
}
Пример #2
0
void BackendSync::Client::init(){
	const std::vector<Bytes> *req = this->link->last_recv();
	last_seq = 0;
	if(req->size() > 1){
		last_seq = req->at(1).Uint64();
	}
	last_key = "";
	if(req->size() > 2){
		last_key = req->at(2).String();
	}
	// is_mirror
	if(req->size() > 3){
		if(req->at(3).String() == "mirror"){
			is_mirror = true;
		}
	}
	const char *type = is_mirror? "mirror" : "sync";
	if(last_key == "" && last_seq != 0){
		log_info("[%s]fd: %d, sync, seq: %" PRIu64", key: '%s'",
			type,
			link->fd(),
			last_seq, hexmem(last_key.data(), last_key.size()).c_str()
			);
		this->status = Client::SYNC;
	}else{
		// a slave must reset its last_key when receiving 'copy_end' command
		log_info("[%s]fd: %d, copy recover, seq: %" PRIu64", key: '%s'",
			type,
			link->fd(),
			last_seq, hexmem(last_key.data(), last_key.size()).c_str()
			);
		this->status = Client::COPY;
	}
}
Пример #3
0
void BackendSync::Client::init(){
	const std::vector<Bytes> *req = this->link->last_recv();
	last_seq = 0;
	if(req->size() > 1){
		last_seq = req->at(1).Uint64();
	}
	last_key = "";
	if(req->size() > 2){
		last_key = req->at(2).String();
	}
	// is_mirror
	if(req->size() > 3){
		if(req->at(3).String() == "mirror"){
			is_mirror = true;
		}
	}
	
	SSDBImpl *ssdb = (SSDBImpl *)backend->ssdb;
	BinlogQueue *logs = ssdb->binlogs;
	if(last_seq != 0 && (last_seq > logs->max_seq() || last_seq < logs->min_seq())){
		log_error("%s:%d fd: %d OUT_OF_SYNC! seq: %" PRIu64 " not in [%" PRIu64 ", %" PRIu64 "]",
			link->remote_ip, link->remote_port, link->fd(),
			last_seq, logs->min_seq(), logs->max_seq()
			);
		this->out_of_sync();
		return;
	}
	
	const char *type = is_mirror? "mirror" : "sync";
	// a slave must reset its last_key when receiving 'copy_end' command
	if(last_key == "" && last_seq != 0){
		log_info("[%s] %s:%d fd: %d, sync recover, seq: %" PRIu64 ", key: '%s'",
			type,
			link->remote_ip, link->remote_port,
			link->fd(),
			last_seq, hexmem(last_key.data(), last_key.size()).c_str()
			);
		this->status = Client::SYNC;
		
		Binlog log(this->last_seq, BinlogType::COPY, BinlogCommand::END, "");
		log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
		link->send(log.repr(), "copy_end");
	}else if(last_key == "" && last_seq == 0){
		log_info("[%s] %s:%d fd: %d, copy begin, seq: %" PRIu64 ", key: '%s'",
			type,
			link->remote_ip, link->remote_port,
			link->fd(),
			last_seq, hexmem(last_key.data(), last_key.size()).c_str()
			);
		this->reset();
	}else{
		log_info("[%s] %s:%d fd: %d, copy recover, seq: %" PRIu64 ", key: '%s'",
			type,
			link->remote_ip, link->remote_port,
			link->fd(),
			last_seq, hexmem(last_key.data(), last_key.size()).c_str()
			);
		this->status = Client::COPY;
	}
}
Пример #4
0
void BackendSync::Client::init() {
    const std::vector<Bytes> *req = this->link->last_recv();
    last_seq = 0;
    if(req->size() > 1) {
        last_seq = req->at(1).Uint64();
    }
    last_key = "";
    if(req->size() > 2) {
        last_key = req->at(2).String();
    }
    // is_mirror
    if(req->size() > 3) {
        if(req->at(3).String() == "mirror") {
            is_mirror = true;
        }
    }
    const char *type = is_mirror? "mirror" : "sync";
    // a slave must reset its last_key when receiving 'copy_end' command
    if(last_key == "" && last_seq != 0) {
        log_info("[%s] %s:%d fd: %d, sync recover, seq: %" PRIu64 ", key: '%s'",
                 type,
                 link->remote_ip, link->remote_port,
                 link->fd(),
                 last_seq, hexmem(last_key.data(), last_key.size()).c_str()
                );
        this->status = Client::SYNC;

        Binlog log(this->last_seq, BinlogType::COPY, BinlogCommand::END, "");
        log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
        link->send(log.repr(), "copy_end");
    } else if(last_key == "" && last_seq == 0) {
        log_info("[%s] %s:%d fd: %d, copy begin, seq: %" PRIu64 ", key: '%s'",
                 type,
                 link->remote_ip, link->remote_port,
                 link->fd(),
                 last_seq, hexmem(last_key.data(), last_key.size()).c_str()
                );
        this->reset();
    } else {
        log_info("[%s] %s:%d fd: %d, copy recover, seq: %" PRIu64 ", key: '%s'",
                 type,
                 link->remote_ip, link->remote_port,
                 link->fd(),
                 last_seq, hexmem(last_key.data(), last_key.size()).c_str()
                );
        this->status = Client::COPY;
    }
}
Пример #5
0
Slave::Slave(SSDB *ssdb, leveldb::DB* meta_db, const char *ip, int port, bool is_mirror){
	thread_quit = false;
	this->ssdb = ssdb;
	this->meta_db = meta_db;
	this->master_ip = std::string(ip);
	this->master_port = port;
	this->is_mirror = is_mirror;
	if(this->is_mirror){
		this->log_type = BinlogType::MIRROR;
	}else{
		this->log_type = BinlogType::SYNC;
	}
	
	this->link = NULL;
	this->last_seq = 0;
	this->last_key = "";
	this->connect_retry = 0;
	
	this->copy_count = 0;
	this->sync_count = 0;

	load_status();
	log_debug("last_seq: %"PRIu64", last_key: %s",
		last_seq, hexmem(last_key.data(), last_key.size()).c_str());
}
Пример #6
0
void BackendSync::Client::dump(){
	Buffer *output = link->output;
	if(!this->iter->next()){
		delete this->iter;
		this->iter = NULL;
		this->status = Client::SYNC;
		log_info("fd: %d, dump end", link->fd());
		output->append_record("dump_end");
		output->append('\n');
	}else{
		Bytes key = this->iter->key();
		Bytes val = this->iter->val();
		this->last_key = key.String();
		log_trace("fd: %d, dump: %s", link->fd(), hexmem(key.data(), key.size()).c_str());

		output->append_record("dump_set");
		if(this->last_seq > 0){
			char buf[20];
			snprintf(buf, sizeof(buf), "%llu", this->last_seq);
			output->append_record(buf);
		}else{
			output->append_record("0");
		}
		output->append_record(key);
		output->append_record(val);
		output->append('\n');
	}
}
Пример #7
0
void BackendSync::Client::init(){
	const std::vector<Bytes>* req = this->link->last_recv();
	last_seq = 0;
	if(req->size() > 1){
		last_seq = req->at(1).Uint64();
	}
	last_key = "";
	if(req->size() > 2){
		last_key = req->at(2).String();
	}
	// is_mirror
	std::string type = "sync";
	if(req->size() > 3){
		if(req->at(3).String() == "mirror"){
			type = "mirror";
			is_mirror = true;
		}
	}
	if(!is_mirror && (last_seq == 0 || last_key != "")){
		if(last_seq == 0){
			// in case that slave has an error that last_key is not empty
			last_key = "";
			log_info("fd: %d, new slave, make a full dumping", link->fd());
			// TODO: send dump_begin
		}else{
			// last_key != ""
			// a slave must reset its last_key when receiving 'dump_end' command
			log_info("fd: %d, dump recover, seq: %llu, key: %s",
				link->fd(),
				last_seq, hexmem(last_key.data(), last_key.size()).c_str()
				);
		}
		std::string end = "";
		int limit = 2147483647;
		this->iter = backend->ssdb->iterator(last_key, end, limit);
		this->status = Client::DUMP;
	}else{
		log_info("[%s]fd: %d, sync recover, seq: %llu, key: %s",
			type.c_str(),
			link->fd(),
			last_seq, hexmem(last_key.data(), last_key.size()).c_str()
			);
		this->status = Client::SYNC;
	}
}
Пример #8
0
std::string Binlog::dumps() const{
	std::string str;
	if(buf.size() < HEADER_LEN){
		return str;
	}
	str.append("binlog# ");
	char buf[20];
	snprintf(buf, sizeof(buf), "%llu ", this->seq());
	str.append(buf);

	switch(this->type()){
		case BinlogType::NOOP:
			str.append("noop ");
			break;
		case BinlogType::SYNC:
			str.append("sync ");
			break;
		case BinlogType::MIRROR:
			str.append("mirror ");
			break;
		case BinlogType::COPY:
			str.append("copy ");
			break;
	}
	switch(this->cmd()){
		case BinlogCommand::NONE:
			str.append("none ");
			break;
		case BinlogCommand::KSET:
			str.append("set ");
			break;
		case BinlogCommand::KDEL:
			str.append("del ");
			break;
		case BinlogCommand::HSET:
			str.append("hset ");
			break;
		case BinlogCommand::HDEL:
			str.append("hdel ");
			break;
		case BinlogCommand::ZSET:
			str.append("zset ");
			break;
		case BinlogCommand::ZDEL:
			str.append("zdel ");
			break;
		case BinlogCommand::BEGIN:
			str.append("begin ");
			break;
		case BinlogCommand::END:
			str.append("end ");
			break;
	}
	Bytes b = this->key();
	str.append(hexmem(b.data(), b.size()));
	return str;
}
Пример #9
0
static int hdel_one(SSDBImpl *ssdb, const Bytes &name, const Bytes &key, char log_type){
	if(name.size() > SSDB_KEY_LEN_MAX ){
		log_error("name too long! %s", hexmem(name.data(), name.size()).c_str());
		return -1;
	}
	if(key.size() > SSDB_KEY_LEN_MAX){
		log_error("key too long! %s", hexmem(key.data(), key.size()).c_str());
		return -1;
	}
	std::string dbval;
	if(ssdb->hget(name, key, &dbval) == 0){
		return 0;
	}

	std::string hkey = encode_hash_key(name, key);
	ssdb->binlogs->Delete(hkey);
	ssdb->binlogs->add_log(log_type, BinlogCommand::HDEL, hkey);
	
	return 1;
}
Пример #10
0
Файл: slave.cpp Проект: 29n/ssdb
void Slave::start(){
	load_status();
	log_debug("last_seq: %" PRIu64 ", last_key: %s",
		last_seq, hexmem(last_key.data(), last_key.size()).c_str());

	thread_quit = false;
	int err = pthread_create(&run_thread_tid, NULL, &Slave::_run_thread, this);
	if(err != 0){
		log_error("can't create thread: %s", strerror(err));
	}
}
Пример #11
0
static int proc_info(Server *serv, Link *link, const Request &req, Response *resp){
	resp->push_back("ok");
	resp->push_back("ssdb-server");
	resp->push_back("version");
	resp->push_back(SSDB_VERSION);
	
	if(req.size() == 1 || req[1] == "cmd"){
		for(Command *cmd=commands; cmd->name; cmd++){
			char buf[128];
			snprintf(buf, sizeof(buf), "cmd.%s", cmd->name);
			resp->push_back(buf);
			snprintf(buf, sizeof(buf), "calls: %"PRIu64"\ttime_wait: %.0f\ttime_proc: %.0f",
				cmd->calls, cmd->time_wait, cmd->time_proc);
			resp->push_back(buf);
		}
	}

	if(req.size() == 1 || req[1] == "range"){
		std::vector<std::string> tmp;
		int ret = serv->ssdb->key_range(&tmp);
		if(ret == 0){
			char buf[512];
			
			resp->push_back("key_range.kv");
			snprintf(buf, sizeof(buf), "\"%s\" - \"%s\"",
				hexmem(tmp[0].data(), tmp[0].size()).c_str(),
				hexmem(tmp[1].data(), tmp[1].size()).c_str()
				);
			resp->push_back(buf);
			
			resp->push_back("key_range.hash");
			snprintf(buf, sizeof(buf), "\"%s\" - \"%s\"",
				hexmem(tmp[2].data(), tmp[2].size()).c_str(),
				hexmem(tmp[3].data(), tmp[3].size()).c_str()
				);
			resp->push_back(buf);
			
			resp->push_back("key_range.zset");
			snprintf(buf, sizeof(buf), "\"%s\" - \"%s\"",
				hexmem(tmp[4].data(), tmp[4].size()).c_str(),
				hexmem(tmp[5].data(), tmp[5].size()).c_str()
				);
			resp->push_back(buf);
		}
	}

	if(req.size() == 1 || req[1] == "leveldb"){
		std::vector<std::string> tmp = serv->ssdb->info();
		for(int i=0; i<(int)tmp.size(); i++){
			std::string block = tmp[i];
			resp->push_back(block);
		}
	}
	
	return 0;
}
Пример #12
0
int BackendSync::Client::copy(){
	if(this->iter == NULL){
		log_debug("new iterator, last_key: '%s'", hexmem(last_key.data(), last_key.size()).c_str());
		this->iter = backend->ssdb->iterator(this->last_key, "", -1);
	}
	for(int i=0; i<1000; i++){
		if(!iter->next()){
			log_info("fd: %d, copy end", link->fd());
			this->status = Client::SYNC;
			delete this->iter;
			this->iter = NULL;

			Binlog log(this->last_seq, BinlogType::COPY, BinlogCommand::END, "");
			log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
			link->send(log.repr(), "copy_end");
			break;
		}else{
			Bytes key = iter->key();
			Bytes val = iter->val();
			this->last_key = key.String();
			
			if(key.size() == 0){
				continue;
			}
			
			char cmd = 0;
			char data_type = key.data()[0];
			if(data_type == DataType::KV){
				cmd = BinlogCommand::KSET;
			}else if(data_type == DataType::HASH){
				cmd = BinlogCommand::HSET;
			}else if(data_type == DataType::ZSET){
				cmd = BinlogCommand::ZSET;
			}else{
				continue;
			}
			
			Binlog log(this->last_seq, BinlogType::COPY, cmd, key.Slice());
			log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
			link->send(log.repr(), val);
			//if(link->output->size() > 1024 * 1024){
			break;
			//}
		}
	}
	
	return 1;
}
Пример #13
0
static void test_unhexmem_one(const char *s, size_t l, int retval) {
        _cleanup_free_ char *hex = NULL;
        _cleanup_free_ void *mem = NULL;
        size_t len;

        assert_se(unhexmem(s, l, &mem, &len) == retval);
        if (retval == 0) {
                char *answer;

                if (l == (size_t) -1)
                        l = strlen(s);

                assert_se(hex = hexmem(mem, len));
                answer = strndupa(strempty(s), l);
                assert_se(streq(delete_chars(answer, WHITESPACE), hex));
        }
}
Пример #14
0
void Slave::load_status(){
	std::string key = status_key();
	std::string status;
	leveldb::Status s;
	s = meta_db->Get(leveldb::ReadOptions(), key, &status);
	if(s.IsNotFound()){
		return;
	}
	if(s.ok()){
		if(status.size() < sizeof(uint64_t)){
			log_error("invlid format of status");
		}else{
			next_seq = *((uint64_t *)(status.data()));
			last_key = std::string(status.data() + sizeof(uint64_t), status.size() - sizeof(uint64_t));
			log_debug("load_status seq: %llu, key: %s",
				next_seq,
				hexmem(last_key.data(), last_key.size()).c_str());
		}
	}
}
Пример #15
0
static std::string serialize_req(T &req){
	std::string ret;
	char buf[50];
	for(int i=0; i<req.size(); i++){
		if(i >= 5 && i < req.size() - 1){
			sprintf(buf, "[%d more...]", (int)req.size() - i - 1);
			ret.append(buf);
			break;
		}
		if(((req[0] == "get" || req[0] == "set") && i == 1) || req[i].size() < 30){
			std::string h = hexmem(req[i].data(), req[i].size());
			ret.append(h);
		}else{
			sprintf(buf, "[%d bytes]", (int)req[i].size());
			ret.append(buf);
		}
		if(i < req.size() - 1){
			ret.append(" ");
		}
	}
	return ret;
}
Пример #16
0
void Slave::migrate_old_status(){
	std::string old_key = "new.slave.status|" + this->id_;
	std::string val;
	int old_found = meta->raw_get(old_key, &val);
	if(!old_found){
		return;
	}
	if(val.size() < sizeof(uint64_t)){
		log_error("invalid format of status");
		return;
	}
	last_seq = *((uint64_t *)(val.data()));
	last_key.assign(val.data() + sizeof(uint64_t), val.size() - sizeof(uint64_t));
	// migrate old status
	log_info("migrate old version slave status to new format, last_seq: %" PRIu64 ", last_key: %s",
		last_seq, hexmem(last_key.data(), last_key.size()).c_str());
	
	save_status();
	if(meta->raw_del(old_key) == -1){
		log_fatal("meta db error!");
		exit(1);
	}
}
Пример #17
0
int Slave::proc_sync(const Binlog &log, const std::vector<Bytes> &req){
	switch(log.cmd()){
		case BinlogCommand::KSET:
			{
				if(req.size() != 2){
					break;
				}
				std::string key;
				if(decode_kv_key(log.key(), &key) == -1){
					break;
				}
				log_trace("set %s", hexmem(key.data(), key.size()).c_str());
				if(ssdb->set(key, req[1], log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::KDEL:
			{
				std::string key;
				if(decode_kv_key(log.key(), &key) == -1){
					break;
				}
				log_trace("del %s", hexmem(key.data(), key.size()).c_str());
				if(ssdb->del(key, log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::HSET:
			{
				if(req.size() != 2){
					break;
				}
				std::string name, key;
				if(decode_hash_key(log.key(), &name, &key) == -1){
					break;
				}
				log_trace("hset %s %s",
					hexmem(name.data(), name.size()).c_str(),
					hexmem(key.data(), key.size()).c_str());
				if(ssdb->hset(name, key, req[1], log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::HDEL:
			{
				std::string name, key;
				if(decode_hash_key(log.key(), &name, &key) == -1){
					break;
				}
				log_trace("hdel %s %s",
					hexmem(name.data(), name.size()).c_str(),
					hexmem(key.data(), key.size()).c_str());
				if(ssdb->hdel(name, key, log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::ZSET:
			{
				if(req.size() != 2){
					break;
				}
				std::string name, key;
				if(decode_zset_key(log.key(), &name, &key) == -1){
					break;
				}
				log_trace("zset %s %s",
					hexmem(name.data(), name.size()).c_str(),
					hexmem(key.data(), key.size()).c_str());
				if(ssdb->zset(name, key, req[1], log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::ZDEL:
			{
				std::string name, key;
				if(decode_zset_key(log.key(), &name, &key) == -1){
					break;
				}
				log_trace("zdel %s %s",
					hexmem(name.data(), name.size()).c_str(),
					hexmem(key.data(), key.size()).c_str());
				if(ssdb->zdel(name, key, log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::QSET:
		case BinlogCommand::QPUSH_BACK:
		case BinlogCommand::QPUSH_FRONT:
			{
				if(req.size() != 2){
					break;
				}
				std::string name;
				uint64_t seq;
				if(decode_qitem_key(log.key(), &name, &seq) == -1){
					break;
				}
				if(seq < QITEM_MIN_SEQ || seq > QITEM_MAX_SEQ){
					break;
				}
				int ret;
				if(log.cmd() == BinlogCommand::QSET){
					log_trace("qset %s %" PRIu64 "", hexmem(name.data(), name.size()).c_str(), seq);
					ret = ssdb->qset_by_seq(name, seq, req[1], log_type);
				}else if(log.cmd() == BinlogCommand::QPUSH_BACK){
					log_trace("qpush_back %s", hexmem(name.data(), name.size()).c_str());
					ret = ssdb->qpush_back(name, req[1], log_type);
				}else{
					log_trace("qpush_front %s", hexmem(name.data(), name.size()).c_str());
					ret = ssdb->qpush_front(name, req[1], log_type);
				}
				if(ret == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::QPOP_BACK:
		case BinlogCommand::QPOP_FRONT:
			{
				int ret;
				const Bytes name = log.key();
				std::string tmp;
				if(log.cmd() == BinlogCommand::QPOP_BACK){
					log_trace("qpop_back %s", hexmem(name.data(), name.size()).c_str());
					ret = ssdb->qpop_back(name, &tmp, log_type);
				}else{
					log_trace("qpop_front %s", hexmem(name.data(), name.size()).c_str());
					ret = ssdb->qpop_front(name, &tmp, log_type);
				}
				if(ret == -1){
					return -1;
				}
			}
			break;
		default:
			log_error("unknown binlog, type=%d, cmd=%d", log.type(), log.cmd());
			break;
	}
	this->last_seq = log.seq();
	if(log.type() == BinlogType::COPY){
		this->last_key = log.key().String();
	}
	this->save_status();
	return 0;
}
Пример #18
0
int BackendSync::Client::sync(SyncLogQueue *logs){
	Buffer *output = link->output;

	uint64_t expect_seq = this->last_seq + 1;
	Synclog log;
	int ret;
	if(this->status == Client::DUMP && this->last_seq == 0){
		ret = logs->find_last(&log);
	}else{
		ret = logs->find(expect_seq, &log);
	}
	if(ret == 0){
		return 0;
	}
	/*
	log_trace("fd: %d, seq: %llu, key: %s",
		link->fd(),
		log.seq(),
		hexmem(log.key().data(), log.key().size()).c_str());
	*/
	
	// writes that are out of dumped range will be discarded.
	if(this->iter && log.key() > this->last_key){
		// update last_seq
		this->last_seq = log.seq();

		log_trace("fd: %d, seq: %llu, drop: %s, last_key: %s",
			link->fd(),
			log.seq(),
			hexmem(log.key().data(), log.key().size()).c_str(),
			hexmem(this->last_key.data(), this->last_key.size()).c_str());
		return 1;
	}

	if(this->last_seq != 0 && log.seq() != expect_seq){
		log_warn("fd: %d, OUT_OF_SYNC! seq: %llu, last_seq: %llu",
			link->fd(),
			log.seq(),
			expect_seq
			);
		this->status = Client::OUT_OF_SYNC;
		return 1;
	}
	
	// update last_seq
	this->last_seq = log.seq();

	char seq_buf[20];
	snprintf(seq_buf, sizeof(seq_buf), "%llu", log.seq());
	char log_type = log.type();
		
	// a synclog from a mirror server will not be send to another mirror server
	if(this->is_mirror && (log_type == Synclog::MIRROR_SET || log_type == Synclog::MIRROR_DEL)){
		if(this->last_seq - this->last_noop_seq >= logs->total/2){
			this->last_noop_seq = this->last_seq;
				
			log_trace("fd: %d, sync noop %llu",
				link->fd(),
				log.seq());

			output->append_record("noop");
			output->append_record(seq_buf);
			output->append('\n');
		}
	}else if(log_type == Synclog::SET || log_type == Synclog::MIRROR_SET){
		std::string val;
		int ret = backend->ssdb->raw_get(log.key(), &val);
		if(ret == -1){
			log_error("raw_get error!");
		}else if(ret == 0){
			log_trace("skip not found: %s", hexmem(log.key().data(), log.key().size()).c_str());
			// not found, ignore
		}else{
			log_trace("fd: %d, sync_set %llu %s",
				link->fd(),
				log.seq(),
				hexmem(log.key().data(), log.key().size()).c_str());

			output->append_record("sync_set");
			output->append_record(seq_buf);
			output->append_record(log.key());
			output->append_record(val);
			output->append('\n');
		}
	}else if(log_type == Synclog::DEL || log_type == Synclog::MIRROR_DEL){
		log_trace("fd: %d, sync_del %llu %s",
			link->fd(),
			log.seq(),
			hexmem(log.key().data(), log.key().size()).c_str());

		output->append_record("sync_del");
		output->append_record(seq_buf);
		output->append_record(log.key());
		output->append('\n');
	}else{
		log_error("unknown sync log type: %d", log.type());
	}
	return 1;
}
Пример #19
0
int Slave::proc_sync(const Binlog &log, const std::vector<Bytes> &req){
	switch(log.cmd()){
		case BinlogCommand::KSET:
			{
				if(req.size() != 2){
					break;
				}
				std::string key;
				if(decode_kv_key(log.key(), &key) == -1){
					break;
				}
				log_trace("set %s", hexmem(key.data(), key.size()).c_str());
				if(ssdb->set(key, req[1], log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::KDEL:
			{
				std::string key;
				if(decode_kv_key(log.key(), &key) == -1){
					break;
				}
				log_trace("del %s", hexmem(key.data(), key.size()).c_str());
				if(ssdb->del(key, log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::HSET:
			{
				if(req.size() != 2){
					break;
				}
				std::string name, key;
				if(decode_hash_key(log.key(), &name, &key) == -1){
					break;
				}
				log_trace("hset %s %s",
					hexmem(name.data(), name.size()).c_str(),
					hexmem(key.data(), key.size()).c_str());
				if(ssdb->hset(name, key, req[1], log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::HDEL:
			{
				std::string name, key;
				if(decode_hash_key(log.key(), &name, &key) == -1){
					break;
				}
				log_trace("hdel %s %s",
					hexmem(name.data(), name.size()).c_str(),
					hexmem(key.data(), key.size()).c_str());
				if(ssdb->hdel(name, key, log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::ZSET:
			{
				if(req.size() != 2){
					break;
				}
				std::string name, key;
				if(decode_zset_key(log.key(), &name, &key) == -1){
					break;
				}
				log_trace("zset %s %s",
					hexmem(name.data(), name.size()).c_str(),
					hexmem(key.data(), key.size()).c_str());
				if(ssdb->zset(name, key, req[1], log_type) == -1){
					return -1;
				}
			}
			break;
		case BinlogCommand::ZDEL:
			{
				std::string name, key;
				if(decode_zset_key(log.key(), &name, &key) == -1){
					break;
				}
				log_trace("zdel %s %s",
					hexmem(name.data(), name.size()).c_str(),
					hexmem(key.data(), key.size()).c_str());
				if(ssdb->zdel(name, key, log_type) == -1){
					return -1;
				}
			}
			break;
		default:
			log_error("unknown binlog, type=%d, cmd=%d", log.type(), log.cmd());
			break;
	}
	this->last_seq = log.seq();
	if(log.type() == BinlogType::COPY){
		this->last_key = log.key().String();
	}
	this->save_status();
	return 0;
}
Пример #20
0
std::string Binlog::dumps() const{
	std::string str;
	if(buf.size() < HEADER_LEN){
		return str;
	}
	char buf[64];
	snprintf(buf, sizeof(buf), "%" PRIu64 " ", this->seq());
	str.append(buf);

	switch(this->type()){
		case BinlogType::NOOP:
			str.append("noop ");
			break;
		case BinlogType::SYNC:
			str.append("sync ");
			break;
		case BinlogType::MIRROR:
			str.append("mirror ");
			break;
		case BinlogType::COPY:
			str.append("copy ");
			break;
	}
	switch(this->cmd()){
		case BinlogCommand::NONE:
			str.append("none ");
			break;
		case BinlogCommand::KSET:
			str.append("set ");
			break;
		case BinlogCommand::KDEL:
			str.append("del ");
			break;
		case BinlogCommand::HSET:
			str.append("hset ");
			break;
		case BinlogCommand::HDEL:
			str.append("hdel ");
			break;
		case BinlogCommand::ZSET:
			str.append("zset ");
			break;
		case BinlogCommand::ZDEL:
			str.append("zdel ");
			break;
		case BinlogCommand::BEGIN:
			str.append("begin ");
			break;
		case BinlogCommand::END:
			str.append("end ");
			break;
		case BinlogCommand::QPUSH_BACK:
			str.append("qpush_back ");
			break;
		case BinlogCommand::QPUSH_FRONT:
			str.append("qpush_front ");
			break;
		case BinlogCommand::QPOP_BACK:
			str.append("qpop_back ");
			break;
		case BinlogCommand::QPOP_FRONT:
			str.append("qpop_front ");
		case BinlogCommand::QSET:
			str.append("qset ");
			break;
	}
	Bytes b = this->key();
	str.append(hexmem(b.data(), b.size()));

    snprintf(buf, sizeof(buf), "%ld ", this->ttl());
    str.append(buf);

	return str;
}
Пример #21
0
int main(int argc, char *argv[]) {
        _cleanup_bus_message_unref_ sd_bus_message *m = NULL;
        int r, boolean;
        const char *x, *y, *z, *a, *b, *c, *d;
        uint8_t u, v;
        void *buffer = NULL;
        size_t sz;
        char *h;
        const int32_t integer_array[] = { -1, -2, 0, 1, 2 }, *return_array;

        r = sd_bus_message_new_method_call(NULL, "foobar.waldo", "/", "foobar.waldo", "Piep", &m);
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "s", "a string");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "s", NULL);
        assert_se(r < 0);

        r = sd_bus_message_append(m, "as", 2, "string #1", "string #2");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "sass", "foobar", 5, "foo", "bar", "waldo", "piep", "pap", "after");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "a{yv}", 2, 3, "s", "foo", 5, "s", "waldo");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "ba(ss)", 255, 3, "aaa", "1", "bbb", "2", "ccc", "3");
        assert_se(r >= 0);

        r = sd_bus_message_open_container(m, 'a', "s");
        assert_se(r >= 0);

        r = sd_bus_message_append_basic(m, 's', "foobar");
        assert_se(r >= 0);

        r = sd_bus_message_append_basic(m, 's', "waldo");
        assert_se(r >= 0);

        r = sd_bus_message_close_container(m);
        assert_se(r >= 0);

        r = sd_bus_message_append_array(m, 'i', integer_array, sizeof(integer_array));
        assert_se(r >= 0);

        r = bus_message_seal(m, 4711);
        assert_se(r >= 0);

        bus_message_dump(m);

        r = bus_message_get_blob(m, &buffer, &sz);
        assert_se(r >= 0);

        h = hexmem(buffer, sz);
        assert_se(h);

        log_info("message size = %lu, contents =\n%s", (unsigned long) sz, h);
        free(h);

#ifdef HAVE_GLIB
        {
                GDBusMessage *g;
                char *p;

#if !defined(GLIB_VERSION_2_36)
                g_type_init();
#endif

                g = g_dbus_message_new_from_blob(buffer, sz, 0, NULL);
                p = g_dbus_message_print(g, 0);
                log_info("%s", p);
                g_free(p);
                g_object_unref(g);
        }
#endif

        {
                DBusMessage *w;
                DBusError error;

                dbus_error_init(&error);

                w = dbus_message_demarshal(buffer, sz, &error);
                if (!w) {
                        log_error("%s", error.message);
                } else
                        dbus_message_unref(w);
        }

        m = sd_bus_message_unref(m);

        r = bus_message_from_malloc(buffer, sz, NULL, 0, NULL, NULL, &m);
        assert_se(r >= 0);

        bus_message_dump(m);

        assert_se(sd_bus_message_rewind(m, true) >= 0);

        r = sd_bus_message_read(m, "sas", &x, 2, &y, &z);
        assert_se(r > 0);
        assert_se(streq(x, "a string"));
        assert_se(streq(y, "string #1"));
        assert_se(streq(z, "string #2"));

        r = sd_bus_message_read(m, "sass", &x, 5, &y, &z, &a, &b, &c, &d);
        assert_se(r > 0);
        assert_se(streq(x, "foobar"));
        assert_se(streq(y, "foo"));
        assert_se(streq(z, "bar"));
        assert_se(streq(a, "waldo"));
        assert_se(streq(b, "piep"));
        assert_se(streq(c, "pap"));
        assert_se(streq(d, "after"));

        r = sd_bus_message_read(m, "a{yv}", 2, &u, "s", &x, &v, "s", &y);
        assert_se(r > 0);
        assert_se(u == 3);
        assert_se(streq(x, "foo"));
        assert_se(v == 5);
        assert_se(streq(y, "waldo"));

        r = sd_bus_message_read(m, "ba(ss)", &boolean, 3, &x, &y, &a, &b, &c, &d);
        assert_se(r > 0);
        assert_se(boolean);
        assert_se(streq(x, "aaa"));
        assert_se(streq(y, "1"));
        assert_se(streq(a, "bbb"));
        assert_se(streq(b, "2"));
        assert_se(streq(c, "ccc"));
        assert_se(streq(d, "3"));

        r = sd_bus_message_read(m, "as", 2, &x, &y);
        assert_se(r > 0);
        assert_se(streq(x, "foobar"));
        assert_se(streq(y, "waldo"));

        r = sd_bus_message_read_array(m, 'i', (const void**) &return_array, &sz);
        assert_se(r > 0);
        assert_se(sz == sizeof(integer_array));
        assert_se(memcmp(integer_array, return_array, sz) == 0);

        r = sd_bus_message_peek_type(m, NULL, NULL);
        assert_se(r == 0);

        return 0;
}
Пример #22
0
void MyReplication::Delete(uint64_t seq, const leveldb::Slice& key){
	Synclog log(seq, Synclog::DEL, key);
	log_trace("%llu, del %s", seq, hexmem(key.data(), key.size()).c_str());
	logs->put(log);
}
Пример #23
0
int main(int argc, char *argv[]) {
        _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL, *copy = NULL;
        int r, boolean;
        const char *x, *x2, *y, *z, *a, *b, *c, *d, *a_signature;
        uint8_t u, v;
        void *buffer = NULL;
        size_t sz;
        char *h;
        const int32_t integer_array[] = { -1, -2, 0, 1, 2 }, *return_array;
        char *s;
        _cleanup_free_ char *first = NULL, *second = NULL, *third = NULL;
        _cleanup_fclose_ FILE *ms = NULL;
        size_t first_size = 0, second_size = 0, third_size = 0;
        _cleanup_(sd_bus_unrefp) sd_bus *bus = NULL;
        double dbl;
        uint64_t u64;

        r = sd_bus_default_system(&bus);
        if (r < 0)
                return EXIT_TEST_SKIP;

        r = sd_bus_message_new_method_call(bus, &m, "foobar.waldo", "/", "foobar.waldo", "Piep");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "s", "a string");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "s", NULL);
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "asg", 2, "string #1", "string #2", "sba(tt)ss");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "sass", "foobar", 5, "foo", "bar", "waldo", "piep", "pap", "after");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "a{yv}", 2, 3, "s", "foo", 5, "s", "waldo");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "y(ty)y(yt)y", 8, 777ULL, 7, 9, 77, 7777ULL, 10);
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "()");
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "ba(ss)", 255, 3, "aaa", "1", "bbb", "2", "ccc", "3");
        assert_se(r >= 0);

        r = sd_bus_message_open_container(m, 'a', "s");
        assert_se(r >= 0);

        r = sd_bus_message_append_basic(m, 's', "foobar");
        assert_se(r >= 0);

        r = sd_bus_message_append_basic(m, 's', "waldo");
        assert_se(r >= 0);

        r = sd_bus_message_close_container(m);
        assert_se(r >= 0);

        r = sd_bus_message_append_string_space(m, 5, &s);
        assert_se(r >= 0);
        strcpy(s, "hallo");

        r = sd_bus_message_append_array(m, 'i', integer_array, sizeof(integer_array));
        assert_se(r >= 0);

        r = sd_bus_message_append_array(m, 'u', NULL, 0);
        assert_se(r >= 0);

        r = sd_bus_message_append(m, "a(stdo)", 1, "foo", 815ULL, 47.0, "/");
        assert_se(r >= 0);

        r = bus_message_seal(m, 4711, 0);
        assert_se(r >= 0);

        bus_message_dump(m, stdout, BUS_MESSAGE_DUMP_WITH_HEADER);

        ms = open_memstream(&first, &first_size);
        bus_message_dump(m, ms, 0);
        fflush(ms);
        assert_se(!ferror(ms));

        r = bus_message_get_blob(m, &buffer, &sz);
        assert_se(r >= 0);

        h = hexmem(buffer, sz);
        assert_se(h);

        log_info("message size = %zu, contents =\n%s", sz, h);
        free(h);

#ifdef HAVE_GLIB
        {
                GDBusMessage *g;
                char *p;

#if !defined(GLIB_VERSION_2_36)
                g_type_init();
#endif

                g = g_dbus_message_new_from_blob(buffer, sz, 0, NULL);
                p = g_dbus_message_print(g, 0);
                log_info("%s", p);
                g_free(p);
                g_object_unref(g);
        }
#endif

#ifdef HAVE_DBUS
        {
                DBusMessage *w;
                DBusError error;

                dbus_error_init(&error);

                w = dbus_message_demarshal(buffer, sz, &error);
                if (!w)
                        log_error("%s", error.message);
                else
                        dbus_message_unref(w);

                dbus_error_free(&error);
        }
#endif

        m = sd_bus_message_unref(m);

        r = bus_message_from_malloc(bus, buffer, sz, NULL, 0, NULL, &m);
        assert_se(r >= 0);

        bus_message_dump(m, stdout, BUS_MESSAGE_DUMP_WITH_HEADER);

        fclose(ms);
        ms = open_memstream(&second, &second_size);
        bus_message_dump(m, ms, 0);
        fflush(ms);
        assert_se(!ferror(ms));
        assert_se(first_size == second_size);
        assert_se(memcmp(first, second, first_size) == 0);

        assert_se(sd_bus_message_rewind(m, true) >= 0);

        r = sd_bus_message_read(m, "ssasg", &x, &x2, 2, &y, &z, &a_signature);
        assert_se(r > 0);
        assert_se(streq(x, "a string"));
        assert_se(streq(x2, ""));
        assert_se(streq(y, "string #1"));
        assert_se(streq(z, "string #2"));
        assert_se(streq(a_signature, "sba(tt)ss"));

        r = sd_bus_message_read(m, "sass", &x, 5, &y, &z, &a, &b, &c, &d);
        assert_se(r > 0);
        assert_se(streq(x, "foobar"));
        assert_se(streq(y, "foo"));
        assert_se(streq(z, "bar"));
        assert_se(streq(a, "waldo"));
        assert_se(streq(b, "piep"));
        assert_se(streq(c, "pap"));
        assert_se(streq(d, "after"));

        r = sd_bus_message_read(m, "a{yv}", 2, &u, "s", &x, &v, "s", &y);
        assert_se(r > 0);
        assert_se(u == 3);
        assert_se(streq(x, "foo"));
        assert_se(v == 5);
        assert_se(streq(y, "waldo"));

        r = sd_bus_message_read(m, "y(ty)", &v, &u64, &u);
        assert_se(r > 0);
        assert_se(v == 8);
        assert_se(u64 == 777);
        assert_se(u == 7);

        r = sd_bus_message_read(m, "y(yt)", &v, &u, &u64);
        assert_se(r > 0);
        assert_se(v == 9);
        assert_se(u == 77);
        assert_se(u64 == 7777);

        r = sd_bus_message_read(m, "y", &v);
        assert_se(r > 0);
        assert_se(v == 10);

        r = sd_bus_message_read(m, "()");
        assert_se(r > 0);

        r = sd_bus_message_read(m, "ba(ss)", &boolean, 3, &x, &y, &a, &b, &c, &d);
        assert_se(r > 0);
        assert_se(boolean);
        assert_se(streq(x, "aaa"));
        assert_se(streq(y, "1"));
        assert_se(streq(a, "bbb"));
        assert_se(streq(b, "2"));
        assert_se(streq(c, "ccc"));
        assert_se(streq(d, "3"));

        assert_se(sd_bus_message_verify_type(m, 'a', "s") > 0);

        r = sd_bus_message_read(m, "as", 2, &x, &y);
        assert_se(r > 0);
        assert_se(streq(x, "foobar"));
        assert_se(streq(y, "waldo"));

        r = sd_bus_message_read_basic(m, 's', &s);
        assert_se(r > 0);
        assert_se(streq(s, "hallo"));

        r = sd_bus_message_read_array(m, 'i', (const void**) &return_array, &sz);
        assert_se(r > 0);
        assert_se(sz == sizeof(integer_array));
        assert_se(memcmp(integer_array, return_array, sz) == 0);

        r = sd_bus_message_read_array(m, 'u', (const void**) &return_array, &sz);
        assert_se(r > 0);
        assert_se(sz == 0);

        r = sd_bus_message_read(m, "a(stdo)", 1, &x, &u64, &dbl, &y);
        assert_se(r > 0);
        assert_se(streq(x, "foo"));
        assert_se(u64 == 815ULL);
        assert_se(fabs(dbl - 47.0) < 0.1);
        assert_se(streq(y, "/"));

        r = sd_bus_message_peek_type(m, NULL, NULL);
        assert_se(r == 0);

        r = sd_bus_message_new_method_call(bus, &copy, "foobar.waldo", "/", "foobar.waldo", "Piep");
        assert_se(r >= 0);

        r = sd_bus_message_rewind(m, true);
        assert_se(r >= 0);

        r = sd_bus_message_copy(copy, m, true);
        assert_se(r >= 0);

        r = bus_message_seal(copy, 4712, 0);
        assert_se(r >= 0);

        fclose(ms);
        ms = open_memstream(&third, &third_size);
        bus_message_dump(copy, ms, 0);
        fflush(ms);
        assert_se(!ferror(ms));

        printf("<%.*s>\n", (int) first_size, first);
        printf("<%.*s>\n", (int) third_size, third);

        assert_se(first_size == third_size);
        assert_se(memcmp(first, third, third_size) == 0);

        r = sd_bus_message_rewind(m, true);
        assert_se(r >= 0);

        assert_se(sd_bus_message_verify_type(m, 's', NULL) > 0);

        r = sd_bus_message_skip(m, "ssasg");
        assert_se(r > 0);

        assert_se(sd_bus_message_verify_type(m, 's', NULL) > 0);

        r = sd_bus_message_skip(m, "sass");
        assert_se(r >= 0);

        assert_se(sd_bus_message_verify_type(m, 'a', "{yv}") > 0);

        r = sd_bus_message_skip(m, "a{yv}y(ty)y(yt)y()");
        assert_se(r >= 0);

        assert_se(sd_bus_message_verify_type(m, 'b', NULL) > 0);

        r = sd_bus_message_read(m, "b", &boolean);
        assert_se(r > 0);
        assert_se(boolean);

        r = sd_bus_message_enter_container(m, 0, NULL);
        assert_se(r > 0);

        r = sd_bus_message_read(m, "(ss)", &x, &y);
        assert_se(r > 0);

        r = sd_bus_message_read(m, "(ss)", &a, &b);
        assert_se(r > 0);

        r = sd_bus_message_read(m, "(ss)", &c, &d);
        assert_se(r > 0);

        r = sd_bus_message_read(m, "(ss)", &x, &y);
        assert_se(r == 0);

        r = sd_bus_message_exit_container(m);
        assert_se(r >= 0);

        assert_se(streq(x, "aaa"));
        assert_se(streq(y, "1"));
        assert_se(streq(a, "bbb"));
        assert_se(streq(b, "2"));
        assert_se(streq(c, "ccc"));
        assert_se(streq(d, "3"));

        test_bus_label_escape();
        test_bus_path_encode();
        test_bus_path_encode_unique();
        test_bus_path_encode_many();

        return 0;
}
Пример #24
0
int BackendSync::Client::copy() {
    if(this->iter == NULL) {
        log_info("new iterator, last_key: '%s'", hexmem(last_key.data(), last_key.size()).c_str());
        std::string key = this->last_key;
        if(this->last_key.empty()) {
            key.push_back(DataType::MIN_PREFIX);
        }
        this->iter = backend->ssdb->iterator(key, "", -1);
        log_info("iterator created, last_key: '%s'", hexmem(last_key.data(), last_key.size()).c_str());
    }
    int ret = 0;
    int iterate_count = 0;
    int64_t stime = time_ms();
    while(true) {
        // Prevent copy() from blocking too long
        if(++iterate_count > 1000 || link->output->size() > 2 * 1024 * 1024) {
            break;
        }

        if(!iter->next()) {
            goto copy_end;
        }
        Bytes key = iter->key();
        if(key.size() == 0) {
            continue;
        }
        // finish copying all valid data types
        if(key.data()[0] > DataType::MAX_PREFIX) {
            goto copy_end;
        }
        Bytes val = iter->val();
        this->last_key = key.String();

        char cmd = 0;
        char data_type = key.data()[0];
        if(data_type == DataType::KV) {
            cmd = BinlogCommand::KSET;
        } else if(data_type == DataType::HASH) {
            cmd = BinlogCommand::HSET;
        } else if(data_type == DataType::ZSET) {
            cmd = BinlogCommand::ZSET;
        } else if(data_type == DataType::QUEUE) {
            cmd = BinlogCommand::QPUSH_BACK;
        } else {
            continue;
        }
        ret++;

        Binlog log(this->last_seq, BinlogType::COPY, cmd, slice(key));
        log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
        link->send(log.repr(), val);

        if(time_ms() - stime > 3000) {
            log_info("copy blocks too long, flush");
            break;
        }
    }
    return ret;

copy_end:
    log_info("%s:%d fd: %d, copy end", link->remote_ip, link->remote_port, link->fd());
    this->status = Client::SYNC;
    delete this->iter;
    this->iter = NULL;

    Binlog log(this->last_seq, BinlogType::COPY, BinlogCommand::END, "");
    log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
    link->send(log.repr(), "copy_end");
    return 1;
}
Пример #25
0
int BackendSync::Client::sync(BinlogQueue *logs){
	Binlog log;
	while(1){
		int ret = 0;
		uint64_t expect_seq = this->last_seq + 1;
		if(this->status == Client::COPY && this->last_seq == 0){
			ret = logs->find_last(&log);
		}else{
			ret = logs->find_next(expect_seq, &log);
		}
		if(ret == 0){
			return 0;
		}
		// writes that are out of copied range will be discarded.
		if(this->status == Client::COPY && log.key() > this->last_key){
			log_trace("fd: %d, last_key: '%s', drop: %s",
				link->fd(),
				hexmem(this->last_key.data(), this->last_key.size()).c_str(),
				log.dumps().c_str());
			this->last_seq = log.seq();
			//if(this->iter){
			//	delete this->iter;
			//	this->iter = NULL;
			//}
			continue;
		}
		if(this->last_seq != 0 && log.seq() != expect_seq){
			log_warn("fd: %d, OUT_OF_SYNC! log.seq: %" PRIu64", expect_seq: %" PRIu64"",
				link->fd(),
				log.seq(),
				expect_seq
				);
			this->status = Client::OUT_OF_SYNC;
			return 1;
		}

		// update last_seq
		this->last_seq = log.seq();

		char type = log.type();
		if(type == BinlogType::MIRROR && this->is_mirror){
			if(this->last_seq - this->last_noop_seq >= 1000){
				this->noop();
				return 1;
			}else{
				continue;
			}
		}

		break;
	}

	int ret = 0;
	std::string val;
	switch(log.cmd()){
		case BinlogCommand::KSET:
		case BinlogCommand::HSET:
		case BinlogCommand::ZSET:
			ret = backend->ssdb->raw_get(log.key(), &val);
			if(ret == -1){
				log_error("fd: %d, raw_get error!", link->fd());
			}else if(ret == 0){
				//log_debug("%s", hexmem(log.key().data(), log.key().size()).c_str());
				log_trace("fd: %d, skip not found: %s", link->fd(), log.dumps().c_str());
			}else{
				log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
				link->send(log.repr(), val);
			}
			break;
		case BinlogCommand::KDEL:
		case BinlogCommand::HDEL:
		case BinlogCommand::ZDEL:
			log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
			link->send(log.repr());
			break;
	}
	return 1;
}
Пример #26
0
void MyReplication::Put(uint64_t seq, const leveldb::Slice& key, const leveldb::Slice& val){
	Synclog log(seq, Synclog::SET, key);
	log_trace("%llu, set %s", seq, hexmem(key.data(), key.size()).c_str());
	logs->put(log);
}
Пример #27
0
int BackendSync::Client::sync(BinlogQueue *logs) {
    Binlog log;
    while(1) {
        int ret = 0;
        uint64_t expect_seq = this->last_seq + 1;
        if(this->status == Client::COPY && this->last_seq == 0) {
            ret = logs->find_last(&log);
        } else {
            ret = logs->find_next(expect_seq, &log);
        }
        if(ret == 0) {
            return 0;
        }
        if(this->status == Client::COPY && log.key() > this->last_key) {
            log_debug("fd: %d, last_key: '%s', drop: %s",
                      link->fd(),
                      hexmem(this->last_key.data(), this->last_key.size()).c_str(),
                      log.dumps().c_str());
            this->last_seq = log.seq();
            // WARN: When there are writes behind last_key, we MUST create
            // a new iterator, because iterator will not know this key.
            // Because iterator ONLY iterates throught keys written before
            // iterator is created.
            if(this->iter) {
                delete this->iter;
                this->iter = NULL;
            }
            continue;
        }
        if(this->last_seq != 0 && log.seq() != expect_seq) {
            log_warn("%s:%d fd: %d, OUT_OF_SYNC! log.seq: %" PRIu64 ", expect_seq: %" PRIu64 "",
                     link->remote_ip, link->remote_port,
                     link->fd(),
                     log.seq(),
                     expect_seq
                    );
            this->status = Client::OUT_OF_SYNC;
            return 1;
        }

        // update last_seq
        this->last_seq = log.seq();

        char type = log.type();
        if(type == BinlogType::MIRROR && this->is_mirror) {
            if(this->last_seq - this->last_noop_seq >= 1000) {
                this->noop();
                return 1;
            } else {
                continue;
            }
        }

        break;
    }

    int ret = 0;
    std::string val;
    switch(log.cmd()) {
    case BinlogCommand::KSET:
    case BinlogCommand::HSET:
    case BinlogCommand::ZSET:
    case BinlogCommand::QSET:
    case BinlogCommand::QPUSH_BACK:
    case BinlogCommand::QPUSH_FRONT:
        ret = backend->ssdb->raw_get(log.key(), &val);
        if(ret == -1) {
            log_error("fd: %d, raw_get error!", link->fd());
        } else if(ret == 0) {
            //log_debug("%s", hexmem(log.key().data(), log.key().size()).c_str());
            log_trace("fd: %d, skip not found: %s", link->fd(), log.dumps().c_str());
        } else {
            log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
            link->send(log.repr(), val);
        }
        break;
    case BinlogCommand::KDEL:
    case BinlogCommand::HDEL:
    case BinlogCommand::ZDEL:
    case BinlogCommand::QPOP_BACK:
    case BinlogCommand::QPOP_FRONT:
        log_trace("fd: %d, %s", link->fd(), log.dumps().c_str());
        link->send(log.repr());
        break;
    }
    return 1;
}
Пример #28
0
int proc_info(NetworkServer *net, Link *link, const Request &req, Response *resp){
	SSDBServer *serv = (SSDBServer *)net->data;
	resp->push_back("ok");
	resp->push_back("ssdb-server");
	resp->push_back("version");
	resp->push_back(SSDB_VERSION);
	{
		resp->push_back("links");
		resp->add(net->link_count);
	}
	{
		int64_t calls = 0;
		proc_map_t::iterator it;
		for(it=net->proc_map.begin(); it!=net->proc_map.end(); it++){
			Command *cmd = it->second;
			calls += cmd->calls;
		}
		resp->push_back("total_calls");
		resp->add(calls);
	}
	
	{
		uint64_t size = serv->ssdb->size();
		resp->push_back("dbsize");
		resp->push_back(str(size));
	}

	{
		std::string s = serv->ssdb->binlogs->stats();
		resp->push_back("binlogs");
		resp->push_back(s);
	}
	{
		std::vector<std::string> syncs = serv->backend_sync->stats();
		std::vector<std::string>::iterator it;
		for(it = syncs.begin(); it != syncs.end(); it++){
			std::string s = *it;
			resp->push_back("replication");
			resp->push_back(s);
		}
	}
	{
		std::vector<Slave *>::iterator it;
		for(it = serv->slaves.begin(); it != serv->slaves.end(); it++){
			Slave *slave = *it;
			std::string s = slave->stats();
			resp->push_back("replication");
			resp->push_back(s);
		}
	}
	{
		std::string val;
		std::string s, e;
		serv->get_kv_range(&s, &e);
		char buf[512];
		{
			snprintf(buf, sizeof(buf), "    kv  : \"%s\" - \"%s\"",
				str_escape(s).c_str(),
				str_escape(e).c_str()
				);
			val.append(buf);
		}
		{
			snprintf(buf, sizeof(buf), "\n    hash: \"\" - \"\"");
			val.append(buf);
		}
		{
			snprintf(buf, sizeof(buf), "\n    zset: \"\" - \"\"");
			val.append(buf);
		}
		{
			snprintf(buf, sizeof(buf), "\n    list: \"\" - \"\"");
			val.append(buf);
		}
		resp->push_back("serv_key_range");
		resp->push_back(val);
	}

	if(req.size() == 1 || req[1] == "range"){
		std::string val;
		std::vector<std::string> tmp;
		int ret = serv->ssdb->key_range(&tmp);
		if(ret == 0){
			char buf[512];
			
			snprintf(buf, sizeof(buf), "    kv  : \"%s\" - \"%s\"",
				hexmem(tmp[0].data(), tmp[0].size()).c_str(),
				hexmem(tmp[1].data(), tmp[1].size()).c_str()
				);
			val.append(buf);
			
			snprintf(buf, sizeof(buf), "\n    hash: \"%s\" - \"%s\"",
				hexmem(tmp[2].data(), tmp[2].size()).c_str(),
				hexmem(tmp[3].data(), tmp[3].size()).c_str()
				);
			val.append(buf);
			
			snprintf(buf, sizeof(buf), "\n    zset: \"%s\" - \"%s\"",
				hexmem(tmp[4].data(), tmp[4].size()).c_str(),
				hexmem(tmp[5].data(), tmp[5].size()).c_str()
				);
			val.append(buf);
			
			snprintf(buf, sizeof(buf), "\n    list: \"%s\" - \"%s\"",
				hexmem(tmp[6].data(), tmp[6].size()).c_str(),
				hexmem(tmp[7].data(), tmp[7].size()).c_str()
				);
			val.append(buf);
		}
		resp->push_back("data_key_range");
		resp->push_back(val);
	}

	if(req.size() == 1 || req[1] == "leveldb"){
		std::vector<std::string> tmp = serv->ssdb->info();
		for(int i=0; i<(int)tmp.size(); i++){
			std::string block = tmp[i];
			resp->push_back(block);
		}
	}

	if(req.size() > 1 && req[1] == "cmd"){
		proc_map_t::iterator it;
		for(it=net->proc_map.begin(); it!=net->proc_map.end(); it++){
			Command *cmd = it->second;
			resp->push_back("cmd." + cmd->name);
			char buf[128];
			snprintf(buf, sizeof(buf), "calls: %" PRIu64 "\ttime_wait: %.0f\ttime_proc: %.0f",
				cmd->calls, cmd->time_wait, cmd->time_proc);
			resp->push_back(buf);
		}
	}
	
	return 0;
}