// TESTING, slow, so not used void BinlogQueue::merge(){ std::map<std::string, uint64_t> key_map; uint64_t start = min_seq; uint64_t end = last_seq; int reduce_count = 0; int total = 0; total = end - start + 1; (void)total; // suppresses warning log_trace("merge begin"); for(; start <= end; start++){ Binlog log; if(this->get(start, &log) == 1){ if(log.type() == BinlogType::NOOP){ continue; } std::string key = log.key().String(); std::map<std::string, uint64_t>::iterator it = key_map.find(key); if(it != key_map.end()){ uint64_t seq = it->second; this->update(seq, BinlogType::NOOP, BinlogCommand::NONE, ""); //log_trace("merge update %" PRIu64 " to NOOP", seq); reduce_count ++; } key_map[key] = log.seq(); } } log_trace("merge reduce %d of %d binlogs", reduce_count, total); }
int BackendSync::Client::sync(BinlogQueue *logs) { Binlog log; while(1) { int ret = 0; uint64_t expect_seq = this->last_seq + 1; if(this->status == Client::COPY && this->last_seq == 0) { ret = logs->find_last(&log); } else { ret = logs->find_next(expect_seq, &log); } if(ret == 0) { return 0; } if(this->status == Client::COPY && log.key() > this->last_key) { log_debug("fd: %d, last_key: '%s', drop: %s", link->fd(), hexmem(this->last_key.data(), this->last_key.size()).c_str(), log.dumps().c_str()); this->last_seq = log.seq(); // WARN: When there are writes behind last_key, we MUST create // a new iterator, because iterator will not know this key. // Because iterator ONLY iterates throught keys written before // iterator is created. if(this->iter) { delete this->iter; this->iter = NULL; } continue; } if(this->last_seq != 0 && log.seq() != expect_seq) { log_warn("%s:%d fd: %d, OUT_OF_SYNC! log.seq: %" PRIu64 ", expect_seq: %" PRIu64 "", link->remote_ip, link->remote_port, link->fd(), log.seq(), expect_seq ); this->status = Client::OUT_OF_SYNC; return 1; } // update last_seq this->last_seq = log.seq(); char type = log.type(); if(type == BinlogType::MIRROR && this->is_mirror) { if(this->last_seq - this->last_noop_seq >= 1000) { this->noop(); return 1; } else { continue; } } break; } int ret = 0; std::string val; switch(log.cmd()) { case BinlogCommand::KSET: case BinlogCommand::HSET: case BinlogCommand::ZSET: case BinlogCommand::QSET: case BinlogCommand::QPUSH_BACK: case BinlogCommand::QPUSH_FRONT: ret = backend->ssdb->raw_get(log.key(), &val); if(ret == -1) { log_error("fd: %d, raw_get error!", link->fd()); } else if(ret == 0) { //log_debug("%s", hexmem(log.key().data(), log.key().size()).c_str()); log_trace("fd: %d, skip not found: %s", link->fd(), log.dumps().c_str()); } else { log_trace("fd: %d, %s", link->fd(), log.dumps().c_str()); link->send(log.repr(), val); } break; case BinlogCommand::KDEL: case BinlogCommand::HDEL: case BinlogCommand::ZDEL: case BinlogCommand::QPOP_BACK: case BinlogCommand::QPOP_FRONT: log_trace("fd: %d, %s", link->fd(), log.dumps().c_str()); link->send(log.repr()); break; } return 1; }
int Slave::proc_sync(const Binlog &log, const std::vector<Bytes> &req){ switch(log.cmd()){ case BinlogCommand::KSET: { if(req.size() != 2){ break; } std::string key; if(decode_kv_key(log.key(), &key) == -1){ break; } log_trace("set %s", hexmem(key.data(), key.size()).c_str()); if(ssdb->set(key, req[1], log_type) == -1){ return -1; } } break; case BinlogCommand::KDEL: { std::string key; if(decode_kv_key(log.key(), &key) == -1){ break; } log_trace("del %s", hexmem(key.data(), key.size()).c_str()); if(ssdb->del(key, log_type) == -1){ return -1; } } break; case BinlogCommand::HSET: { if(req.size() != 2){ break; } std::string name, key; if(decode_hash_key(log.key(), &name, &key) == -1){ break; } log_trace("hset %s %s", hexmem(name.data(), name.size()).c_str(), hexmem(key.data(), key.size()).c_str()); if(ssdb->hset(name, key, req[1], log_type) == -1){ return -1; } } break; case BinlogCommand::HDEL: { std::string name, key; if(decode_hash_key(log.key(), &name, &key) == -1){ break; } log_trace("hdel %s %s", hexmem(name.data(), name.size()).c_str(), hexmem(key.data(), key.size()).c_str()); if(ssdb->hdel(name, key, log_type) == -1){ return -1; } } break; case BinlogCommand::ZSET: { if(req.size() != 2){ break; } std::string name, key; if(decode_zset_key(log.key(), &name, &key) == -1){ break; } log_trace("zset %s %s", hexmem(name.data(), name.size()).c_str(), hexmem(key.data(), key.size()).c_str()); if(ssdb->zset(name, key, req[1], log_type) == -1){ return -1; } } break; case BinlogCommand::ZDEL: { std::string name, key; if(decode_zset_key(log.key(), &name, &key) == -1){ break; } log_trace("zdel %s %s", hexmem(name.data(), name.size()).c_str(), hexmem(key.data(), key.size()).c_str()); if(ssdb->zdel(name, key, log_type) == -1){ return -1; } } break; case BinlogCommand::QSET: case BinlogCommand::QPUSH_BACK: case BinlogCommand::QPUSH_FRONT: { if(req.size() != 2){ break; } std::string name; uint64_t seq; if(decode_qitem_key(log.key(), &name, &seq) == -1){ break; } if(seq < QITEM_MIN_SEQ || seq > QITEM_MAX_SEQ){ break; } int ret; if(log.cmd() == BinlogCommand::QSET){ log_trace("qset %s %" PRIu64 "", hexmem(name.data(), name.size()).c_str(), seq); ret = ssdb->qset_by_seq(name, seq, req[1], log_type); }else if(log.cmd() == BinlogCommand::QPUSH_BACK){ log_trace("qpush_back %s", hexmem(name.data(), name.size()).c_str()); ret = ssdb->qpush_back(name, req[1], log_type); }else{ log_trace("qpush_front %s", hexmem(name.data(), name.size()).c_str()); ret = ssdb->qpush_front(name, req[1], log_type); } if(ret == -1){ return -1; } } break; case BinlogCommand::QPOP_BACK: case BinlogCommand::QPOP_FRONT: { int ret; const Bytes name = log.key(); std::string tmp; if(log.cmd() == BinlogCommand::QPOP_BACK){ log_trace("qpop_back %s", hexmem(name.data(), name.size()).c_str()); ret = ssdb->qpop_back(name, &tmp, log_type); }else{ log_trace("qpop_front %s", hexmem(name.data(), name.size()).c_str()); ret = ssdb->qpop_front(name, &tmp, log_type); } if(ret == -1){ return -1; } } break; default: log_error("unknown binlog, type=%d, cmd=%d", log.type(), log.cmd()); break; } this->last_seq = log.seq(); if(log.type() == BinlogType::COPY){ this->last_key = log.key().String(); } this->save_status(); return 0; }
int Slave::proc_sync(const Binlog &log, const std::vector<Bytes> &req){ switch(log.cmd()){ case BinlogCommand::KSET: { if(req.size() != 2){ break; } std::string key; if(decode_kv_key(log.key(), &key) == -1){ break; } log_trace("set %s", hexmem(key.data(), key.size()).c_str()); if(ssdb->set(key, req[1], log_type) == -1){ return -1; } } break; case BinlogCommand::KDEL: { std::string key; if(decode_kv_key(log.key(), &key) == -1){ break; } log_trace("del %s", hexmem(key.data(), key.size()).c_str()); if(ssdb->del(key, log_type) == -1){ return -1; } } break; case BinlogCommand::HSET: { if(req.size() != 2){ break; } std::string name, key; if(decode_hash_key(log.key(), &name, &key) == -1){ break; } log_trace("hset %s %s", hexmem(name.data(), name.size()).c_str(), hexmem(key.data(), key.size()).c_str()); if(ssdb->hset(name, key, req[1], log_type) == -1){ return -1; } } break; case BinlogCommand::HDEL: { std::string name, key; if(decode_hash_key(log.key(), &name, &key) == -1){ break; } log_trace("hdel %s %s", hexmem(name.data(), name.size()).c_str(), hexmem(key.data(), key.size()).c_str()); if(ssdb->hdel(name, key, log_type) == -1){ return -1; } } break; case BinlogCommand::ZSET: { if(req.size() != 2){ break; } std::string name, key; if(decode_zset_key(log.key(), &name, &key) == -1){ break; } log_trace("zset %s %s", hexmem(name.data(), name.size()).c_str(), hexmem(key.data(), key.size()).c_str()); if(ssdb->zset(name, key, req[1], log_type) == -1){ return -1; } } break; case BinlogCommand::ZDEL: { std::string name, key; if(decode_zset_key(log.key(), &name, &key) == -1){ break; } log_trace("zdel %s %s", hexmem(name.data(), name.size()).c_str(), hexmem(key.data(), key.size()).c_str()); if(ssdb->zdel(name, key, log_type) == -1){ return -1; } } break; default: log_error("unknown binlog, type=%d, cmd=%d", log.type(), log.cmd()); break; } this->last_seq = log.seq(); if(log.type() == BinlogType::COPY){ this->last_key = log.key().String(); } this->save_status(); return 0; }
int BackendSync::Client::sync(BinlogQueue *logs){ Binlog log; while(1){ int ret = 0; uint64_t expect_seq = this->last_seq + 1; if(this->status == Client::COPY && this->last_seq == 0){ ret = logs->find_last(&log); }else{ ret = logs->find_next(expect_seq, &log); } if(ret == 0){ return 0; } // writes that are out of copied range will be discarded. if(this->status == Client::COPY && log.key() > this->last_key){ log_trace("fd: %d, last_key: '%s', drop: %s", link->fd(), hexmem(this->last_key.data(), this->last_key.size()).c_str(), log.dumps().c_str()); this->last_seq = log.seq(); //if(this->iter){ // delete this->iter; // this->iter = NULL; //} continue; } if(this->last_seq != 0 && log.seq() != expect_seq){ log_warn("fd: %d, OUT_OF_SYNC! log.seq: %" PRIu64", expect_seq: %" PRIu64"", link->fd(), log.seq(), expect_seq ); this->status = Client::OUT_OF_SYNC; return 1; } // update last_seq this->last_seq = log.seq(); char type = log.type(); if(type == BinlogType::MIRROR && this->is_mirror){ if(this->last_seq - this->last_noop_seq >= 1000){ this->noop(); return 1; }else{ continue; } } break; } int ret = 0; std::string val; switch(log.cmd()){ case BinlogCommand::KSET: case BinlogCommand::HSET: case BinlogCommand::ZSET: ret = backend->ssdb->raw_get(log.key(), &val); if(ret == -1){ log_error("fd: %d, raw_get error!", link->fd()); }else if(ret == 0){ //log_debug("%s", hexmem(log.key().data(), log.key().size()).c_str()); log_trace("fd: %d, skip not found: %s", link->fd(), log.dumps().c_str()); }else{ log_trace("fd: %d, %s", link->fd(), log.dumps().c_str()); link->send(log.repr(), val); } break; case BinlogCommand::KDEL: case BinlogCommand::HDEL: case BinlogCommand::ZDEL: log_trace("fd: %d, %s", link->fd(), log.dumps().c_str()); link->send(log.repr()); break; } return 1; }