Session* Server::accept_session(){
Link *link = serv_link->accept();
if(link == NULL){
log_error("accept failed! %s", strerror(errno));
return NULL;
}
link->nodelay();
link->noblock();
link->create_time = millitime();
link->active_time = link->create_time;
Session *sess = new Session();
sess->link = link;
this->sessions[sess->id] = sess;
for(int i=0; i<this->handlers.size(); i++){
Handler *handler = this->handlers[i];
HandlerState state = handler->accept(*sess);
if(state == HANDLE_FAIL){
delete link;
delete sess;
return NULL;
}
}
this->link_count ++;
log_debug("new link from %s:%d, fd: %d, links: %d",
link->remote_ip, link->remote_port, link->fd(), this->link_count);
fdes->set(link->fd(), FDEVENT_IN, DEFAULT_TYPE, sess);
return sess;
}
int NetworkServer::proc_result(ProcJob *job, ready_list_t *ready_list){
Link *link = job->link;
int result = job->result;
if(job->cmd){
job->cmd->calls += 1;
job->cmd->time_wait += job->time_wait;
job->cmd->time_proc += job->time_proc;
}
delete job;
if(result == PROC_ERROR){
link->mark_error();
ready_list->push_back(link);
}else{
if(link->output->empty()){
fdes->clr(link->fd(), FDEVENT_OUT);
if(link->input->empty()){
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}else{
ready_list->push_back(link);
}
}else{
fdes->clr(link->fd(), FDEVENT_IN);
fdes->set(link->fd(), FDEVENT_OUT, 1, link);
}
}
return result;
}
/*
event:
read => ready_list OR close
write => NONE
proc =>
done: write & (read OR ready_list)
async: stop (read & write)
1. When writing to a link, it may happen to be in the ready_list,
so we cannot close that link in write process, we could only
just mark it as closed.
2. When reading from a link, it is never in the ready_list, so it
is safe to close it in read process, also safe to put it into
ready_list.
3. Ignore FDEVENT_ERR
A link is in either one of these places:
1. ready list
2. async worker queue
So it safe to delete link when processing ready list and async worker result.
*/
int NetworkServer::proc_client_event(const Fdevent *fde, ready_list_t *ready_list){
Link *link = (Link *)fde->data.ptr;
if(fde->events & FDEVENT_IN){
ready_list->push_back(link);
if(link->error()){
return 0;
}
int len = link->read();
//log_debug("fd: %d read: %d", link->fd(), len);
if(len <= 0){
log_debug("fd: %d, read: %d, delete link", link->fd(), len);
link->mark_error();
return 0;
}
}
if(fde->events & FDEVENT_OUT){
if(link->error()){
return 0;
}
int len = link->write();
if(len <= 0){
log_debug("fd: %d, write: %d, delete link", link->fd(), len);
link->mark_error();
return 0;
}
if(link->output->empty()){
fdes->clr(link->fd(), FDEVENT_OUT);
}
}
return 0;
}
int Server::read_session(Session *sess){
Link *link = sess->link;
if(link->error()){
return 0;
}
int len = link->read();
if(len <= 0){
this->close_session(sess);
return -1;
}
while(1){
Request req;
int ret = link->recv(&req.msg);
if(ret == -1){
log_info("fd: %d, parse error, delete link", link->fd());
this->close_session(sess);
return -1;
}else if(ret == 0){
// 报文未就绪, 继续读网络
break;
}
req.stime = millitime();
req.sess = *sess;
Response resp;
for(int i=0; i<this->handlers.size(); i++){
Handler *handler = this->handlers[i];
req.time_wait = 1000 * (millitime() - req.stime);
HandlerState state = handler->proc(req, &resp);
req.time_proc = 1000 * (millitime() - req.stime) - req.time_wait;
if(state == HANDLE_RESP){
link->send(resp.msg);
if(link && !link->output.empty()){
fdes->set(link->fd(), FDEVENT_OUT, DEFAULT_TYPE, sess);
}
if(log_level() >= Logger::LEVEL_DEBUG){
log_debug("w:%.3f,p:%.3f, req: %s resp: %s",
req.time_wait, req.time_proc,
msg_str(req.msg).c_str(),
msg_str(resp.msg).c_str());
}
}else if(state == HANDLE_FAIL){
this->close_session(sess);
return -1;
}
}
}
return 0;
}
int Server::close_session(Session *sess){
Link *link = sess->link;
for(int i=0; i<this->handlers.size(); i++){
Handler *handler = this->handlers[i];
handler->close(*sess);
}
this->link_count --;
log_debug("delete link %s:%d, fd: %d, links: %d",
link->remote_ip, link->remote_port, link->fd(), this->link_count);
fdes->del(link->fd());
this->sessions.erase(sess->id);
delete link;
delete sess;
return 0;
}
int Server::write_session(Session *sess){
Link *link = sess->link;
if(link->error()){
return 0;
}
int len = link->write();
if(len <= 0){
log_debug("fd: %d, write: %d, delete link", link->fd(), len);
this->close_session(sess);
return -1;
}
if(link->output.empty()){
fdes->clr(link->fd(), FDEVENT_OUT);
}
return 0;
}
int Server::loop_once(){
const Fdevents::events_t *events;
events = fdes->wait(20);
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
return 0;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
this->accept_session();
}else if(fde->data.num == HANDLER_TYPE){
Handler *handler = (Handler *)fde->data.ptr;
while(Response *resp = handler->handle()){
Session *sess = this->get_session(resp->sess.id);
if(sess){
Link *link = sess->link;
link->send(resp->msg);
if(link && !link->output.empty()){
fdes->set(link->fd(), FDEVENT_OUT, DEFAULT_TYPE, sess);
}
}
delete resp;
}
}else{
Session *sess = (Session *)fde->data.ptr;
Link *link = sess->link;
if(fde->events & FDEVENT_IN){
if(this->read_session(sess) == -1){
continue;
}
}
if(fde->events & FDEVENT_OUT){
if(this->write_session(sess) == -1){
continue;
}
}
if(link && !link->output.empty()){
fdes->set(link->fd(), FDEVENT_OUT, DEFAULT_TYPE, sess);
}
}
}
return 0;
}
int NetworkServer::proc_client_event(const Fdevent *fde, ready_list_t *ready_list){
Link *link = (Link *)fde->data.ptr;
if(fde->events & FDEVENT_ERR){
ready_list->push_back(link);
log_warn("fd: %d, wait error, delete link", link->fd());
link->mark_error();
return 0;
}
if(fde->events & FDEVENT_IN){
ready_list->push_back(link);
if(link->error()){
return 0;
}
int len = link->read();
if(len <= 0){
if(link->readed < sizeof(head_t) || link->request.head.bodylen < MAX_PACKAGE_SIZE){
log_frame("fd: %d, read: %d, delete link [ readed: %d, head_size: %d, bodylen: %u, max: %d ]",
link->fd(), len, link->readed, sizeof(head_t), link->request.head.bodylen, MAX_PACKAGE_SIZE);
link->mark_error();
}
return 0;
}
}
if(fde->events & FDEVENT_OUT){
if(link->error()){
return 0;
}
int len = link->write();
if(len < 0){
log_warn("[%s] fd: %d, write: %d, delete link", link->get_basic_string(), link->fd(), len);
link->mark_error();
return 0;
}
if(link->response.head.bodylen + sizeof(head_t) <= link->writed){
log_frame("[%s] fd: %d, write: %d, finish writing", link->get_basic_string());
fdes->clr(link->fd(), FDEVENT_OUT);
}
if(!long_connect || 0 != link->response.head.status){
link_count--;
fdes->del(link->fd());
log_frame("[%s] short connection, close [ fd: %d ]", link->get_basic_string(), link->fd());
SAFE_DELETE(link);
}else{
link->readed = link->writed = 0;
link->request.reset();
link->response.reset();
link->reset_basic_string();
fdes->set(link->fd(), FDEVENT_IN, 1, link);
log_frame("[%s] long connection, try to receive request next time [ fd: %d ]", link->get_basic_string(), link->fd());
}
}
return 0;
}
void init_links(int num, const char *ip, int port){
fdes = new Fdevents();
free_links = new std::vector<Link *>();
for(int i=0; i<num; i++){
Link *link = Link::connect(ip, port);
if(!link){
fprintf(stderr, "connect error! %s\n", strerror(errno));
exit(0);
}
fdes->set(link->fd(), FDEVENT_IN, 0, link);
free_links->push_back(link);
}
}
/*
event:
read => ready_list OR close
write => ready_list
proc_result =>
done: write & (read OR ready_list)
async: stop (read & write)
1. When writing to a link, it may happen to be in the ready_list,
so we cannot close that link in write process, we could only
just mark it as closed.
2. When reading from a link, it is never in the ready_list, so it
is safe to close it in read process, also safe to put it into
ready_list.
3. Ignore FDEVENT_ERR
A link is in either one of these places:
1. ready list
2. async worker queue
3. fdes
So it safe to delete link when processing ready list and async worker result.
*/
int NetworkServer::proc_client_event(const Fdevent *fde, ready_list_t *ready_list){
Link *link = (Link *)fde->data.ptr;
if(fde->events & FDEVENT_IN){
int len = link->read();
//log_debug("fd: %d read: %d", link->fd(), len);
if(len <= 0){
double serv_time = millitime() - link->create_time;
log_debug("fd: %d, read: %d, delete link, s:%.3f", link->fd(), len, serv_time);
link->mark_error();
ready_list->push_back(link);
return 0;
}
ready_list->push_back(link);
}else if(fde->events & FDEVENT_OUT){
int len = link->write();
//log_debug("fd: %d, write: %d", link->fd(), len);
if(len <= 0){
log_debug("fd: %d, write: %d, delete link", link->fd(), len);
link->mark_error();
ready_list->push_back(link);
return 0;
}
if(link->output->empty()){
fdes->clr(link->fd(), FDEVENT_OUT);
if(link->input->empty()){
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}else{
ready_list->push_back(link);
}
}else{
fdes->clr(link->fd(), FDEVENT_IN);
fdes->set(link->fd(), FDEVENT_OUT, 1, link);
}
}
return 0;
}
int NetworkServer::proc_result(ProcJob *job, ready_list_t *ready_list){
Link *link = job->link;
int result = job->result;
log_debug("[%s] process finished, wait time:%.3f,proc time:%.3f, method: %s, status: %u",
job->link->get_basic_string(), job->time_wait, job->time_proc,
job->req->head.method,
job->link->response.head.status);
if(job->cmd){
job->cmd->calls += 1;
job->cmd->time_wait += job->time_wait;
job->cmd->time_proc += job->time_proc;
}
SAFE_DELETE(job);
int len = 0;
if(result == PROC_ERROR){
log_info("[%s] fd: %d, proc error, maybe network is unavailable, delete link", link->get_basic_string(), link->fd());
goto proc_err;
}
//try to write for one time
//if there is no writing buffer, 0 will be returned
len = link->write();
if(len < 0){
log_warn("[%s] fd: %d, write: %d, delete link", link->get_basic_string(), link->fd(), len);
goto proc_err;
}
if(link->response.head.bodylen + sizeof(head_t) > link->writed){
//need more writing operation
fdes->set(link->fd(), FDEVENT_OUT, 1, link);
}else{
if(!long_connect || 0 != link->response.head.status){
//TODO: for short connection
link_count--;
fdes->del(link->fd());
log_frame("[%s] short connection, close [ fd: %d ]", link->get_basic_string(), link->fd());
SAFE_DELETE(link);
}else{
//TODO: long connection£¬ continue to read data
link->readed = link->writed = 0;
link->request.reset();
link->response.reset();
link->reset_basic_string();
fdes->set(link->fd(), FDEVENT_IN, 1, link);
log_frame("[%s] long connection, try to receive request next time [ fd: %d ]", link->get_basic_string(), link->fd());
}
}
return PROC_OK;
proc_err:
this->link_count --;
fdes->del(link->fd());
SAFE_DELETE(link);
return PROC_ERROR;
}
int proc_result(ProcJob &job, Fdevents &select, ready_list_t &ready_list){
Link *link = job.link;
if(job.result == PROC_ERROR){
log_info("fd: %d, proc error, delete link", link->fd());
select.del(link->fd());
delete link;
return PROC_ERROR;
}
if(job.cmd){
job.cmd->calls += 1;
job.cmd->time_wait += job.time_wait;
job.cmd->time_proc += job.time_proc;
}
if(!link->output->empty()){
//log_trace("add %d to select.out", link->fd());
select.set(link->fd(), FDEVENT_OUT, 1, link);
if(select.isset(link->fd(), FDEVENT_IN)){
//log_trace("delete %d from select.in", link->fd());
select.clr(link->fd(), FDEVENT_IN);
}
}else{
if(link->input->empty()){
if(!select.isset(link->fd(), FDEVENT_IN)){
//log_trace("add %d to select.in", link->fd());
select.set(link->fd(), FDEVENT_IN, 1, link);
}
}else{
if(select.isset(link->fd(), FDEVENT_IN)){
//log_trace("delete %d from select.in", link->fd());
select.clr(link->fd(), FDEVENT_IN);
}
ready_list.push_back(link);
}
}
return PROC_OK;
}
int NetworkServer::proc_result(ProcJob *job, ready_list_t *ready_list){
Link *link = job->link;
int len;
if(job->cmd){
job->cmd->calls += 1;
job->cmd->time_wait += job->time_wait;
job->cmd->time_proc += job->time_proc;
double total_time = job->time_wait + job->time_proc;
if (total_time >= this->slow_time) {
slowlog_warn("proc slow, remote_ip: %s wait: %.3f proc: %.3f total: %.3f req: %s",
link->remote_ip, job->time_wait, job->time_proc, total_time,
serialize_req(*link->last_recv()).c_str());
}
}
if(job->result == PROC_ERROR){
log_info("fd: %d, proc error, delete link", link->fd());
goto proc_err;
}
if(link->error()){
log_debug("fd: %d, link error, delete link", link->fd());
goto proc_err;
}
len = link->write();
//log_debug("write: %d", len);
if(len < 0){
log_debug("fd: %d, write: %d, delete link", link->fd(), len);
goto proc_err;
}
this->bytes_written += len;
if(!link->output->empty()){
fdes->set(link->fd(), FDEVENT_OUT, 1, link);
}
if(link->input->empty()){
fdes->set(link->fd(), FDEVENT_IN, 1, link);
link->unRef();
log_debug("proc_result remote_ip: %s ref: %d", link->remote_ip, link->ref_count);
}else{
fdes->clr(link->fd(), FDEVENT_IN);
ready_list->push_back(link);
}
return PROC_OK;
proc_err:
destroy_link(link);
return PROC_ERROR;
}
int NetworkServer::proc_result(ProcJob *job, link_dict_t *ready_dict){
Link *link = job->link;
int len;
if(job->cmd){
total_calls ++;
job->cmd->calls ++;
job->cmd->time_wait += job->time_wait;
job->cmd->time_proc += job->time_proc;
}
if(job->result == PROC_ERROR){
log_info("fd: %d, proc error, delete link", link->fd());
goto proc_err;
}
len = link->write();
//log_debug("write: %d", len);
if(len < 0){
log_debug("fd: %d, write: %d, delete link", link->fd(), len);
goto proc_err;
}
if(!link->output->empty()){
fdes->set(link->fd(), FDEVENT_OUT, 1, link);
}
if(link->input->empty()){
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}else{
fdes->clr(link->fd(), FDEVENT_IN);
ready_dict->insert(std::make_pair(link->fd(), link));
}
return PROC_OK;
proc_err:
this->link_count --;
fdes->del(link->fd());
delete link;
return PROC_ERROR;
}
void* BackendDump::_run_thread(void *arg){
struct run_arg *p = (struct run_arg*)arg;
const BackendDump *backend = p->backend;
Link *link = (Link *)p->link;
delete p;
//
link->noblock(false);
const std::vector<Bytes>* req = link->last_recv();
std::string start = "";
if(req->size() > 1){
Bytes b = req->at(1);
start.assign(b.data(), b.size());
}
std::string end = "";
if(req->size() > 2){
Bytes b = req->at(2);
end.assign(b.data(), b.size());
}
int limit = 10;
if(req->size() > 3){
Bytes b = req->at(3);
limit = b.Int();
}
Buffer *output = link->output;
int count = 0;
bool quit = false;
Iterator *it = backend->ssdb->iterator(start, end, limit);
link->send("begin");
while(!quit){
if(!it->next()){
quit = true;
char buf[20];
snprintf(buf, sizeof(buf), "%d", count);
link->send("end", buf);
}else{
count ++;
Bytes key = it->key();
Bytes val = it->val();
output->append_record("set");
output->append_record(key);
output->append_record(val);
output->append('\n');
if(output->size() < output->total()/2){
continue;
}
}
if(link->flush() == -1){
log_info("fd: %d, send error", link->fd());
break;
}
}
log_info("fd: %d, delete link", link->fd());
delete link;
return (void *)NULL;
}
void NetworkServer::serve(){
writer = new ProcWorkerPool("writer");
writer->start(WRITER_THREADS);
reader = new ProcWorkerPool("reader");
reader->start(READER_THREADS);
ready_list_t ready_list;
ready_list_t ready_list_2;
ready_list_t::iterator it;
const Fdevents::events_t *events;
fdes->set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
fdes->set(this->reader->fd(), FDEVENT_IN, 0, this->reader);
fdes->set(this->writer->fd(), FDEVENT_IN, 0, this->writer);
uint32_t last_ticks = g_ticks;
while(!quit){
// status report
if((uint32_t)(g_ticks - last_ticks) >= STATUS_REPORT_TICKS){
last_ticks = g_ticks;
log_info("server running, links: %d", this->link_count);
}
ready_list.swap(ready_list_2);
ready_list_2.clear();
if(!ready_list.empty()){
// ready_list not empty, so we should return immediately
events = fdes->wait(0);
}else{
events = fdes->wait(50);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = accept_link();
if(link){
this->link_count ++;
log_debug("new link from %s:%d, fd: %d, links: %d",
link->remote_ip, link->remote_port, link->fd(), this->link_count);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}
}else if(fde->data.ptr == this->reader || fde->data.ptr == this->writer){
ProcWorkerPool *worker = (ProcWorkerPool *)fde->data.ptr;
ProcJob job;
if(worker->pop(&job) == 0){
log_fatal("reading result from workers error!");
exit(0);
}
if(proc_result(&job, &ready_list) == PROC_ERROR){
//
}
}else{
proc_client_event(fde, &ready_list);
}
}
for(it = ready_list.begin(); it != ready_list.end(); it ++){
Link *link = *it;
if(link->error()){
destroy_link(link);
continue;
}
const Request *req = link->recv();
if(req == NULL){
log_warn("fd: %d, link parse error, delete link", link->fd());
destroy_link(link);
continue;
}
if(req->empty()){
link->unRef();
fdes->set(link->fd(), FDEVENT_IN, 1, link);
log_debug("serve parse incomplete request, remote_ip: %s ref: %d", link->remote_ip, link->ref_count);
continue;
}
link->active_time = millitime();
//FIXME
char remote_ip_port[32];
snprintf(remote_ip_port, 32, "%s:%d", link->remote_ip, link->remote_port);
this->active_links.add(remote_ip_port,(int64_t)link->active_time);
ProcJob job;
job.link = link;
this->proc(&job);
if(job.result == PROC_THREAD){
fdes->del(link->fd());
continue;
}
if(job.result == PROC_BACKEND){
fdes->del(link->fd());
this->link_count --;
char remote_ip_port[32];
snprintf(remote_ip_port, 32, "%s:%d", link->remote_ip, link->remote_port);
this->link_map.erase(remote_ip_port);
this->active_links.del(remote_ip_port);
// don't delete link
continue;
}
if(proc_result(&job, &ready_list_2) == PROC_ERROR){
//
}
} // end foreach ready link
//every event loop
destroy_idle_link();
}
}
void run(int argc, char **argv){
typedef std::vector<Link *> ready_list_t;
ready_list_t ready_list;
ready_list_t ready_list_2;
ready_list_t::iterator it;
const Fdevents::events_t *events;
Fdevents select;
select.set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
CommandProc proc(ssdb);
while(!quit){
if(!ready_list.empty()){
// give links that are not in ready_list a chance
events = select.wait(0);
}else{
events = select.wait(200);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = serv_link->accept();
if(link == NULL){
log_error("accept fail!");
continue;
}
//
link->nodelay();
link->noblock();
//
select.set(link->fd(), FDEVENT_IN, 1, link);
log_info("new link: %d", link->fd());
}else{
Link *link = (Link *)fde->data.ptr;
// 不能同时监听读和写事件, 只能监听其中一个
if(fde->events & FDEVENT_IN){
int len = link->read();
log_trace("fd: %d read: %d", link->fd(), len);
if(len <= 0){
log_info("fd: %d, read: %d, delete link", link->fd(), len);
select.del(link->fd());
delete link;
}else{
ready_list.push_back(link);
}
}else if(fde->events & FDEVENT_OUT){
int len = link->write();
log_trace("fd: %d write: %d", link->fd(), len);
if(len < 0){
log_info("fd: %d, write: %d, delete link", link->fd(), len);
select.del(link->fd());
delete link;
}else if(link->output->empty()){
//log_trace("delete %d from select.out", link->fd());
select.clr(link->fd(), FDEVENT_OUT);
if(!link->input->empty()){
ready_list.push_back(link);
}else{
//log_trace("add %d to select.in", link->fd());
select.set(link->fd(), FDEVENT_IN, 1, link);
}
}
}
}
}
ready_list_2.clear();
for(it = ready_list.begin(); it != ready_list.end(); it ++){
Link *link = *it;
const Request *req = link->recv();
if(req == NULL){
log_warn("fd: %d, link parse error, delete link", link->fd());
select.del(link->fd());
delete link;
continue;
}
if(req->empty()){
if(!select.isset(link->fd(), FDEVENT_IN)){
//log_trace("add %d to select.in", link->fd());
select.set(link->fd(), FDEVENT_IN, 1, link);
}
continue;
}
Response resp;
int status = proc.proc(*link, *req, &resp);
if(status == PROC_ERROR){
log_info("fd: %d, proc error, delete link", link->fd());
select.del(link->fd());
delete link;
continue;
}else if(status == PROC_BACKEND){
select.del(link->fd());
continue;
}
if(link->send(resp) == -1){
log_info("fd: %d, send error, delete link", link->fd());
select.del(link->fd());
delete link;
continue;
}
if(!link->output->empty()){
//log_trace("add %d to select.out", link->fd());
select.set(link->fd(), FDEVENT_OUT, 1, link);
if(select.isset(link->fd(), FDEVENT_IN)){
//log_trace("delete %d from select.in", link->fd());
select.clr(link->fd(), FDEVENT_IN);
}
}else{
if(link->input->empty()){
if(!select.isset(link->fd(), FDEVENT_IN)){
//log_trace("add %d to select.in", link->fd());
select.set(link->fd(), FDEVENT_IN, 1, link);
}
}else{
if(select.isset(link->fd(), FDEVENT_IN)){
//log_trace("delete %d from select.in", link->fd());
select.clr(link->fd(), FDEVENT_IN);
}
ready_list_2.push_back(link);
}
}
} // end foreach ready link
ready_list.swap(ready_list_2);
}
}
void NetworkServer::serve(){
{
auto_mask mask(3, SIGPIPE, SIGINT, SIGTERM, SIGQUIT);
writer = new (std::nothrow)ProcWorkerPool("writer");
ASSERT_SYS(NULL != writer, "try to new ProcWorkerPool failed, no memory");
writer->start(num_writers);
reader = new (std::nothrow)ProcWorkerPool("reader");
ASSERT_SYS(NULL != reader, "try to new ProcWorkerPool failed, no memory");
reader->start(num_readers);
}
ready_list_t ready_list;
ready_list_t ready_list_2;
ready_list_t::iterator it;
const Fdevents::events_t *events;
fdes->set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
fdes->set(this->reader->fd(), FDEVENT_IN, 0, this->reader);
fdes->set(this->writer->fd(), FDEVENT_IN, 0, this->writer);
uint32_t last_ticks = g_ticks;
while(!quit){
if((uint32_t)(g_ticks - last_ticks) >= STATUS_REPORT_TICKS){
last_ticks = g_ticks;
log_info("server is running, links: %d", this->link_count);
}
ready_list.swap(ready_list_2);
ready_list_2.clear();
if(!ready_list.empty()){
events = fdes->wait(0);
}else{
events = fdes->wait(50);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = accept_link();
if(link){
this->link_count ++;
log_debug("new link from %s:%d, fd: %d, links count: %d",
link->remote_ip, link->remote_port,
link->fd(), this->link_count);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}
}else if(fde->data.ptr == this->reader || fde->data.ptr == this->writer){
ProcWorkerPool *worker = (ProcWorkerPool *)fde->data.ptr;
ProcJob *job = NULL;
if(worker->pop(&job) == 0){
log_fatal("reading result from workers error!");
exit(0);
}
if(proc_result(job, &ready_list) == PROC_ERROR){
//
}
}else{
proc_client_event(fde, &ready_list);
}
}
for(it = ready_list.begin(); it != ready_list.end(); it ++){
Link *link = *it;
if(link->error()){
this->link_count --;
fdes->del(link->fd());
SAFE_DELETE(link);
continue;
}
if(link->readed >= sizeof(head_t) && 0x00 == link->get_basic_string()[0]){
if(0 == link->request.head.logid){
link->request.head.logid = pow(2, 64) * (rand() / (RAND_MAX + 1.0));
}
link->add_basic_info("client_ip", link->remote_ip);
link->add_basic_info("client_port", link->remote_port);
link->add_basic_info("logid", link->request.head.logid);
link->add_basic_info("method", link->request.head.method);
link->add_basic_info("magic_num", link->request.head.magic_num);
}
if(link->readed >= sizeof(head_t) && link->request.head.bodylen > MAX_PACKAGE_SIZE){
log_frame("[ %s ]fd: %d, body len is too big, give up reading[ body len: %u, max: %u ].",
link->get_basic_string(), link->request.head.bodylen, MAX_PACKAGE_SIZE);
fdes->del(link->fd());
}else if(link->readed < sizeof(head_t) || link->request.head.bodylen + sizeof(head_t) > link->readed){
//head has not been read over or body has not been read over, continue to read
log_frame("[ %s ]fd: %d, readed: %d, bodylen: %u, continue to read",
(link->readed >= sizeof(head_t) ? link->get_basic_string() : ""), link->fd(),
link->readed, link->request.head.bodylen);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
continue;
}else{
log_frame("[ %s ]fd: %d, readed: %d, bodylen: %u, read data finished", link->get_basic_string(), link->fd(),
link->readed, link->request.head.bodylen);
fdes->del(link->fd());
}
link->active_time = millitime();
//prepare response head first
link->response.head = link->request.head;
snprintf(link->response.head.provider, sizeof(link->response.head.provider), "%s", provider);
link->response.head.bodylen = 0;
ProcJob *job = new (std::nothrow)ProcJob();
ASSERT_SYS(NULL != job, "try to new ProcJob failed, no memory");
job->link = link;
job->req = &link->request;
int result = this->proc(job);
if(result == PROC_THREAD){
fdes->del(link->fd());
continue;
}
if(result == PROC_BACKEND){
job->req = &link->request;
this->link_count --;
continue;
}
if(proc_result(job, &ready_list_2) == PROC_ERROR){
//
}
}
// end foreach ready link
}
}
void* BackendSync::_run_thread(void *arg){
struct run_arg *p = (struct run_arg*)arg;
BackendSync *backend = (BackendSync *)p->backend;
Link *link = (Link *)p->link;
delete p;
//
link->noblock(false);
SSDB *ssdb = (SSDB *)backend->ssdb;
BinlogQueue *logs = ssdb->binlogs;
Client client(backend);
client.link = link;
client.init();
int idle = 0;
while(!backend->thread_quit){
// TODO: test
//usleep(2000 * 1000);
if(client.status == Client::OUT_OF_SYNC){
client.reset();
continue;
}
bool is_empty = true;
if(client.status == Client::COPY){
if(client.copy()){
is_empty = false;
}
}
if(client.sync(logs)){
is_empty = false;
}
if(is_empty){
if(idle == 10){
idle = 0;
client.noop();
}else{
idle ++;
// sleep longer to reduce logs.find
usleep(300 * 1000);
}
}else{
idle = 0;
}
if(link->flush() == -1){
log_error("fd: %d, send error: %s", link->fd(), strerror(errno));
break;
}
}
log_info("Sync Client quit, fd: %d, delete link", link->fd());
delete link;
Locking l(&backend->mutex);
backend->workers.erase(pthread_self());
return (void *)NULL;
}
void* BackendSync::_run_thread(void *arg){
struct run_arg *p = (struct run_arg*)arg;
BackendSync *backend = (BackendSync *)p->backend;
Link *link = (Link *)p->link;
delete p;
//
link->noblock(false);
SSDB *ssdb = (SSDB *)backend->ssdb;
SyncLogQueue *logs = ssdb->replication->logs;
Client client(backend);
client.link = link;
client.init();
int idle = 0;
while(!backend->thread_quit){
// TEST: simulate slow network
#ifndef NDEBUG
//usleep(1000 * 1000);
#endif
if(client.status == Client::OUT_OF_SYNC){
// TODO: tell slave to clear database
client.reset_sync();
}
bool is_empty = true;
if(client.iter){
is_empty = false;
client.dump();
}
if(client.sync(logs)){
is_empty = false;
}
if(is_empty){
idle ++;
if(idle == 10){
idle = 0;
if(link->send("noop") == -1){
log_debug("fd: %d, send error", link->fd());
break;
}
}
// sleep longer to reduce logs.find
usleep(200 * 1000);
}else{
idle = 0;
}
if(link->flush() == -1){
log_debug("fd: %d, send error", link->fd());
break;
}
}
log_info("SyncClient quit, fd: %d, delete link", link->fd());
delete link;
Locking l(&backend->mutex);
backend->workers.erase(pthread_self());
return (void *)NULL;
}
void run(int argc, char **argv){
ready_list_t ready_list;
ready_list_t ready_list_2;
ready_list_t::iterator it;
const Fdevents::events_t *events;
Server serv(ssdb);
Fdevents select;
select.set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
select.set(serv.writer.fd(), FDEVENT_IN, 0, &serv.writer);
int link_count = 0;
while(!quit){
bool write_pending = false;
ready_list.clear();
ready_list_2.clear();
if(write_pending || !ready_list.empty()){
// give links that are not in ready_list a chance
events = select.wait(0);
}else{
events = select.wait(50);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = serv_link->accept();
if(link == NULL){
log_error("accept fail!");
continue;
}
link_count ++;
log_info("new link from %s:%d, fd: %d, link_count: %d",
link->remote_ip, link->remote_port, link->fd(), link_count);
link->nodelay();
link->noblock();
link->create_time = millitime();
link->active_time = link->create_time;
select.set(link->fd(), FDEVENT_IN, 1, link);
}else if(fde->data.ptr == &serv.writer){
ProcJob job;
if(serv.writer.pop(&job) == 0){
log_fatal("reading result from workers error!");
exit(0);
}
if(proc_result(job, select, ready_list_2) == PROC_ERROR){
link_count --;
}
}else{
Link *link = (Link *)fde->data.ptr;
// 不能同时监听读和写事件, 只能监听其中一个
if(fde->events & FDEVENT_ERR){
log_info("fd: %d error, delete link", link->fd());
link_count --;
select.del(link->fd());
delete link;
}else if(fde->events & FDEVENT_IN){
int len = link->read();
//log_trace("fd: %d read: %d", link->fd(), len);
if(len <= 0){
log_info("fd: %d, read: %d, delete link", link->fd(), len);
link_count --;
select.del(link->fd());
delete link;
}else{
ready_list.push_back(link);
}
}else if(fde->events & FDEVENT_OUT){
int len = link->write();
//log_trace("fd: %d write: %d", link->fd(), len);
if(len <= 0){
log_info("fd: %d, write: %d, delete link", link->fd(), len);
link_count --;
select.del(link->fd());
delete link;
}else if(link->output->empty()){
//log_trace("delete %d from select.out", link->fd());
select.clr(link->fd(), FDEVENT_OUT);
if(!link->input->empty()){
ready_list.push_back(link);
}else{
//log_trace("add %d to select.in", link->fd());
select.set(link->fd(), FDEVENT_IN, 1, link);
}
}else{
write_pending = true;
}
}
}
}
for(it = ready_list.begin(); it != ready_list.end(); it ++){
Link *link = *it;
const Request *req = link->recv();
if(req == NULL){
log_warn("fd: %d, link parse error, delete link", link->fd());
link_count --;
select.del(link->fd());
delete link;
continue;
}
if(req->empty()){
if(!select.isset(link->fd(), FDEVENT_IN)){
//log_trace("add %d to select.in", link->fd());
select.set(link->fd(), FDEVENT_IN, 1, link);
}
continue;
}
ProcJob job;
job.link = link;
serv.proc(&job);
if(job.result == PROC_BACKEND){
select.del(link->fd());
continue;
}
link->active_time = millitime();
if(proc_result(job, select, ready_list_2) == PROC_ERROR){
link_count --;
}
} // end foreach ready link
ready_list.swap(ready_list_2);
}
}
void* BackendSync::_run_thread(void *arg) {
pthread_detach(pthread_self());
struct run_arg *p = (struct run_arg*)arg;
BackendSync *backend = (BackendSync *)p->backend;
Link *link = (Link *)p->link;
delete p;
// set Link non block
link->noblock(false);
SSDBImpl *ssdb = (SSDBImpl *)backend->ssdb;
BinlogQueue *logs = ssdb->binlogs;
Client client(backend);
client.link = link;
client.init();
{
pthread_t tid = pthread_self();
Locking l(&backend->mutex);
backend->workers[tid] = &client;
}
// sleep longer to reduce logs.find
#define TICK_INTERVAL_MS 300
#define NOOP_IDLES (3000/TICK_INTERVAL_MS)
int idle = 0;
while(!backend->thread_quit) {
// TODO: test
//usleep(2000 * 1000);
if(client.status == Client::OUT_OF_SYNC) {
client.reset();
continue;
}
bool is_empty = true;
// WARN: MUST do first sync() before first copy(), because
// sync() will refresh last_seq, and copy() will not
if(client.sync(logs)) {
is_empty = false;
}
if(client.status == Client::COPY) {
if(client.copy()) {
is_empty = false;
}
}
if(is_empty) {
if(idle >= NOOP_IDLES) {
idle = 0;
client.noop();
} else {
idle ++;
usleep(TICK_INTERVAL_MS * 1000);
}
} else {
idle = 0;
}
float data_size_mb = link->output->size() / 1024.0 / 1024.0;
if(link->flush() == -1) {
log_info("%s:%d fd: %d, send error: %s", link->remote_ip, link->remote_port, link->fd(), strerror(errno));
break;
}
if(backend->sync_speed > 0) {
usleep((data_size_mb / backend->sync_speed) * 1000 * 1000);
}
}
log_info("Sync Client quit, %s:%d fd: %d, delete link", link->remote_ip, link->remote_port, link->fd());
delete link;
Locking l(&backend->mutex);
backend->workers.erase(pthread_self());
return (void *)NULL;
}
void NetworkServer::serve(){
writer = new ProcWorkerPool("writer");
writer->start(num_writers);
reader = new ProcWorkerPool("reader");
reader->start(num_readers);
ready_list_t ready_list;
ready_list_t ready_list_2;
ready_list_t::iterator it;
const Fdevents::events_t *events;
fdes->set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
fdes->set(this->reader->fd(), FDEVENT_IN, 0, this->reader);
fdes->set(this->writer->fd(), FDEVENT_IN, 0, this->writer);
uint32_t last_ticks = g_ticks;
while(!quit){
double loop_stime = millitime();
// status report
if((uint32_t)(g_ticks - last_ticks) >= STATUS_REPORT_TICKS){
last_ticks = g_ticks;
log_info("server running, links: %d", this->link_count);
}
ready_list.swap(ready_list_2);
ready_list_2.clear();
if(!ready_list.empty()){
// ready_list not empty, so we should return immediately
events = fdes->wait(0);
}else{
events = fdes->wait(50);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = accept_link();
if(link){
this->link_count ++;
log_debug("new link from %s:%d, fd: %d, links: %d",
link->remote_ip, link->remote_port, link->fd(), this->link_count);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}else{
log_debug("accept return NULL");
}
}else if(fde->data.ptr == this->reader || fde->data.ptr == this->writer){
ProcWorkerPool *worker = (ProcWorkerPool *)fde->data.ptr;
ProcJob *job = NULL;
if(worker->pop(&job) == 0){
log_fatal("reading result from workers error!");
exit(0);
}
proc_result(job, &ready_list);
}else{
proc_client_event(fde, &ready_list);
}
}
for(it = ready_list.begin(); it != ready_list.end(); it ++){
Link *link = *it;
fdes->del(link->fd());
if(link->error()){
this->link_count --;
delete link;
continue;
}
const Request *req = link->recv();
if(req == NULL){
log_warn("fd: %d, link parse error, delete link", link->fd());
link->mark_error();
ready_list_2.push_back(link);
continue;
}
if(req->empty()){
fdes->set(link->fd(), FDEVENT_IN, 1, link);
continue;
}
link->active_time = millitime();
ProcJob *job = new ProcJob();
job->link = link;
job->req = link->last_recv();
int result = this->proc(job);
if(result == PROC_THREAD){
//
}else if(result == PROC_BACKEND){
// link_count does not include backend links
this->link_count --;
}else{
proc_result(job, &ready_list_2);
}
} // end foreach ready link
double loop_time = millitime() - loop_stime;
if(loop_time > 0.5){
log_warn("long loop time: %.3f", loop_time);
}
}
}
void* BackendDump::_run_thread(void *arg){
struct run_arg *p = (struct run_arg*)arg;
const BackendDump *backend = p->backend;
Link *link = (Link *)p->link;
delete p;
//
link->noblock(false);
const std::vector<Bytes>* req = link->last_recv();
std::string start = "";
if(req->size() > 1){
Bytes b = req->at(1);
start.assign(b.data(), b.size());
}
if(start.empty()){
start = "A";
}
std::string end = "";
if(req->size() > 2){
Bytes b = req->at(2);
end.assign(b.data(), b.size());
}
uint64_t limit = 10;
if(req->size() > 3){
Bytes b = req->at(3);
limit = b.Uint64();
}
log_info("fd: %d, begin to dump data: '%s', '%s', %" PRIu64 "",
link->fd(), start.c_str(), end.c_str(), limit);
Buffer *output = link->output;
int count = 0;
bool quit = false;
Iterator *it = backend->ssdb->iterator(start, end, limit);
link->send("begin");
while(!quit){
if(!it->next()){
quit = true;
char buf[20];
snprintf(buf, sizeof(buf), "%d", count);
link->send("end", buf);
}else{
count ++;
Bytes key = it->key();
Bytes val = it->val();
output->append_record("set");
output->append_record(key);
output->append_record(val);
output->append('\n');
if(output->size() < 32 * 1024){
continue;
}
}
if(link->flush() == -1){
log_error("fd: %d, send error: %s", link->fd(), strerror(errno));
break;
}
}
// wait for client to close connection,
// or client may get a "Connection reset by peer" error.
link->read();
log_info("fd: %d, delete link", link->fd());
delete link;
return (void *)NULL;
}
void bench(std::string cmd){
int total = (int)ds->size();
int finished = 0;
int num_sent = 0;
printf("========== %s ==========\n", cmd.c_str());
std::map<std::string, Data *>::iterator it;
it = ds->begin();
double stime = millitime();
while(1){
while(!free_links->empty()){
if(num_sent == total){
break;
}
num_sent ++;
Link *link = free_links->back();
free_links->pop_back();
send_req(link, cmd, it->second);
it ++;
}
const Fdevents::events_t *events;
events = fdes->wait(50);
if(events == NULL){
log_error("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
Link *link = (Link *)fde->data.ptr;
int len = link->read();
if(len <= 0){
log_error("fd: %d, read: %d, delete link", link->fd(), len);
exit(0);
}
const std::vector<Bytes> *resp = link->recv();
if(resp == NULL){
log_error("error");
break;
}else if(resp->empty()){
continue;
}else{
if(resp->at(0) != "ok"){
log_error("bad response: %s", resp->at(0).String().c_str());
exit(0);
}
free_links->push_back(link);
finished ++;
if(finished == total){
double etime = millitime();
double ts = (stime == etime)? 1 : (etime - stime);
double speed = total / ts;
printf("qps: %d, time: %.3f s\n", (int)speed, ts);
return;
}
}
}
}
}
void NetworkServer::serve(){
writer = new ProcWorkerPool("writer");
writer->start(num_writers);
reader = new ProcWorkerPool("reader");
reader->start(num_readers);
pthread_t tid;
int err = pthread_create(&tid, NULL, &NetworkServer::_ops_timer_thread, this);
if (err != 0) {
log_error("can't start ops timer thread: %s", strerror(err));
exit(-1);
}
link_dict_t ready_dict;
link_dict_t tmp_dict;
link_dict_t blocked_dict;
link_dict_t::iterator it;
const Fdevents::events_t *events;
fdes->set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
fdes->set(this->reader->fd(), FDEVENT_IN, 0, this->reader);
fdes->set(this->writer->fd(), FDEVENT_IN, 0, this->writer);
uint32_t last_ticks = g_ticks;
while(!quit){
// status report
if((uint32_t)(g_ticks - last_ticks) >= STATUS_REPORT_TICKS){
last_ticks = g_ticks;
log_debug("server running, links: %d", this->link_count);
}
ready_dict.swap(tmp_dict);
tmp_dict.clear();
if(!ready_dict.empty()){
/* ready_dict not empty, so we should return immediately */
events = fdes->wait(0);
}else{
events = fdes->wait(50);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = accept_link();
if(link){
this->link_count ++;
log_debug("new link from %s:%d, fd: %d, links: %d",
link->remote_ip, link->remote_port, link->fd(), this->link_count);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}
}else if(fde->data.ptr == this->reader || fde->data.ptr == this->writer){
ProcWorkerPool *worker = (ProcWorkerPool *)fde->data.ptr;
ProcJob job;
if(worker->pop(&job) == 0){
log_fatal("reading result from workers error!");
exit(0);
}
if(proc_result(&job, &ready_dict) == PROC_ERROR){
//
}
}else{
proc_client_event(fde, &ready_dict);
}
}
/* if clients paused, add specified link into blocked_list and disable parsing request */
if(NetworkServer::clients_paused) {
if(NetworkServer::clients_pause_end_time < time_ms()) {
NetworkServer::clients_paused = 0;
NetworkServer::clients_pause_end_time = 0;
ready_dict.insert(blocked_dict.begin(), blocked_dict.end());
blocked_dict.clear();
} else {
blocked_dict.insert(ready_dict.begin(), ready_dict.end());
ready_dict.clear();
continue;
}
}
for(it = ready_dict.begin(); it != ready_dict.end(); it ++){
Link *link = it->second;
if(link->error()){
this->link_count --;
fdes->del(link->fd());
delete link;
continue;
}
const Request *req = link->recv();
if(req == NULL){
log_warn("fd: %d, link parse error, delete link", link->fd());
this->link_count --;
fdes->del(link->fd());
delete link;
continue;
}
if(req->empty()){
fdes->set(link->fd(), FDEVENT_IN, 1, link);
continue;
}
link->active_time = millitime();
ProcJob job;
job.link = link;
this->proc(&job);
if(job.result == PROC_THREAD){
fdes->del(link->fd());
continue;
}
if(job.result == PROC_BACKEND){
fdes->del(link->fd());
this->link_count --;
continue;
}
if(proc_result(&job, &tmp_dict) == PROC_ERROR){
//
}
} // end foreach ready link
}
}
