/*
event:
read => ready_list OR close
write => NONE
proc =>
done: write & (read OR ready_list)
async: stop (read & write)
1. When writing to a link, it may happen to be in the ready_list,
so we cannot close that link in write process, we could only
just mark it as closed.
2. When reading from a link, it is never in the ready_list, so it
is safe to close it in read process, also safe to put it into
ready_list.
3. Ignore FDEVENT_ERR
A link is in either one of these places:
1. ready list
2. async worker queue
So it safe to delete link when processing ready list and async worker result.
*/
int NetworkServer::proc_client_event(const Fdevent *fde, ready_list_t *ready_list){
Link *link = (Link *)fde->data.ptr;
if(fde->events & FDEVENT_IN){
ready_list->push_back(link);
if(link->error()){
return 0;
}
int len = link->read();
//log_debug("fd: %d read: %d", link->fd(), len);
if(len <= 0){
log_debug("fd: %d, read: %d, delete link", link->fd(), len);
link->mark_error();
return 0;
}
}
if(fde->events & FDEVENT_OUT){
if(link->error()){
return 0;
}
int len = link->write();
if(len <= 0){
log_debug("fd: %d, write: %d, delete link", link->fd(), len);
link->mark_error();
return 0;
}
if(link->output->empty()){
fdes->clr(link->fd(), FDEVENT_OUT);
}
}
return 0;
}
int NetworkServer::proc_client_event(const Fdevent *fde, ready_list_t *ready_list){
Link *link = (Link *)fde->data.ptr;
if(fde->events & FDEVENT_ERR){
ready_list->push_back(link);
log_warn("fd: %d, wait error, delete link", link->fd());
link->mark_error();
return 0;
}
if(fde->events & FDEVENT_IN){
ready_list->push_back(link);
if(link->error()){
return 0;
}
int len = link->read();
if(len <= 0){
if(link->readed < sizeof(head_t) || link->request.head.bodylen < MAX_PACKAGE_SIZE){
log_frame("fd: %d, read: %d, delete link [ readed: %d, head_size: %d, bodylen: %u, max: %d ]",
link->fd(), len, link->readed, sizeof(head_t), link->request.head.bodylen, MAX_PACKAGE_SIZE);
link->mark_error();
}
return 0;
}
}
if(fde->events & FDEVENT_OUT){
if(link->error()){
return 0;
}
int len = link->write();
if(len < 0){
log_warn("[%s] fd: %d, write: %d, delete link", link->get_basic_string(), link->fd(), len);
link->mark_error();
return 0;
}
if(link->response.head.bodylen + sizeof(head_t) <= link->writed){
log_frame("[%s] fd: %d, write: %d, finish writing", link->get_basic_string());
fdes->clr(link->fd(), FDEVENT_OUT);
}
if(!long_connect || 0 != link->response.head.status){
link_count--;
fdes->del(link->fd());
log_frame("[%s] short connection, close [ fd: %d ]", link->get_basic_string(), link->fd());
SAFE_DELETE(link);
}else{
link->readed = link->writed = 0;
link->request.reset();
link->response.reset();
link->reset_basic_string();
fdes->set(link->fd(), FDEVENT_IN, 1, link);
log_frame("[%s] long connection, try to receive request next time [ fd: %d ]", link->get_basic_string(), link->fd());
}
}
return 0;
}
int Server::read_session(Session *sess){
Link *link = sess->link;
if(link->error()){
return 0;
}
int len = link->read();
if(len <= 0){
this->close_session(sess);
return -1;
}
while(1){
Request req;
int ret = link->recv(&req.msg);
if(ret == -1){
log_info("fd: %d, parse error, delete link", link->fd());
this->close_session(sess);
return -1;
}else if(ret == 0){
// 报文未就绪, 继续读网络
break;
}
req.stime = millitime();
req.sess = *sess;
Response resp;
for(int i=0; i<this->handlers.size(); i++){
Handler *handler = this->handlers[i];
req.time_wait = 1000 * (millitime() - req.stime);
HandlerState state = handler->proc(req, &resp);
req.time_proc = 1000 * (millitime() - req.stime) - req.time_wait;
if(state == HANDLE_RESP){
link->send(resp.msg);
if(link && !link->output.empty()){
fdes->set(link->fd(), FDEVENT_OUT, DEFAULT_TYPE, sess);
}
if(log_level() >= Logger::LEVEL_DEBUG){
log_debug("w:%.3f,p:%.3f, req: %s resp: %s",
req.time_wait, req.time_proc,
msg_str(req.msg).c_str(),
msg_str(resp.msg).c_str());
}
}else if(state == HANDLE_FAIL){
this->close_session(sess);
return -1;
}
}
}
return 0;
}
int NetworkServer::proc_result(ProcJob *job, ready_list_t *ready_list){
Link *link = job->link;
int len;
if(job->cmd){
job->cmd->calls += 1;
job->cmd->time_wait += job->time_wait;
job->cmd->time_proc += job->time_proc;
double total_time = job->time_wait + job->time_proc;
if (total_time >= this->slow_time) {
slowlog_warn("proc slow, remote_ip: %s wait: %.3f proc: %.3f total: %.3f req: %s",
link->remote_ip, job->time_wait, job->time_proc, total_time,
serialize_req(*link->last_recv()).c_str());
}
}
if(job->result == PROC_ERROR){
log_info("fd: %d, proc error, delete link", link->fd());
goto proc_err;
}
if(link->error()){
log_debug("fd: %d, link error, delete link", link->fd());
goto proc_err;
}
len = link->write();
//log_debug("write: %d", len);
if(len < 0){
log_debug("fd: %d, write: %d, delete link", link->fd(), len);
goto proc_err;
}
this->bytes_written += len;
if(!link->output->empty()){
fdes->set(link->fd(), FDEVENT_OUT, 1, link);
}
if(link->input->empty()){
fdes->set(link->fd(), FDEVENT_IN, 1, link);
link->unRef();
log_debug("proc_result remote_ip: %s ref: %d", link->remote_ip, link->ref_count);
}else{
fdes->clr(link->fd(), FDEVENT_IN);
ready_list->push_back(link);
}
return PROC_OK;
proc_err:
destroy_link(link);
return PROC_ERROR;
}
int Server::write_session(Session *sess){
Link *link = sess->link;
if(link->error()){
return 0;
}
int len = link->write();
if(len <= 0){
log_debug("fd: %d, write: %d, delete link", link->fd(), len);
this->close_session(sess);
return -1;
}
if(link->output.empty()){
fdes->clr(link->fd(), FDEVENT_OUT);
}
return 0;
}
void NetworkServer::serve(){
writer = new ProcWorkerPool("writer");
writer->start(num_writers);
reader = new ProcWorkerPool("reader");
reader->start(num_readers);
ready_list_t ready_list;
ready_list_t ready_list_2;
ready_list_t::iterator it;
const Fdevents::events_t *events;
fdes->set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
fdes->set(this->reader->fd(), FDEVENT_IN, 0, this->reader);
fdes->set(this->writer->fd(), FDEVENT_IN, 0, this->writer);
uint32_t last_ticks = g_ticks;
while(!quit){
double loop_stime = millitime();
// status report
if((uint32_t)(g_ticks - last_ticks) >= STATUS_REPORT_TICKS){
last_ticks = g_ticks;
log_info("server running, links: %d", this->link_count);
}
ready_list.swap(ready_list_2);
ready_list_2.clear();
if(!ready_list.empty()){
// ready_list not empty, so we should return immediately
events = fdes->wait(0);
}else{
events = fdes->wait(50);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = accept_link();
if(link){
this->link_count ++;
log_debug("new link from %s:%d, fd: %d, links: %d",
link->remote_ip, link->remote_port, link->fd(), this->link_count);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}else{
log_debug("accept return NULL");
}
}else if(fde->data.ptr == this->reader || fde->data.ptr == this->writer){
ProcWorkerPool *worker = (ProcWorkerPool *)fde->data.ptr;
ProcJob *job = NULL;
if(worker->pop(&job) == 0){
log_fatal("reading result from workers error!");
exit(0);
}
proc_result(job, &ready_list);
}else{
proc_client_event(fde, &ready_list);
}
}
for(it = ready_list.begin(); it != ready_list.end(); it ++){
Link *link = *it;
fdes->del(link->fd());
if(link->error()){
this->link_count --;
delete link;
continue;
}
const Request *req = link->recv();
if(req == NULL){
log_warn("fd: %d, link parse error, delete link", link->fd());
link->mark_error();
ready_list_2.push_back(link);
continue;
}
if(req->empty()){
fdes->set(link->fd(), FDEVENT_IN, 1, link);
continue;
}
link->active_time = millitime();
ProcJob *job = new ProcJob();
job->link = link;
job->req = link->last_recv();
int result = this->proc(job);
if(result == PROC_THREAD){
//
}else if(result == PROC_BACKEND){
// link_count does not include backend links
this->link_count --;
}else{
proc_result(job, &ready_list_2);
}
} // end foreach ready link
double loop_time = millitime() - loop_stime;
if(loop_time > 0.5){
log_warn("long loop time: %.3f", loop_time);
}
}
}
void NetworkServer::serve(){
{
auto_mask mask(3, SIGPIPE, SIGINT, SIGTERM, SIGQUIT);
writer = new (std::nothrow)ProcWorkerPool("writer");
ASSERT_SYS(NULL != writer, "try to new ProcWorkerPool failed, no memory");
writer->start(num_writers);
reader = new (std::nothrow)ProcWorkerPool("reader");
ASSERT_SYS(NULL != reader, "try to new ProcWorkerPool failed, no memory");
reader->start(num_readers);
}
ready_list_t ready_list;
ready_list_t ready_list_2;
ready_list_t::iterator it;
const Fdevents::events_t *events;
fdes->set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
fdes->set(this->reader->fd(), FDEVENT_IN, 0, this->reader);
fdes->set(this->writer->fd(), FDEVENT_IN, 0, this->writer);
uint32_t last_ticks = g_ticks;
while(!quit){
if((uint32_t)(g_ticks - last_ticks) >= STATUS_REPORT_TICKS){
last_ticks = g_ticks;
log_info("server is running, links: %d", this->link_count);
}
ready_list.swap(ready_list_2);
ready_list_2.clear();
if(!ready_list.empty()){
events = fdes->wait(0);
}else{
events = fdes->wait(50);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = accept_link();
if(link){
this->link_count ++;
log_debug("new link from %s:%d, fd: %d, links count: %d",
link->remote_ip, link->remote_port,
link->fd(), this->link_count);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}
}else if(fde->data.ptr == this->reader || fde->data.ptr == this->writer){
ProcWorkerPool *worker = (ProcWorkerPool *)fde->data.ptr;
ProcJob *job = NULL;
if(worker->pop(&job) == 0){
log_fatal("reading result from workers error!");
exit(0);
}
if(proc_result(job, &ready_list) == PROC_ERROR){
//
}
}else{
proc_client_event(fde, &ready_list);
}
}
for(it = ready_list.begin(); it != ready_list.end(); it ++){
Link *link = *it;
if(link->error()){
this->link_count --;
fdes->del(link->fd());
SAFE_DELETE(link);
continue;
}
if(link->readed >= sizeof(head_t) && 0x00 == link->get_basic_string()[0]){
if(0 == link->request.head.logid){
link->request.head.logid = pow(2, 64) * (rand() / (RAND_MAX + 1.0));
}
link->add_basic_info("client_ip", link->remote_ip);
link->add_basic_info("client_port", link->remote_port);
link->add_basic_info("logid", link->request.head.logid);
link->add_basic_info("method", link->request.head.method);
link->add_basic_info("magic_num", link->request.head.magic_num);
}
if(link->readed >= sizeof(head_t) && link->request.head.bodylen > MAX_PACKAGE_SIZE){
log_frame("[ %s ]fd: %d, body len is too big, give up reading[ body len: %u, max: %u ].",
link->get_basic_string(), link->request.head.bodylen, MAX_PACKAGE_SIZE);
fdes->del(link->fd());
}else if(link->readed < sizeof(head_t) || link->request.head.bodylen + sizeof(head_t) > link->readed){
//head has not been read over or body has not been read over, continue to read
log_frame("[ %s ]fd: %d, readed: %d, bodylen: %u, continue to read",
(link->readed >= sizeof(head_t) ? link->get_basic_string() : ""), link->fd(),
link->readed, link->request.head.bodylen);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
continue;
}else{
log_frame("[ %s ]fd: %d, readed: %d, bodylen: %u, read data finished", link->get_basic_string(), link->fd(),
link->readed, link->request.head.bodylen);
fdes->del(link->fd());
}
link->active_time = millitime();
//prepare response head first
link->response.head = link->request.head;
snprintf(link->response.head.provider, sizeof(link->response.head.provider), "%s", provider);
link->response.head.bodylen = 0;
ProcJob *job = new (std::nothrow)ProcJob();
ASSERT_SYS(NULL != job, "try to new ProcJob failed, no memory");
job->link = link;
job->req = &link->request;
int result = this->proc(job);
if(result == PROC_THREAD){
fdes->del(link->fd());
continue;
}
if(result == PROC_BACKEND){
job->req = &link->request;
this->link_count --;
continue;
}
if(proc_result(job, &ready_list_2) == PROC_ERROR){
//
}
}
// end foreach ready link
}
}
void NetworkServer::serve(){
writer = new ProcWorkerPool("writer");
writer->start(WRITER_THREADS);
reader = new ProcWorkerPool("reader");
reader->start(READER_THREADS);
ready_list_t ready_list;
ready_list_t ready_list_2;
ready_list_t::iterator it;
const Fdevents::events_t *events;
fdes->set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
fdes->set(this->reader->fd(), FDEVENT_IN, 0, this->reader);
fdes->set(this->writer->fd(), FDEVENT_IN, 0, this->writer);
uint32_t last_ticks = g_ticks;
while(!quit){
// status report
if((uint32_t)(g_ticks - last_ticks) >= STATUS_REPORT_TICKS){
last_ticks = g_ticks;
log_info("server running, links: %d", this->link_count);
}
ready_list.swap(ready_list_2);
ready_list_2.clear();
if(!ready_list.empty()){
// ready_list not empty, so we should return immediately
events = fdes->wait(0);
}else{
events = fdes->wait(50);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = accept_link();
if(link){
this->link_count ++;
log_debug("new link from %s:%d, fd: %d, links: %d",
link->remote_ip, link->remote_port, link->fd(), this->link_count);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}
}else if(fde->data.ptr == this->reader || fde->data.ptr == this->writer){
ProcWorkerPool *worker = (ProcWorkerPool *)fde->data.ptr;
ProcJob job;
if(worker->pop(&job) == 0){
log_fatal("reading result from workers error!");
exit(0);
}
if(proc_result(&job, &ready_list) == PROC_ERROR){
//
}
}else{
proc_client_event(fde, &ready_list);
}
}
for(it = ready_list.begin(); it != ready_list.end(); it ++){
Link *link = *it;
if(link->error()){
destroy_link(link);
continue;
}
const Request *req = link->recv();
if(req == NULL){
log_warn("fd: %d, link parse error, delete link", link->fd());
destroy_link(link);
continue;
}
if(req->empty()){
link->unRef();
fdes->set(link->fd(), FDEVENT_IN, 1, link);
log_debug("serve parse incomplete request, remote_ip: %s ref: %d", link->remote_ip, link->ref_count);
continue;
}
link->active_time = millitime();
//FIXME
char remote_ip_port[32];
snprintf(remote_ip_port, 32, "%s:%d", link->remote_ip, link->remote_port);
this->active_links.add(remote_ip_port,(int64_t)link->active_time);
ProcJob job;
job.link = link;
this->proc(&job);
if(job.result == PROC_THREAD){
fdes->del(link->fd());
continue;
}
if(job.result == PROC_BACKEND){
fdes->del(link->fd());
this->link_count --;
char remote_ip_port[32];
snprintf(remote_ip_port, 32, "%s:%d", link->remote_ip, link->remote_port);
this->link_map.erase(remote_ip_port);
this->active_links.del(remote_ip_port);
// don't delete link
continue;
}
if(proc_result(&job, &ready_list_2) == PROC_ERROR){
//
}
} // end foreach ready link
//every event loop
destroy_idle_link();
}
}
void NetworkServer::serve(){
writer = new ProcWorkerPool("writer");
writer->start(num_writers);
reader = new ProcWorkerPool("reader");
reader->start(num_readers);
pthread_t tid;
int err = pthread_create(&tid, NULL, &NetworkServer::_ops_timer_thread, this);
if (err != 0) {
log_error("can't start ops timer thread: %s", strerror(err));
exit(-1);
}
link_dict_t ready_dict;
link_dict_t tmp_dict;
link_dict_t blocked_dict;
link_dict_t::iterator it;
const Fdevents::events_t *events;
fdes->set(serv_link->fd(), FDEVENT_IN, 0, serv_link);
fdes->set(this->reader->fd(), FDEVENT_IN, 0, this->reader);
fdes->set(this->writer->fd(), FDEVENT_IN, 0, this->writer);
uint32_t last_ticks = g_ticks;
while(!quit){
// status report
if((uint32_t)(g_ticks - last_ticks) >= STATUS_REPORT_TICKS){
last_ticks = g_ticks;
log_debug("server running, links: %d", this->link_count);
}
ready_dict.swap(tmp_dict);
tmp_dict.clear();
if(!ready_dict.empty()){
/* ready_dict not empty, so we should return immediately */
events = fdes->wait(0);
}else{
events = fdes->wait(50);
}
if(events == NULL){
log_fatal("events.wait error: %s", strerror(errno));
break;
}
for(int i=0; i<(int)events->size(); i++){
const Fdevent *fde = events->at(i);
if(fde->data.ptr == serv_link){
Link *link = accept_link();
if(link){
this->link_count ++;
log_debug("new link from %s:%d, fd: %d, links: %d",
link->remote_ip, link->remote_port, link->fd(), this->link_count);
fdes->set(link->fd(), FDEVENT_IN, 1, link);
}
}else if(fde->data.ptr == this->reader || fde->data.ptr == this->writer){
ProcWorkerPool *worker = (ProcWorkerPool *)fde->data.ptr;
ProcJob job;
if(worker->pop(&job) == 0){
log_fatal("reading result from workers error!");
exit(0);
}
if(proc_result(&job, &ready_dict) == PROC_ERROR){
//
}
}else{
proc_client_event(fde, &ready_dict);
}
}
/* if clients paused, add specified link into blocked_list and disable parsing request */
if(NetworkServer::clients_paused) {
if(NetworkServer::clients_pause_end_time < time_ms()) {
NetworkServer::clients_paused = 0;
NetworkServer::clients_pause_end_time = 0;
ready_dict.insert(blocked_dict.begin(), blocked_dict.end());
blocked_dict.clear();
} else {
blocked_dict.insert(ready_dict.begin(), ready_dict.end());
ready_dict.clear();
continue;
}
}
for(it = ready_dict.begin(); it != ready_dict.end(); it ++){
Link *link = it->second;
if(link->error()){
this->link_count --;
fdes->del(link->fd());
delete link;
continue;
}
const Request *req = link->recv();
if(req == NULL){
log_warn("fd: %d, link parse error, delete link", link->fd());
this->link_count --;
fdes->del(link->fd());
delete link;
continue;
}
if(req->empty()){
fdes->set(link->fd(), FDEVENT_IN, 1, link);
continue;
}
link->active_time = millitime();
ProcJob job;
job.link = link;
this->proc(&job);
if(job.result == PROC_THREAD){
fdes->del(link->fd());
continue;
}
if(job.result == PROC_BACKEND){
fdes->del(link->fd());
this->link_count --;
continue;
}
if(proc_result(&job, &tmp_dict) == PROC_ERROR){
//
}
} // end foreach ready link
}
}
