/*
* Processes an incoming "handle a new connection" item. This is called when
* input arrives on the libevent wakeup pipe.
*/
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
CQ_ITEM *item;
char buf[1];
if (read(fd, buf, 1) != 1)
if (settings.verbose > 0)
fprintf(stderr, "Can't read from libevent pipe\n");
item = cq_pop(me->new_conn_queue);
if (NULL != item) {
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport, me->base);
if (c == NULL) {
if (IS_UDP(item->transport)) {
fprintf(stderr, "Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
fprintf(stderr, "Can't listen for events on fd %d\n",
item->sfd);
}
close(item->sfd);
}
} else {
c->thread = me;
}
cqi_free(item);
}
}
/*
* Dispatches a new connection to another thread. This is only ever called
* from the main thread, either during initialization (for UDP) or because
* of an incoming connection.
*/
void dispatch_conn_new(int sfd, enum conn_states init_state, int event_flags,
int read_buffer_size, enum network_transport transport) {
CQ_ITEM *item = cqi_new();
char buf[1];
if (item == NULL) {
close(sfd);
/* given that malloc failed this may also fail, but let's try */
fprintf(stderr, "Failed to allocate memory for connection object\n");
return ;
}
int tid = (last_thread + 1) % settings.num_threads;
LIBEVENT_THREAD *thread = threads + tid;
last_thread = tid;
#ifdef DUP_AWARE
fprintf(stderr, "last_thread = %d, tid = %d\n", last_thread, tid);
#endif
item->sfd = sfd;
item->init_state = init_state;
item->event_flags = event_flags;
item->read_buffer_size = read_buffer_size;
item->transport = transport;
item->priority = -1; // ignored unless DUP_AWARE
#ifdef DUP_AWARE
if (!IS_UDP(transport)) {
if (settings.num_threads != NUM_THREADS) {
fprintf(stderr, "settings.num_threads must be %d for duplicate awareness"
"to work in its current implementation. You specified %d "
"threads.", NUM_THREADS, settings.num_threads);
exit(1);
}
static int is_priority = NUM_THREADS;
// The first set of `num_thread` connections will all be primary, while any
// subsequent will be duplicate.
item->priority = (is_priority-- > 0) ? PRIMARY : DUPLICATE;
fprintf(stderr, "<%d item->priority is %s\n", sfd, (item->priority == PRIMARY) ? "PRIMARY" : "DUPLICATE");
}
#endif
cq_push(thread->new_conn_queue, item);
MEMCACHED_CONN_DISPATCH(sfd, thread->thread_id);
buf[0] = 'c';
if (write(thread->notify_send_fd, buf, 1) != 1) {
perror("Writing to thread notify pipe");
}
}
/*
* Processes an incoming "handle a new connection" item. This is called when
* input arrives on the libevent wakeup pipe.
*/
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
CQ_ITEM *item;
char buf[1];
unsigned int timeout_fd;
if (read(fd, buf, 1) != 1) {
if (settings.verbose > 0)
fprintf(stderr, "Can't read from libevent pipe\n");
return;
}
switch (buf[0]) {
case 'c':
item = cq_pop(me->new_conn_queue);
if (NULL != item) {
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport,
me->base);
if (c == NULL) {
if (IS_UDP(item->transport)) {
fprintf(stderr, "Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
fprintf(stderr, "Can't listen for events on fd %d\n",
item->sfd);
}
close(item->sfd);
}
} else {
c->thread = me;
}
cqi_free(item);
}
break;
/* we were told to pause and report in */
case 'p':
register_thread_initialized();
break;
/* a client socket timed out */
case 't':
if (read(fd, &timeout_fd, sizeof(timeout_fd)) != sizeof(timeout_fd)) {
if (settings.verbose > 0)
fprintf(stderr, "Can't read timeout fd from libevent pipe\n");
return;
}
conn_close_idle(conns[timeout_fd]);
break;
}
}
//工作子线程读管道有数据到来，thread_libevent_process从读管道读取到信息
//对应的主线程写管道在dispatch_conn_new或者switch_item_lock_type
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
CQ_ITEM *item;
char buf[1];
if (read(fd, buf, 1) != 1)
if (settings.verbose > 0)
fprintf(stderr, "Can't read from libevent pipe\n");
switch (buf[0]) {
case 'c': //dispatch_conn_new
//从CQ队列中读取一个item，因为是pop所以读取后，CQ队列会把这个item从队列中删除
item = cq_pop(me->new_conn_queue);
if (NULL != item) {
//为sfd分配一个conn结构体，并且为这个sfd建立一个event，然后让base监听这个event
//这个sfd的事件回调函数是event_handler
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport, me->base);
if (c == NULL) {
if (IS_UDP(item->transport)) {
fprintf(stderr, "Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
fprintf(stderr, "Can't listen for events on fd %d\n",
item->sfd);
}
close(item->sfd);
}
} else {
c->thread = me;
}
cqi_free(item);
}
break;
//switch_item_lock_type触发走到这里
/* we were told to flip the lock type and report in */
case 'l': //参考switch_item_lock_type //切换item到段级别
//唤醒睡眠在init_cond条件变量上的迁移线程
me->item_lock_type = ITEM_LOCK_GRANULAR;
register_thread_initialized();
break;
case 'g'://切换item锁到全局级别
me->item_lock_type = ITEM_LOCK_GLOBAL;
register_thread_initialized();
break;
}
}
//worker线程的事件处理函数
//主线程分发client_fd给worker线程后，向管道写入字符，唤醒worker线程调用此函数
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
CQ_ITEM *item;
char buf[1];
if (read(fd, buf, 1) != 1)
if (settings.verbose > 0)
fprintf(stderr, "Can't read from libevent pipe\n");
switch (buf[0]) {
case 'c':
//从队列中取出主线程放入的CQ_ITEM
item = cq_pop(me->new_conn_queue);
if (NULL != item) {
//创建监听事件，把worker线程传过来的client_fd加入监听事件，使用的是worker线程的libevent
//worker线程监听两种fd，一是管道接收fd，一是client_fd
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport, me->base);
if (c == NULL) {
if (IS_UDP(item->transport)) {
fprintf(stderr, "Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
fprintf(stderr, "Can't listen for events on fd %d\n",
item->sfd);
}
close(item->sfd);
}
} else {
//设置监听连接的线程为当前worker线程
c->thread = me;
}
cqi_free(item);
}
break;
/* we were told to flip the lock type and report in */
case 'l':
me->item_lock_type = ITEM_LOCK_GRANULAR;
register_thread_initialized();
break;
case 'g':
me->item_lock_type = ITEM_LOCK_GLOBAL;
register_thread_initialized();
break;
}
}
/*
* Processes an incoming "handle a new connection" item. This is called when
* input arrives on the libevent wakeup pipe.
*
* 当管道有数据可读的时候会触发此函数的调用
*/
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
CQ_ITEM *item;
char buf[1];
if (read(fd, buf, 1) != 1)
if (settings.verbose > 0)
fprintf(stderr, "Can't read from libevent pipe\n");
switch (buf[0]) {
case 'c':
// 取出一个任务
item = cq_pop(me->new_conn_queue);
if (NULL != item) {
// 为新的请求建立一个连接结构体. 连接其实已经建立, 这里只是为了填充连接结构体. 最关键的动作是在 libevent 中注册了事件, 回调函数是 event_handler()
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport, me->base);
if (c == NULL) {
if (IS_UDP(item->transport)) {
fprintf(stderr, "Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
fprintf(stderr, "Can't listen for events on fd %d\n",
item->sfd);
}
close(item->sfd);
}
} else {
c->thread = me;
}
cqi_free(item);
}
break;
/* we were told to flip the lock type and report in */
case 'l':
me->item_lock_type = ITEM_LOCK_GRANULAR;
register_thread_initialized();
break;
case 'g':
me->item_lock_type = ITEM_LOCK_GLOBAL;
register_thread_initialized();
break;
}
}
void thread_libevent_process(int fd,short which,void *arg) //处理函数，即当主线程通知workerThread时，主线程会插入一个item到某个thread的queue中，queue是一个工具类
//workerThread将item pop出来并封装为conn，封装期间就建立了和item所指向的对象的联系，也使用libevent完成
{
LIBEVENT_THREAD *me=(LIBEVENT_THREAD*)arg;
CQ_ITEM *item;
char buf[1];
if(read(fd,buf,1)!=1)
fprintf(stderr,"can't read from libevent pipe\n");
item=cq_pop(me->new_conn_queue);
printf("process item fd is:%d\n",item->sfd);
if(NULL!=item)
{
conn *c= conn_new (item->sfd,item->init_state,item->event_flags,
item->read_buffer_size,item->transport,me->base);
if(NULL==c)
{
if( IS_UDP(item->transport))
{
fprintf(stderr,"can't listen for events on UDP\n");
exit(1);
}
else
{
fprintf(stderr,"can't listen for events on fd %d\n",item->sfd);
close(item->sfd);
}
}
else
{
printf("%lu get content\n",pthread_self());
printf("sfd:%d,read_size:%d\n",item->sfd,item->read_buffer_size);
c->thread=me;
//test
write(item->read_buffer_size,"abc",3);
}
cqi_free(item);
}
}
/*
* Processes an incoming "handle a new connection" item. This is called when
* input arrives on the libevent wakeup pipe.
*/
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
CQ_ITEM *item;
char buf[1];
//响应pipe可读事件，读取主线程向管道内写的1字节数据(见dispatch_conn_new()函数)
if (read(fd, buf, 1) != 1)
if (settings.verbose > 0)
fprintf(stderr, "Can't read from libevent pipe\n");
switch (buf[0]) {
case 'c':
//从链接队列中取出一个conn
item = cq_pop(me->new_conn_queue);
if (NULL != item) {
//使用conn创建新的任务，并注册事件
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport, me->base);
if (c == NULL) {
if (IS_UDP(item->transport)) {
fprintf(stderr, "Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
fprintf(stderr, "Can't listen for events on fd %d\n",
item->sfd);
}
close(item->sfd);
}
} else {
c->thread = me;
}
cqi_free(item);
}
break;
/* we were told to pause and report in */
case 'p':
register_thread_initialized();
break;
}
}
static void thread_libevent_process(int fd, short which, void *arg) {
thread_cg *me = arg;
char buf[1];
CQ_ITEM *item;
if (read(fd, buf, 1) != 1)
{
perror("thread_libevent_process read error");
exit(1);
}
switch(buf[0]){
case 'c':
//从连接管理中，pop出来一个连接，将连接(conn_new)注册到event_handler来处理
item = cq_pop(me->new_conn_queue);
if (NULL != item) {
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport, me->base);
if (c == NULL) {
if (IS_UDP(item->transport)) {
fprintf(stderr, "Can't listen for events on UDP socket\n");
exit(1);
} else {
fprintf(stderr, "Can't listen for events on fd %d\n",
item->sfd);
close(item->sfd);
}
} else {
c->thread = me;
}
cqi_free(item);
}
break;
case 'p':
break;
}
}
/*
* Processes an incoming "handle a new connection" item. This is called when
* input arrives on the libevent wakeup pipe.
*/
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
CQ_ITEM *cq_item;
char buf[1];
(void)which;
if (read(fd, buf, 1) != 1)
if (settings.verbose > 0)
moxi_log_write("Can't read from libevent pipe\n");
cq_item = cq_pop(me->new_conn_queue);
if (NULL != cq_item) {
conn *c = conn_new(cq_item->sfd, cq_item->init_state, cq_item->event_flags,
cq_item->read_buffer_size,
cq_item->transport,
me->base,
cq_item->funcs, cq_item->extra);
if (c == NULL) {
if (IS_UDP(cq_item->transport)) {
moxi_log_write("Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
moxi_log_write("Can't listen for events on fd %d\n",
cq_item->sfd);
}
close(cq_item->sfd);
}
} else {
c->protocol = cq_item->protocol;
c->thread = me;
}
cqi_free(cq_item);
}
}
static int server_socket(const char *interface,
int port,
enum network_transport transport,
FILE *portnumber_file){
/*
该函数主要作用：
1.监听端口，注册监听端口事件.
2.一旦有端口有数据，建立连接，将连接放到连接池中，conn_new为每一个连接注册时间监听
3.通知线程有连接需要处理
*/
int sfd;
struct linger ling = {0, 0};
struct addrinfo *ai;
struct addrinfo *next;
struct addrinfo hints = { .ai_flags = AI_PASSIVE,
.ai_family = AF_UNSPEC };
char port_buf[NI_MAXSERV];
int error;
int success = 0;
int flags =1;
hints.ai_socktype = SOCK_STREAM;
if (port == -1) {
port = 0;
}
snprintf(port_buf, sizeof(port_buf), "%d", port);
error= getaddrinfo(interface, port_buf, &hints, &ai);
if (error != 0) {
if (error != EAI_SYSTEM)
fprintf(stderr, "getaddrinfo(): %s\n", gai_strerror(error));
else
perror("getaddrinfo()");
return 1;
}
for (next= ai; next; next= next->ai_next) {
conn *listen_conn_add;
if ((sfd = new_socket(next)) == -1) {
/* getaddrinfo can return "junk" addresses,
* we make sure at least one works before erroring.
*/
if (errno == EMFILE) {
/* ...unless we're out of fds */
perror("server_socket");
exit(EX_OSERR);
}
continue;
}
setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (void *)&flags, sizeof(flags));
error = setsockopt(sfd, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags));
if (error != 0)
perror("setsockopt");
error = setsockopt(sfd, SOL_SOCKET, SO_LINGER, (void *)&ling, sizeof(ling));
if (error != 0)
perror("setsockopt");
error = setsockopt(sfd, IPPROTO_TCP, TCP_NODELAY, (void *)&flags, sizeof(flags));
if (error != 0)
perror("setsockopt");
if (bind(sfd, next->ai_addr, next->ai_addrlen) == -1) {
if (errno != EADDRINUSE) {
perror("bind()");
close(sfd);
freeaddrinfo(ai);
return 1;
}
close(sfd);
continue;
} else {
success++;
if (!IS_UDP(transport) && listen(sfd, 1024) == -1) {
perror("listen()");
close(sfd);
freeaddrinfo(ai);
return 1;
}
if (portnumber_file != NULL &&
(next->ai_addr->sa_family == AF_INET ||
next->ai_addr->sa_family == AF_INET6)) {
union {
struct sockaddr_in in;
struct sockaddr_in6 in6;
} my_sockaddr;
socklen_t len = sizeof(my_sockaddr);
if (getsockname(sfd, (struct sockaddr*)&my_sockaddr, &len)==0) {
if (next->ai_addr->sa_family == AF_INET) {
fprintf(portnumber_file, "%s INET: %u\n",
IS_UDP(transport) ? "UDP" : "TCP",
ntohs(my_sockaddr.in.sin_port));
} else {
fprintf(portnumber_file, "%s INET6: %u\n",
IS_UDP(transport) ? "UDP" : "TCP",
ntohs(my_sockaddr.in6.sin6_port));
}
}
}
}
if (!(listen_conn_add = conn_new(sfd, conn_listening,
EV_READ | EV_PERSIST, 1,
transport, main_base))) {
fprintf(stderr, "failed to create listening connection\n");
exit(EXIT_FAILURE);
}
listen_conn_add->next = listen_conn;
listen_conn = listen_conn_add;
}
freeaddrinfo(ai);
/* Return zero iff we detected no errors in starting up connections */
return success == 0;
}
/*
* Processes an incoming "handle a new connection" item. This is called when
* input arrives on the libevent wakeup pipe.
*/
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
assert(me->type == GENERAL);
CQ_ITEM *item;
if (recv(fd, devnull, sizeof(devnull), 0) == -1) {
if (settings.verbose > 0) {
settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL,
"Can't read from libevent pipe: %s\n",
strerror(errno));
}
}
if (memcached_shutdown) {
event_base_loopbreak(me->base);
return ;
}
while ((item = cq_pop(me->new_conn_queue)) != NULL) {
conn *c = conn_new(item->sfd, item->parent_port, item->init_state,
item->event_flags, item->read_buffer_size,
item->transport, me->base, NULL);
if (c == NULL) {
if (IS_UDP(item->transport)) {
settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL,
"Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
settings.extensions.logger->log(EXTENSION_LOG_INFO, NULL,
"Can't listen for events on fd %d\n",
item->sfd);
}
closesocket(item->sfd);
}
} else {
assert(c->thread == NULL);
c->thread = me;
}
cqi_free(item);
}
LOCK_THREAD(me);
conn* pending = me->pending_io;
me->pending_io = NULL;
while (pending != NULL) {
conn *c = pending;
assert(me == c->thread);
pending = pending->next;
c->next = NULL;
if (c->sfd != INVALID_SOCKET && !c->registered_in_libevent) {
// The socket may have been shut down while we're looping
// in delayed shutdown
register_event(c, 0);
}
/*
* We don't want the thread to keep on serving all of the data
* from the context of the notification pipe, so just let it
* run one time to set up the correct mask in libevent
*/
c->nevents = 1;
do {
if (settings.verbose) {
settings.extensions.logger->log(EXTENSION_LOG_DEBUG, c,
"%d - Running task: (%s)\n",
c->sfd, state_text(c->state));
}
} while (c->state(c));
}
UNLOCK_THREAD(me);
}
/*
* Processes an incoming "handle a new connection" item. This is called when
* input arrives on the libevent wakeup pipe.
*/
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
assert(me->type == GENERAL);
CQ_ITEM *item;
if (recv(fd, devnull, sizeof(devnull), 0) == -1) {
if (settings.verbose > 0) {
settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL,
"Can't read from libevent pipe: %s\n",
strerror(errno));
}
}
if (memcached_shutdown) {
event_base_loopbreak(me->base);
return ;
}
while ((item = cq_pop(me->new_conn_queue)) != NULL) {
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport, me->base,
NULL);
if (c == NULL) {
if (IS_UDP(item->transport)) {
settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL,
"Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
settings.extensions.logger->log(EXTENSION_LOG_INFO, NULL,
"Can't listen for events on fd %d\n",
item->sfd);
}
closesocket(item->sfd);
}
} else {
assert(c->thread == NULL);
c->thread = me;
}
cqi_free(item);
}
pthread_mutex_lock(&me->mutex);
conn* pending = me->pending_io;
me->pending_io = NULL;
pthread_mutex_unlock(&me->mutex);
while (pending != NULL) {
conn *c = pending;
assert(me == c->thread);
pending = pending->next;
c->next = NULL;
register_event(c, 0);
/*
* We don't want the thread to keep on serving all of the data
* from the context of the notification pipe, so just let it
* run one time to set up the correct mask in libevent
*/
c->nevents = 1;
/* c->nevents = settings.reqs_per_event; */
while (c->state(c)) {
/* do task */
}
}
}
/*
* Processes an incoming "handle a new connection" item. This is called when
* input arrives on the libevent wakeup pipe.
*/
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
assert(me->type == GENERAL);
CQ_ITEM *item;
char buf[1];
if (memcached_shutdown > 1) {
if (settings.verbose > 0) {
mc_logger->log(EXTENSION_LOG_INFO, NULL,
"Worker thread[%d] is now terminating from libevent process.\n",
me->index);
}
event_base_loopbreak(me->base);
return;
}
if (read(fd, buf, 1) != 1) {
if (settings.verbose > 0) {
mc_logger->log(EXTENSION_LOG_WARNING, NULL,
"Can't read from libevent pipe: %s\n", strerror(errno));
}
}
item = cq_pop(me->new_conn_queue);
if (NULL != item) {
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport, me->base, NULL);
if (c == NULL) {
if (IS_UDP(item->transport)) {
mc_logger->log(EXTENSION_LOG_WARNING, NULL,
"Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
mc_logger->log(EXTENSION_LOG_INFO, NULL,
"Can't listen for events on fd %d\n", item->sfd);
}
close(item->sfd);
}
} else {
assert(c->thread == NULL);
c->thread = me;
/* link to the conn_list of the thread */
if (me->conn_list != NULL) {
c->conn_next = me->conn_list;
me->conn_list->conn_prev = c;
}
me->conn_list = c;
}
cqi_free(item);
}
pthread_mutex_lock(&me->mutex);
conn* pending = me->pending_io;
me->pending_io = NULL;
pthread_mutex_unlock(&me->mutex);
while (pending != NULL) {
conn *c = pending;
assert(me == c->thread);
pending = pending->next;
c->next = NULL;
event_add(&c->event, 0);
c->nevents = settings.reqs_per_event;
while (c->state(c)) {
/* do task */
}
}
}