/* * Worker thread: main event loop */ static void *worker_libevent(void *arg) { LIBEVENT_THREAD *me = arg; struct conn *conn; CQ_ITEM *item; /* Any per-thread setup can happen here; thread_init() will block until * all threads have finished initializing. */ pthread_mutex_lock(&init_lock); init_count++; pthread_cond_signal(&init_cond); pthread_mutex_unlock(&init_lock); event_base_loop(me->base, 0); /* close all connections */ conn = me->conn_list; while (conn != NULL) { close(conn->sfd); conn = conn->conn_next; } item = cq_pop(me->new_conn_queue); while (item != NULL) { close(item->sfd); cqi_free(item); item = cq_pop(me->new_conn_queue); } return NULL; }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; CQ_ITEM *item; char buf[1]; if (read(fd, buf, 1) != 1) if (settings.verbose > 0) fprintf(stderr, "Can't read from libevent pipe\n"); item = cq_pop(me->new_conn_queue); if (NULL != item) { conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base); if (c == NULL) { if (IS_UDP(item->transport)) { fprintf(stderr, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { fprintf(stderr, "Can't listen for events on fd %d\n", item->sfd); } close(item->sfd); } } else { c->thread = me; } cqi_free(item); } }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(evutil_socket_t fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; CQ_ITEM *item; conn* pending; cb_assert(me->type == GENERAL); drain_notification_channel(fd); if (memcached_shutdown) { event_base_loopbreak(me->base); return ; } while ((item = cq_pop(me->new_conn_queue)) != NULL) { conn *c = conn_new(item->sfd, item->parent_port, item->init_state, item->event_flags, item->read_buffer_size, me->base); if (c == NULL) { if (settings.verbose > 0) { settings.extensions.logger->log(EXTENSION_LOG_INFO, NULL, "Can't listen for events on fd %d\n", item->sfd); } closesocket(item->sfd); } else { cb_assert(c->thread == NULL); c->thread = me; } cqi_free(item); } LOCK_THREAD(me); pending = me->pending_io; me->pending_io = NULL; while (pending != NULL) { conn *c = pending; cb_assert(me == c->thread); pending = pending->next; c->next = NULL; if (c->sfd != INVALID_SOCKET && !c->registered_in_libevent) { /* The socket may have been shut down while we're looping */ /* in delayed shutdown */ register_event(c, 0); } /* * We don't want the thread to keep on serving all of the data * from the context of the notification pipe, so just let it * run one time to set up the correct mask in libevent */ c->nevents = 1; run_event_loop(c); } UNLOCK_THREAD(me); }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; CQ_ITEM *item; char buf[1]; unsigned int timeout_fd; if (read(fd, buf, 1) != 1) { if (settings.verbose > 0) fprintf(stderr, "Can't read from libevent pipe\n"); return; } switch (buf[0]) { case 'c': item = cq_pop(me->new_conn_queue); if (NULL != item) { conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base); if (c == NULL) { if (IS_UDP(item->transport)) { fprintf(stderr, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { fprintf(stderr, "Can't listen for events on fd %d\n", item->sfd); } close(item->sfd); } } else { c->thread = me; } cqi_free(item); } break; /* we were told to pause and report in */ case 'p': register_thread_initialized(); break; /* a client socket timed out */ case 't': if (read(fd, &timeout_fd, sizeof(timeout_fd)) != sizeof(timeout_fd)) { if (settings.verbose > 0) fprintf(stderr, "Can't read timeout fd from libevent pipe\n"); return; } conn_close_idle(conns[timeout_fd]); break; } }
//工作子线程读管道有数据到来,thread_libevent_process从读管道读取到信息 //对应的主线程写管道在dispatch_conn_new或者switch_item_lock_type static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; CQ_ITEM *item; char buf[1]; if (read(fd, buf, 1) != 1) if (settings.verbose > 0) fprintf(stderr, "Can't read from libevent pipe\n"); switch (buf[0]) { case 'c': //dispatch_conn_new //从CQ队列中读取一个item,因为是pop所以读取后,CQ队列会把这个item从队列中删除 item = cq_pop(me->new_conn_queue); if (NULL != item) { //为sfd分配一个conn结构体,并且为这个sfd建立一个event,然后让base监听这个event //这个sfd的事件回调函数是event_handler conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base); if (c == NULL) { if (IS_UDP(item->transport)) { fprintf(stderr, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { fprintf(stderr, "Can't listen for events on fd %d\n", item->sfd); } close(item->sfd); } } else { c->thread = me; } cqi_free(item); } break; //switch_item_lock_type触发走到这里 /* we were told to flip the lock type and report in */ case 'l': //参考switch_item_lock_type //切换item到段级别 //唤醒睡眠在init_cond条件变量上的迁移线程 me->item_lock_type = ITEM_LOCK_GRANULAR; register_thread_initialized(); break; case 'g'://切换item锁到全局级别 me->item_lock_type = ITEM_LOCK_GLOBAL; register_thread_initialized(); break; } }
//worker线程的事件处理函数 //主线程分发client_fd给worker线程后,向管道写入字符,唤醒worker线程调用此函数 static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; CQ_ITEM *item; char buf[1]; if (read(fd, buf, 1) != 1) if (settings.verbose > 0) fprintf(stderr, "Can't read from libevent pipe\n"); switch (buf[0]) { case 'c': //从队列中取出主线程放入的CQ_ITEM item = cq_pop(me->new_conn_queue); if (NULL != item) { //创建监听事件,把worker线程传过来的client_fd加入监听事件,使用的是worker线程的libevent //worker线程监听两种fd,一是管道接收fd,一是client_fd conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base); if (c == NULL) { if (IS_UDP(item->transport)) { fprintf(stderr, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { fprintf(stderr, "Can't listen for events on fd %d\n", item->sfd); } close(item->sfd); } } else { //设置监听连接的线程为当前worker线程 c->thread = me; } cqi_free(item); } break; /* we were told to flip the lock type and report in */ case 'l': me->item_lock_type = ITEM_LOCK_GRANULAR; register_thread_initialized(); break; case 'g': me->item_lock_type = ITEM_LOCK_GLOBAL; register_thread_initialized(); break; } }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. * * 当管道有数据可读的时候会触发此函数的调用 */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; CQ_ITEM *item; char buf[1]; if (read(fd, buf, 1) != 1) if (settings.verbose > 0) fprintf(stderr, "Can't read from libevent pipe\n"); switch (buf[0]) { case 'c': // 取出一个任务 item = cq_pop(me->new_conn_queue); if (NULL != item) { // 为新的请求建立一个连接结构体. 连接其实已经建立, 这里只是为了填充连接结构体. 最关键的动作是在 libevent 中注册了事件, 回调函数是 event_handler() conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base); if (c == NULL) { if (IS_UDP(item->transport)) { fprintf(stderr, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { fprintf(stderr, "Can't listen for events on fd %d\n", item->sfd); } close(item->sfd); } } else { c->thread = me; } cqi_free(item); } break; /* we were told to flip the lock type and report in */ case 'l': me->item_lock_type = ITEM_LOCK_GRANULAR; register_thread_initialized(); break; case 'g': me->item_lock_type = ITEM_LOCK_GLOBAL; register_thread_initialized(); break; } }
void thread_libevent_process(int fd,short which,void *arg) //处理函数,即当主线程通知workerThread时,主线程会插入一个item到某个thread的queue中,queue是一个工具类 //workerThread将item pop出来并封装为conn,封装期间就建立了和item所指向的对象的联系,也使用libevent完成 { LIBEVENT_THREAD *me=(LIBEVENT_THREAD*)arg; CQ_ITEM *item; char buf[1]; if(read(fd,buf,1)!=1) fprintf(stderr,"can't read from libevent pipe\n"); item=cq_pop(me->new_conn_queue); printf("process item fd is:%d\n",item->sfd); if(NULL!=item) { conn *c= conn_new (item->sfd,item->init_state,item->event_flags, item->read_buffer_size,item->transport,me->base); if(NULL==c) { if( IS_UDP(item->transport)) { fprintf(stderr,"can't listen for events on UDP\n"); exit(1); } else { fprintf(stderr,"can't listen for events on fd %d\n",item->sfd); close(item->sfd); } } else { printf("%lu get content\n",pthread_self()); printf("sfd:%d,read_size:%d\n",item->sfd,item->read_buffer_size); c->thread=me; //test write(item->read_buffer_size,"abc",3); } cqi_free(item); } }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; CQ_ITEM *item; char buf[1]; //响应pipe可读事件,读取主线程向管道内写的1字节数据(见dispatch_conn_new()函数) if (read(fd, buf, 1) != 1) if (settings.verbose > 0) fprintf(stderr, "Can't read from libevent pipe\n"); switch (buf[0]) { case 'c': //从链接队列中取出一个conn item = cq_pop(me->new_conn_queue); if (NULL != item) { //使用conn创建新的任务,并注册事件 conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base); if (c == NULL) { if (IS_UDP(item->transport)) { fprintf(stderr, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { fprintf(stderr, "Can't listen for events on fd %d\n", item->sfd); } close(item->sfd); } } else { c->thread = me; } cqi_free(item); } break; /* we were told to pause and report in */ case 'p': register_thread_initialized(); break; } }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = (LIBEVENT_THREAD *)arg; CQ_ITEM *item; char buf[1]; if (read(fd, buf, 1) != 1) if (g_settings.verbose > 0) { log_debug(LOG_ERR, "Can't read from libevent pipe\n"); } if (buf[0] != 'c') { return; } item = cq_pop(me->new_conn_queue); if (NULL != item) { conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base); if (c == NULL) { if (g_settings.verbose > 0) { log_debug(LOG_ERR, "Can't listen for events on fd %d\n", item->sfd); } close(item->sfd); } else { c->thread = me; } cqi_free(item); } }
void threads_shutdown(void) { for (int ii = 0; ii < nthreads; ++ii) { notify_thread(&threads[ii]); pthread_join(thread_ids[ii], NULL); } for (int ii = 0; ii < nthreads; ++ii) { safe_close(threads[ii].notify[0]); safe_close(threads[ii].notify[1]); cache_destroy(threads[ii].suffix_cache); event_base_free(threads[ii].base); CQ_ITEM *it; while ((it = cq_pop(threads[ii].new_conn_queue)) != NULL) { cqi_free(it); } free(threads[ii].new_conn_queue); } free(thread_ids); free(threads); }
void threads_cleanup(void) { int ii; for (ii = 0; ii < nthreads; ++ii) { CQ_ITEM *it; safe_close(threads[ii].notify[0]); safe_close(threads[ii].notify[1]); event_base_free(threads[ii].base); while ((it = cq_pop(threads[ii].new_conn_queue)) != NULL) { cqi_free(it); } free(threads[ii].new_conn_queue); free(threads[ii].read.buf); free(threads[ii].write.buf); subdoc_op_free(threads[ii].subdoc_op); } free(thread_ids); free(threads); }
static void thread_libevent_process(int fd, short which, void *arg) { thread_cg *me = arg; char buf[1]; CQ_ITEM *item; if (read(fd, buf, 1) != 1) { perror("thread_libevent_process read error"); exit(1); } switch(buf[0]){ case 'c': //从连接管理中,pop出来一个连接,将连接(conn_new)注册到event_handler来处理 item = cq_pop(me->new_conn_queue); if (NULL != item) { conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base); if (c == NULL) { if (IS_UDP(item->transport)) { fprintf(stderr, "Can't listen for events on UDP socket\n"); exit(1); } else { fprintf(stderr, "Can't listen for events on fd %d\n", item->sfd); close(item->sfd); } } else { c->thread = me; } cqi_free(item); } break; case 'p': break; } }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; CQ_ITEM *cq_item; char buf[1]; (void)which; if (read(fd, buf, 1) != 1) if (settings.verbose > 0) moxi_log_write("Can't read from libevent pipe\n"); cq_item = cq_pop(me->new_conn_queue); if (NULL != cq_item) { conn *c = conn_new(cq_item->sfd, cq_item->init_state, cq_item->event_flags, cq_item->read_buffer_size, cq_item->transport, me->base, cq_item->funcs, cq_item->extra); if (c == NULL) { if (IS_UDP(cq_item->transport)) { moxi_log_write("Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { moxi_log_write("Can't listen for events on fd %d\n", cq_item->sfd); } close(cq_item->sfd); } } else { c->protocol = cq_item->protocol; c->thread = me; } cqi_free(cq_item); } }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = (LIBEVENT_THREAD *)arg; CQ_ITEM *item; char buf[1]; if (read(fd, buf, 1) != 1) if (settings.verbose > 0) fprintf(stderr, "Can't read from libevent pipe\n"); switch (buf[0]) { case 'c': item = cq_pop(me->new_conn_queue); if (NULL != item) { conn_process(item, me); } cqi_free(item); break; } /* we were told to flip the lock type and report in */ }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; assert(me->type == GENERAL); CQ_ITEM *item; if (recv(fd, devnull, sizeof(devnull), 0) == -1) { if (settings.verbose > 0) { settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL, "Can't read from libevent pipe: %s\n", strerror(errno)); } } if (memcached_shutdown) { event_base_loopbreak(me->base); return ; } while ((item = cq_pop(me->new_conn_queue)) != NULL) { conn *c = conn_new(item->sfd, item->parent_port, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base, NULL); if (c == NULL) { if (IS_UDP(item->transport)) { settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { settings.extensions.logger->log(EXTENSION_LOG_INFO, NULL, "Can't listen for events on fd %d\n", item->sfd); } closesocket(item->sfd); } } else { assert(c->thread == NULL); c->thread = me; } cqi_free(item); } LOCK_THREAD(me); conn* pending = me->pending_io; me->pending_io = NULL; while (pending != NULL) { conn *c = pending; assert(me == c->thread); pending = pending->next; c->next = NULL; if (c->sfd != INVALID_SOCKET && !c->registered_in_libevent) { // The socket may have been shut down while we're looping // in delayed shutdown register_event(c, 0); } /* * We don't want the thread to keep on serving all of the data * from the context of the notification pipe, so just let it * run one time to set up the correct mask in libevent */ c->nevents = 1; do { if (settings.verbose) { settings.extensions.logger->log(EXTENSION_LOG_DEBUG, c, "%d - Running task: (%s)\n", c->sfd, state_text(c->state)); } } while (c->state(c)); } UNLOCK_THREAD(me); }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; assert(me->type == GENERAL); CQ_ITEM *item; char buf[1]; if (memcached_shutdown > 1) { if (settings.verbose > 0) { mc_logger->log(EXTENSION_LOG_INFO, NULL, "Worker thread[%d] is now terminating from libevent process.\n", me->index); } event_base_loopbreak(me->base); return; } if (read(fd, buf, 1) != 1) { if (settings.verbose > 0) { mc_logger->log(EXTENSION_LOG_WARNING, NULL, "Can't read from libevent pipe: %s\n", strerror(errno)); } } item = cq_pop(me->new_conn_queue); if (NULL != item) { conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base, NULL); if (c == NULL) { if (IS_UDP(item->transport)) { mc_logger->log(EXTENSION_LOG_WARNING, NULL, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { mc_logger->log(EXTENSION_LOG_INFO, NULL, "Can't listen for events on fd %d\n", item->sfd); } close(item->sfd); } } else { assert(c->thread == NULL); c->thread = me; /* link to the conn_list of the thread */ if (me->conn_list != NULL) { c->conn_next = me->conn_list; me->conn_list->conn_prev = c; } me->conn_list = c; } cqi_free(item); } pthread_mutex_lock(&me->mutex); conn* pending = me->pending_io; me->pending_io = NULL; pthread_mutex_unlock(&me->mutex); while (pending != NULL) { conn *c = pending; assert(me == c->thread); pending = pending->next; c->next = NULL; event_add(&c->event, 0); c->nevents = settings.reqs_per_event; while (c->state(c)) { /* do task */ } } }
static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = (LIBEVENT_THREAD *)arg; CQ_ITEM *item; char buf[1]; if (read(fd, buf, 1) != 1) merror("can't read from libevent pipe!"); switch(buf[0]) { case 'c': { item = cq_pop(me->new_conn_queue); if (NULL != item) { listener_info *li = (listener_info *)item->data; conn *c = conn_new(); if (NULL == c) { } else { struct bufferevent* bev = bufferevent_socket_new(me->base, item->fd, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS); if (NULL == bev) { merror("create bufferevent failed!"); } else { strncpy(c->addrtext, li->addrtext, 32); evbuffer *input = bufferevent_get_input(bev); evbuffer_enable_locking(input, NULL); bufferevent_setcb(bev, conn_read_cb, conn_write_cb, conn_event_cb, c); bufferevent_enable(bev, EV_READ); c->data = li->l; c->bev = bev; c->thread = me; mdebug("new connection %s established!", c->addrtext); } } free(item->data); cqi_free(item); } } break; case 't': { item = cq_pop(me->new_conn_queue); if (NULL != item) { connector *cr = (connector *)item->data; conn *c = conn_new(); if (NULL == c) { } else { struct bufferevent *bev = bufferevent_socket_new(me->base, -1, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS); if (NULL == bev) { merror("create bufferevent failed!"); } else { evbuffer *input = bufferevent_get_input(bev); evbuffer_enable_locking(input, NULL); bufferevent_setcb(bev, NULL, NULL, connecting_event_cb, c); c->bev = bev; c->data = cr; c->thread = me; cr->c = c; cr->state = STATE_NOT_CONNECTED; minfo("connecting %s!", cr->addrtext); bufferevent_socket_connect(c->bev, cr->sa, cr->socklen); } } cqi_free(item); } } break; case 'k': { event_base_loopbreak(me->base); } break; } }
/* * Processes an incoming "handle a new connection" item. This is called when * input arrives on the libevent wakeup pipe. */ static void thread_libevent_process(int fd, short which, void *arg) { LIBEVENT_THREAD *me = arg; assert(me->type == GENERAL); CQ_ITEM *item; if (recv(fd, devnull, sizeof(devnull), 0) == -1) { if (settings.verbose > 0) { settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL, "Can't read from libevent pipe: %s\n", strerror(errno)); } } if (memcached_shutdown) { event_base_loopbreak(me->base); return ; } while ((item = cq_pop(me->new_conn_queue)) != NULL) { conn *c = conn_new(item->sfd, item->init_state, item->event_flags, item->read_buffer_size, item->transport, me->base, NULL); if (c == NULL) { if (IS_UDP(item->transport)) { settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL, "Can't listen for events on UDP socket\n"); exit(1); } else { if (settings.verbose > 0) { settings.extensions.logger->log(EXTENSION_LOG_INFO, NULL, "Can't listen for events on fd %d\n", item->sfd); } closesocket(item->sfd); } } else { assert(c->thread == NULL); c->thread = me; } cqi_free(item); } pthread_mutex_lock(&me->mutex); conn* pending = me->pending_io; me->pending_io = NULL; pthread_mutex_unlock(&me->mutex); while (pending != NULL) { conn *c = pending; assert(me == c->thread); pending = pending->next; c->next = NULL; register_event(c, 0); /* * We don't want the thread to keep on serving all of the data * from the context of the notification pipe, so just let it * run one time to set up the correct mask in libevent */ c->nevents = 1; /* c->nevents = settings.reqs_per_event; */ while (c->state(c)) { /* do task */ } } }