static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg,
grpc_error *error_unused) {
grpc_error *error = GRPC_ERROR_NONE;
grpc_pollset *pollset = arg;
gpr_mu_lock(&pollset->pollable.po.mu);
if (pollset->root_worker != NULL) {
grpc_pollset_worker *worker = pollset->root_worker;
do {
if (worker->pollable != &pollset->pollable) {
gpr_mu_lock(&worker->pollable->po.mu);
}
if (worker->initialized_cv) {
worker->kicked = true;
gpr_cv_signal(&worker->cv);
} else {
append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup),
"pollset_shutdown");
}
if (worker->pollable != &pollset->pollable) {
gpr_mu_unlock(&worker->pollable->po.mu);
}
worker = worker->links[PWL_POLLSET].next;
} while (worker != pollset->root_worker);
}
pollset->kick_alls_pending--;
pollset_maybe_finish_shutdown(exec_ctx, pollset);
gpr_mu_unlock(&pollset->pollable.po.mu);
GRPC_LOG_IF_ERROR("kick_all", error);
}
static void wakeup(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {
GPR_TIMER_MARK("workqueue.wakeup", 0);
grpc_error *err = grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd);
if (!GRPC_LOG_IF_ERROR("wakeupfd_wakeup", err)) {
drain(exec_ctx, workqueue);
}
}
static void test_threading_wakeup(grpc_exec_ctx *exec_ctx, void *arg,
grpc_error *error) {
threading_shared *shared = arg;
++shared->wakeups;
++thread_wakeups;
if (error == GRPC_ERROR_NONE) {
GPR_ASSERT(GRPC_LOG_IF_ERROR(
"consume_wakeup", grpc_wakeup_fd_consume_wakeup(shared->wakeup_fd)));
grpc_fd_notify_on_read(exec_ctx, shared->wakeup_desc, &shared->on_wakeup);
GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_next",
grpc_wakeup_fd_wakeup(shared->wakeup_fd)));
}
}
void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) {
/* pollset->mu already held */
if (specific_worker != NULL) {
if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) {
for (specific_worker = p->root_worker.next;
specific_worker != &p->root_worker;
specific_worker = specific_worker->next) {
grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd);
}
p->kicked_without_pollers = 1;
} else if (gpr_tls_get(&g_current_thread_worker) !=
(gpr_intptr)specific_worker) {
grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd);
}
} else if (gpr_tls_get(&g_current_thread_poller) != (gpr_intptr)p) {
specific_worker = pop_front_worker(p);
if (specific_worker != NULL) {
push_back_worker(p, specific_worker);
grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd);
} else {
p->kicked_without_pollers = 1;
}
}
}
static void test_threading(void) {
threading_shared shared;
shared.pollset = gpr_zalloc(grpc_pollset_size());
grpc_pollset_init(shared.pollset, &shared.mu);
gpr_thd_id thds[10];
for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) {
gpr_thd_options opt = gpr_thd_options_default();
gpr_thd_options_set_joinable(&opt);
gpr_thd_new(&thds[i], test_threading_loop, &shared, &opt);
}
grpc_wakeup_fd fd;
GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_fd_init", grpc_wakeup_fd_init(&fd)));
shared.wakeup_fd = &fd;
shared.wakeup_desc = grpc_fd_create(fd.read_fd, "wakeup");
shared.wakeups = 0;
{
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
grpc_pollset_add_fd(&exec_ctx, shared.pollset, shared.wakeup_desc);
grpc_fd_notify_on_read(
&exec_ctx, shared.wakeup_desc,
GRPC_CLOSURE_INIT(&shared.on_wakeup, test_threading_wakeup, &shared,
grpc_schedule_on_exec_ctx));
grpc_exec_ctx_finish(&exec_ctx);
}
GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_first",
grpc_wakeup_fd_wakeup(shared.wakeup_fd)));
for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) {
gpr_thd_join(thds[i]);
}
fd.read_fd = 0;
grpc_wakeup_fd_destroy(&fd);
{
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
grpc_fd_shutdown(&exec_ctx, shared.wakeup_desc, GRPC_ERROR_CANCELLED);
grpc_fd_orphan(&exec_ctx, shared.wakeup_desc, NULL, NULL,
false /* already_closed */, "done");
grpc_pollset_shutdown(&exec_ctx, shared.pollset,
GRPC_CLOSURE_CREATE(destroy_pollset, shared.pollset,
grpc_schedule_on_exec_ctx));
grpc_exec_ctx_finish(&exec_ctx);
}
gpr_free(shared.pollset);
}
void grpc_kick_poller(void) {
grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd);
}
void grpc_pollset_kick_ext(grpc_pollset *p,
grpc_pollset_worker *specific_worker,
uint32_t flags) {
GPR_TIMER_BEGIN("grpc_pollset_kick_ext", 0);
/* pollset->mu already held */
if (specific_worker != NULL) {
if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) {
GPR_TIMER_BEGIN("grpc_pollset_kick_ext.broadcast", 0);
GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
for (specific_worker = p->root_worker.next;
specific_worker != &p->root_worker;
specific_worker = specific_worker->next) {
grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
}
p->kicked_without_pollers = 1;
GPR_TIMER_END("grpc_pollset_kick_ext.broadcast", 0);
} else if (gpr_tls_get(&g_current_thread_worker) !=
(intptr_t)specific_worker) {
GPR_TIMER_MARK("different_thread_worker", 0);
if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
specific_worker->reevaluate_polling_on_wakeup = 1;
}
specific_worker->kicked_specifically = 1;
grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
} else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) {
GPR_TIMER_MARK("kick_yoself", 0);
if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) {
specific_worker->reevaluate_polling_on_wakeup = 1;
}
specific_worker->kicked_specifically = 1;
grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
}
} else if (gpr_tls_get(&g_current_thread_poller) != (intptr_t)p) {
GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0);
GPR_TIMER_MARK("kick_anonymous", 0);
specific_worker = pop_front_worker(p);
if (specific_worker != NULL) {
if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) {
GPR_TIMER_MARK("kick_anonymous_not_self", 0);
push_back_worker(p, specific_worker);
specific_worker = pop_front_worker(p);
if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 &&
gpr_tls_get(&g_current_thread_worker) ==
(intptr_t)specific_worker) {
push_back_worker(p, specific_worker);
specific_worker = NULL;
}
}
if (specific_worker != NULL) {
GPR_TIMER_MARK("finally_kick", 0);
push_back_worker(p, specific_worker);
grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd);
}
} else {
GPR_TIMER_MARK("kicked_no_pollers", 0);
p->kicked_without_pollers = 1;
}
}
GPR_TIMER_END("grpc_pollset_kick_ext", 0);
}
void test_poll_cv_trigger(void) {
grpc_wakeup_fd cvfd1, cvfd2, cvfd3;
struct pollfd pfds[6];
poll_args pargs;
gpr_thd_id t_id;
gpr_thd_options opt;
GPR_ASSERT(grpc_wakeup_fd_init(&cvfd1) == GRPC_ERROR_NONE);
GPR_ASSERT(grpc_wakeup_fd_init(&cvfd2) == GRPC_ERROR_NONE);
GPR_ASSERT(grpc_wakeup_fd_init(&cvfd3) == GRPC_ERROR_NONE);
GPR_ASSERT(cvfd1.read_fd < 0);
GPR_ASSERT(cvfd2.read_fd < 0);
GPR_ASSERT(cvfd3.read_fd < 0);
GPR_ASSERT(cvfd1.read_fd != cvfd2.read_fd);
GPR_ASSERT(cvfd2.read_fd != cvfd3.read_fd);
GPR_ASSERT(cvfd1.read_fd != cvfd3.read_fd);
pfds[0].fd = cvfd1.read_fd;
pfds[1].fd = cvfd2.read_fd;
pfds[2].fd = 20;
pfds[3].fd = 30;
pfds[4].fd = cvfd3.read_fd;
pfds[5].fd = 50;
pfds[0].events = 0;
pfds[1].events = POLLIN;
pfds[2].events = POLLIN | POLLHUP;
pfds[3].events = POLLIN | POLLHUP;
pfds[4].events = POLLIN;
pfds[5].events = POLLIN;
pargs.fds = pfds;
pargs.nfds = 6;
pargs.timeout = 1000;
pargs.result = -2;
opt = gpr_thd_options_default();
gpr_thd_options_set_joinable(&opt);
gpr_thd_new(&t_id, &background_poll, &pargs, &opt);
// Wakeup wakeup_fd not listening for events
GPR_ASSERT(grpc_wakeup_fd_wakeup(&cvfd1) == GRPC_ERROR_NONE);
gpr_thd_join(t_id);
GPR_ASSERT(pargs.result == 0);
GPR_ASSERT(pfds[0].revents == 0);
GPR_ASSERT(pfds[1].revents == 0);
GPR_ASSERT(pfds[2].revents == 0);
GPR_ASSERT(pfds[3].revents == 0);
GPR_ASSERT(pfds[4].revents == 0);
GPR_ASSERT(pfds[5].revents == 0);
// Pollin on socket fd
pargs.timeout = -1;
pargs.result = -2;
gpr_thd_new(&t_id, &background_poll, &pargs, &opt);
trigger_socket_event();
gpr_thd_join(t_id);
GPR_ASSERT(pargs.result == 1);
GPR_ASSERT(pfds[0].revents == 0);
GPR_ASSERT(pfds[1].revents == 0);
GPR_ASSERT(pfds[2].revents == POLLIN);
GPR_ASSERT(pfds[3].revents == 0);
GPR_ASSERT(pfds[4].revents == 0);
GPR_ASSERT(pfds[5].revents == 0);
// Pollin on wakeup fd
reset_socket_event();
pargs.result = -2;
gpr_thd_new(&t_id, &background_poll, &pargs, &opt);
GPR_ASSERT(grpc_wakeup_fd_wakeup(&cvfd2) == GRPC_ERROR_NONE);
gpr_thd_join(t_id);
GPR_ASSERT(pargs.result == 1);
GPR_ASSERT(pfds[0].revents == 0);
GPR_ASSERT(pfds[1].revents == POLLIN);
GPR_ASSERT(pfds[2].revents == 0);
GPR_ASSERT(pfds[3].revents == 0);
GPR_ASSERT(pfds[4].revents == 0);
GPR_ASSERT(pfds[5].revents == 0);
// Pollin on wakeupfd before poll()
pargs.result = -2;
gpr_thd_new(&t_id, &background_poll, &pargs, &opt);
gpr_thd_join(t_id);
GPR_ASSERT(pargs.result == 1);
GPR_ASSERT(pfds[0].revents == 0);
GPR_ASSERT(pfds[1].revents == POLLIN);
GPR_ASSERT(pfds[2].revents == 0);
GPR_ASSERT(pfds[3].revents == 0);
GPR_ASSERT(pfds[4].revents == 0);
GPR_ASSERT(pfds[5].revents == 0);
// No Events
pargs.result = -2;
pargs.timeout = 1000;
reset_socket_event();
GPR_ASSERT(grpc_wakeup_fd_consume_wakeup(&cvfd1) == GRPC_ERROR_NONE);
GPR_ASSERT(grpc_wakeup_fd_consume_wakeup(&cvfd2) == GRPC_ERROR_NONE);
gpr_thd_new(&t_id, &background_poll, &pargs, &opt);
gpr_thd_join(t_id);
GPR_ASSERT(pargs.result == 0);
GPR_ASSERT(pfds[0].revents == 0);
GPR_ASSERT(pfds[1].revents == 0);
GPR_ASSERT(pfds[2].revents == 0);
GPR_ASSERT(pfds[3].revents == 0);
GPR_ASSERT(pfds[4].revents == 0);
GPR_ASSERT(pfds[5].revents == 0);
}
static void make_test_fds_readable(test_fd *tfds, const int num_fds) {
for (int i = 0; i < num_fds; i++) {
GPR_ASSERT(GRPC_ERROR_NONE == grpc_wakeup_fd_wakeup(&tfds[i].wakeup_fd));
}
}
static void epoll_kick(grpc_pollset *pollset) {
pollset_hdr *h = pollset->data.ptr;
grpc_wakeup_fd_wakeup(&h->wakeup_fd);
}
static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p,
grpc_pollset_worker *specific_worker) {
if (GRPC_TRACER_ON(grpc_polling_trace)) {
gpr_log(GPR_DEBUG,
"PS:%p kick %p tls_pollset=%p tls_worker=%p "
"root_worker=(pollset:%p pollable:%p)",
p, specific_worker, (void *)gpr_tls_get(&g_current_thread_pollset),
(void *)gpr_tls_get(&g_current_thread_worker), pollset->root_worker,
p->root_worker);
}
if (specific_worker == NULL) {
if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) {
if (pollset->root_worker == NULL) {
if (GRPC_TRACER_ON(grpc_polling_trace)) {
gpr_log(GPR_DEBUG, "PS:%p kicked_any_without_poller", p);
}
pollset->kicked_without_poller = true;
return GRPC_ERROR_NONE;
} else {
if (GRPC_TRACER_ON(grpc_polling_trace)) {
gpr_log(GPR_DEBUG, "PS:%p kicked_any_via_wakeup_fd", p);
}
grpc_error *err = pollable_materialize(p);
if (err != GRPC_ERROR_NONE) return err;
return grpc_wakeup_fd_wakeup(&p->wakeup);
}
} else {
if (GRPC_TRACER_ON(grpc_polling_trace)) {
gpr_log(GPR_DEBUG, "PS:%p kicked_any_but_awake", p);
}
return GRPC_ERROR_NONE;
}
} else if (specific_worker->kicked) {
if (GRPC_TRACER_ON(grpc_polling_trace)) {
gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_already_kicked", p);
}
return GRPC_ERROR_NONE;
} else if (gpr_tls_get(&g_current_thread_worker) ==
(intptr_t)specific_worker) {
if (GRPC_TRACER_ON(grpc_polling_trace)) {
gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_awake", p);
}
specific_worker->kicked = true;
return GRPC_ERROR_NONE;
} else if (specific_worker == p->root_worker) {
if (GRPC_TRACER_ON(grpc_polling_trace)) {
gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_wakeup_fd", p);
}
grpc_error *err = pollable_materialize(p);
if (err != GRPC_ERROR_NONE) return err;
specific_worker->kicked = true;
return grpc_wakeup_fd_wakeup(&p->wakeup);
} else {
if (GRPC_TRACER_ON(grpc_polling_trace)) {
gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_cv", p);
}
specific_worker->kicked = true;
gpr_cv_signal(&specific_worker->cv);
return GRPC_ERROR_NONE;
}
}