/* * stpoll_handler is scheduled by sched_stpoll when appropriate, typically * once per polling systimer tick. */ static void stpoll_handler(netmsg_t msg) { struct stpoll_ctx *st_ctx = &stpoll_context; struct thread *td = curthread; int i; KKASSERT(&td->td_msgport == netisr_portfn(0)); crit_enter_quick(td); /* Reply ASAP */ lwkt_replymsg(&msg->lmsg, 0); if (st_ctx->poll_handlers == 0) { crit_exit_quick(td); return; } for (i = 0; i < st_ctx->poll_handlers; ++i) { const struct stpoll_rec *rec = &st_ctx->pr[i]; struct ifnet *ifp = rec->ifp; if (!lwkt_serialize_try(rec->serializer)) continue; if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == (IFF_RUNNING | IFF_NPOLLING)) rec->status_func(ifp, st_ctx->pollhz); lwkt_serialize_exit(rec->serializer); } crit_exit_quick(td); }
/* * rxpoll_handler and txpoll_handler are scheduled by sched_iopoll when * appropriate, typically once per polling systimer tick. * * Note that the message is replied immediately in order to allow a new * ISR to be scheduled in the handler. */ static void rxpoll_handler(netmsg_t msg) { struct iopoll_ctx *io_ctx; struct thread *td = curthread; int i, cycles; io_ctx = msg->lmsg.u.ms_resultp; KKASSERT(&td->td_msgport == netisr_portfn(io_ctx->poll_cpuid)); crit_enter_quick(td); /* Reply ASAP */ lwkt_replymsg(&msg->lmsg, 0); if (io_ctx->poll_handlers == 0) { crit_exit_quick(td); return; } io_ctx->phase = 3; if (io_ctx->residual_burst == 0) { /* First call in this tick */ ifpoll_time_get(&io_ctx->poll_start_t); io_ctx->residual_burst = io_ctx->poll_burst; } cycles = (io_ctx->residual_burst < io_ctx->poll_each_burst) ? io_ctx->residual_burst : io_ctx->poll_each_burst; io_ctx->residual_burst -= cycles; for (i = 0; i < io_ctx->poll_handlers; i++) { const struct iopoll_rec *rec = &io_ctx->pr[i]; struct ifnet *ifp = rec->ifp; if (!lwkt_serialize_try(rec->serializer)) continue; if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == (IFF_RUNNING | IFF_NPOLLING)) rec->poll_func(ifp, rec->arg, cycles); lwkt_serialize_exit(rec->serializer); } /* * Do a quick exit/enter to catch any higher-priority * interrupt sources. */ crit_exit_quick(td); crit_enter_quick(td); sched_iopollmore(io_ctx); io_ctx->phase = 4; crit_exit_quick(td); }
/* * lwkt_thread_waitmsg() * * Wait for a particular message to be replied. We must be the only * thread waiting on the message. The port must be owned by the * caller. */ static int lwkt_thread_waitmsg(lwkt_msg_t msg, int flags) { thread_t td = curthread; KASSERT((msg->ms_flags & MSGF_DROPABLE) == 0, ("can't wait dropable message")); if ((msg->ms_flags & MSGF_DONE) == 0) { /* * If the done bit was not set we have to block until it is. */ lwkt_port_t port = msg->ms_reply_port; int sentabort; KKASSERT(port->mpu_td == td); crit_enter_quick(td); sentabort = 0; while ((msg->ms_flags & MSGF_DONE) == 0) { port->mp_flags |= MSGPORTF_WAITING; /* same cpu */ if (sentabort == 0) { if ((sentabort = lwkt_sleep("waitmsg", flags)) != 0) { lwkt_abortmsg(msg); } } else { lwkt_sleep("waitabt", 0); } port->mp_flags &= ~MSGPORTF_WAITING; } if (msg->ms_flags & MSGF_QUEUED) _lwkt_pullmsg(port, msg); crit_exit_quick(td); } else { /* * If the done bit was set we only have to mess around with the * message if it is queued on the reply port. */ crit_enter_quick(td); if (msg->ms_flags & MSGF_QUEUED) { lwkt_port_t port = msg->ms_reply_port; thread_t td __debugvar = curthread; KKASSERT(port->mpu_td == td); _lwkt_pullmsg(port, msg); } crit_exit_quick(td); } return(msg->ms_error); }
static void txpoll_handler(netmsg_t msg) { struct iopoll_ctx *io_ctx; struct thread *td = curthread; int i; io_ctx = msg->lmsg.u.ms_resultp; KKASSERT(&td->td_msgport == netisr_cpuport(io_ctx->poll_cpuid)); crit_enter_quick(td); /* Reply ASAP */ lwkt_replymsg(&msg->lmsg, 0); if (io_ctx->poll_handlers == 0) { crit_exit_quick(td); return; } io_ctx->phase = 3; for (i = 0; i < io_ctx->poll_handlers; i++) { const struct iopoll_rec *rec = &io_ctx->pr[i]; struct ifnet *ifp = rec->ifp; if (!lwkt_serialize_try(rec->serializer)) continue; if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) == (IFF_RUNNING | IFF_NPOLLING)) rec->poll_func(ifp, rec->arg, -1); lwkt_serialize_exit(rec->serializer); } /* * Do a quick exit/enter to catch any higher-priority * interrupt sources. */ crit_exit_quick(td); crit_enter_quick(td); sched_iopollmore(io_ctx); io_ctx->phase = 4; crit_exit_quick(td); }
static void txpollmore_handler(netmsg_t msg) { struct thread *td = curthread; struct iopoll_ctx *io_ctx; uint32_t pending_polls; io_ctx = msg->lmsg.u.ms_resultp; KKASSERT(&td->td_msgport == netisr_portfn(io_ctx->poll_cpuid)); crit_enter_quick(td); /* Replay ASAP */ lwkt_replymsg(&msg->lmsg, 0); if (io_ctx->poll_handlers == 0) { crit_exit_quick(td); return; } io_ctx->phase = 5; io_ctx->pending_polls--; pending_polls = io_ctx->pending_polls; if (pending_polls == 0) { /* We are done */ io_ctx->phase = 0; } else { /* * Last cycle was long and caused us to miss one or more * hardclock ticks. Restart processing again. */ sched_iopoll(io_ctx); io_ctx->phase = 6; } crit_exit_quick(td); }
/* * lwkt_thread_getport() * * Retrieve the next message from the port or NULL if no messages * are ready. */ static void * lwkt_thread_getport(lwkt_port_t port) { lwkt_msg_t msg; KKASSERT(port->mpu_td == curthread); crit_enter_quick(port->mpu_td); if ((msg = _lwkt_pollmsg(port)) != NULL) _lwkt_pullmsg(port, msg); crit_exit_quick(port->mpu_td); return(msg); }
/* * lwkt_thread_dropmsg() - Backend to lwkt_dropmsg() * * This function could _only_ be used when caller is in the same thread * as the message's target port owner thread. */ static int lwkt_thread_dropmsg(lwkt_port_t port, lwkt_msg_t msg) { int error; KASSERT(port->mpu_td == curthread, ("message could only be dropped in the same thread " "as the message target port thread")); crit_enter_quick(port->mpu_td); if ((msg->ms_flags & (MSGF_REPLY|MSGF_QUEUED)) == MSGF_QUEUED) { _lwkt_pullmsg(port, msg); atomic_set_int(&msg->ms_flags, MSGF_DONE); error = 0; } else { error = ENOENT; } crit_exit_quick(port->mpu_td); return (error); }
/* * Install the TLS. * * It shouldn't be possible for a preemptive thread switch to do anything * more than set gd_user_fs and wrmsr for us. Even though there is a window * where gd_user_fs/gd_user_gs do not match the MSRs no preemptive thread * switch should ever switch to any heavy weight thread other than our own. * * Still, use a critical section to be safe. * * MPSAFE */ void set_user_TLS(void) { struct mdglobaldata *gd = mdcpu; thread_t td = gd->mi.gd_curthread; crit_enter_quick(td); td->td_pcb->pcb_fsbase = (register_t)td->td_tls.info[0].base; td->td_pcb->pcb_gsbase = (register_t)td->td_tls.info[1].base; if (gd->gd_user_fs != td->td_pcb->pcb_fsbase) { gd->gd_user_fs = td->td_pcb->pcb_fsbase; wrmsr(MSR_FSBASE, gd->gd_user_fs); } if (gd->gd_user_gs != td->td_pcb->pcb_gsbase) { gd->gd_user_gs = td->td_pcb->pcb_gsbase; wrmsr(MSR_KGSBASE, gd->gd_user_gs); } clear_quickret(); crit_exit_quick(td); }
/* * lwkt_thread_waitport() * * Wait for a new message to be available on the port. We must be the * the only thread waiting on the port. The port must be owned by caller. */ static void * lwkt_thread_waitport(lwkt_port_t port, int flags) { thread_t td = curthread; lwkt_msg_t msg; int error; KKASSERT(port->mpu_td == td); crit_enter_quick(td); while ((msg = _lwkt_pollmsg(port)) == NULL) { port->mp_flags |= MSGPORTF_WAITING; error = lwkt_sleep("waitport", flags); port->mp_flags &= ~MSGPORTF_WAITING; if (error) goto done; } _lwkt_pullmsg(port, msg); done: crit_exit_quick(td); return(msg); }
/* * rxpollmore_handler and txpollmore_handler are called after other netisr's, * possibly scheduling another rxpoll_handler or txpoll_handler call, or * adapting the burst size for the next cycle. * * It is very bad to fetch large bursts of packets from a single card at once, * because the burst could take a long time to be completely processed leading * to unfairness. To reduce the problem, and also to account better for time * spent in network-related processing, we split the burst in smaller chunks * of fixed size, giving control to the other netisr's between chunks. This * helps in improving the fairness, reducing livelock and accounting for the * work performed in low level handling. */ static void rxpollmore_handler(netmsg_t msg) { struct thread *td = curthread; struct iopoll_ctx *io_ctx; union ifpoll_time t; int kern_load; uint32_t pending_polls; io_ctx = msg->lmsg.u.ms_resultp; KKASSERT(&td->td_msgport == netisr_portfn(io_ctx->poll_cpuid)); crit_enter_quick(td); /* Replay ASAP */ lwkt_replymsg(&msg->lmsg, 0); if (io_ctx->poll_handlers == 0) { crit_exit_quick(td); return; } io_ctx->phase = 5; if (io_ctx->residual_burst > 0) { sched_iopoll(io_ctx); crit_exit_quick(td); /* Will run immediately on return, followed by netisrs */ return; } /* Here we can account time spent in iopoll's in this tick */ ifpoll_time_get(&t); kern_load = ifpoll_time_diff(&io_ctx->poll_start_t, &t); kern_load = (kern_load * io_ctx->pollhz) / 10000; /* 0..100 */ io_ctx->kern_frac = kern_load; if (kern_load > (100 - io_ctx->user_frac)) { /* Try decrease ticks */ if (io_ctx->poll_burst > 1) io_ctx->poll_burst--; } else { if (io_ctx->poll_burst < io_ctx->poll_burst_max) io_ctx->poll_burst++; } io_ctx->pending_polls--; pending_polls = io_ctx->pending_polls; if (pending_polls == 0) { /* We are done */ io_ctx->phase = 0; } else { /* * Last cycle was long and caused us to miss one or more * hardclock ticks. Restart processing again, but slightly * reduce the burst size to prevent that this happens again. */ io_ctx->poll_burst -= (io_ctx->poll_burst / 8); if (io_ctx->poll_burst < 1) io_ctx->poll_burst = 1; sched_iopoll(io_ctx); io_ctx->phase = 6; } crit_exit_quick(td); }