static void
in_pcbinswildcardhash_handler(netmsg_t msg)
{
struct netmsg_inswildcard *nm = (struct netmsg_inswildcard *)msg;
int cpu = mycpuid, nextcpu;
in_pcbinswildcardhash_oncpu(nm->nm_inp, &tcbinfo[cpu]);
nextcpu = cpu + 1;
if (nextcpu < ncpus2)
lwkt_forwardmsg(cpu_portfn(nextcpu), &nm->base.lmsg);
else
lwkt_replymsg(&nm->base.lmsg, 0);
}
static void
udp_notifyall_oncpu(netmsg_t msg)
{
struct netmsg_udp_notify *nm = (struct netmsg_udp_notify *)msg;
int nextcpu, cpu = mycpuid;
in_pcbnotifyall(&udbinfo[cpu], nm->nm_faddr, nm->nm_arg, nm->nm_notify);
nextcpu = cpu + 1;
if (nextcpu < ncpus2)
lwkt_forwardmsg(netisr_cpuport(nextcpu), &nm->base.lmsg);
else
lwkt_replymsg(&nm->base.lmsg, 0);
}
static void
ifpoll_register_handler(netmsg_t nmsg)
{
const struct ifpoll_info *info = nmsg->lmsg.u.ms_resultp;
int cpuid = mycpuid, nextcpu;
int error;
KKASSERT(cpuid < ncpus2);
KKASSERT(&curthread->td_msgport == netisr_portfn(cpuid));
if (cpuid == 0) {
error = stpoll_register(info->ifpi_ifp, &info->ifpi_status);
if (error)
goto failed;
}
error = iopoll_register(info->ifpi_ifp, rxpoll_context[cpuid],
&info->ifpi_rx[cpuid]);
if (error)
goto failed;
error = iopoll_register(info->ifpi_ifp, txpoll_context[cpuid],
&info->ifpi_tx[cpuid]);
if (error)
goto failed;
/* Adjust polling frequency, after all registration is done */
poll_comm_adjust_pollhz(poll_common[cpuid]);
nextcpu = cpuid + 1;
if (nextcpu < ncpus2)
lwkt_forwardmsg(netisr_portfn(nextcpu), &nmsg->lmsg);
else
lwkt_replymsg(&nmsg->lmsg, 0);
return;
failed:
lwkt_replymsg(&nmsg->lmsg, error);
}
static void
ifpoll_deregister_handler(netmsg_t nmsg)
{
struct ifnet *ifp = nmsg->lmsg.u.ms_resultp;
int cpuid = mycpuid, nextcpu;
KKASSERT(cpuid < ncpus2);
KKASSERT(&curthread->td_msgport == netisr_portfn(cpuid));
/* Ignore errors */
if (cpuid == 0)
stpoll_deregister(ifp);
iopoll_deregister(ifp, rxpoll_context[cpuid]);
iopoll_deregister(ifp, txpoll_context[cpuid]);
/* Adjust polling frequency, after all deregistration is done */
poll_comm_adjust_pollhz(poll_common[cpuid]);
nextcpu = cpuid + 1;
if (nextcpu < ncpus2)
lwkt_forwardmsg(netisr_portfn(nextcpu), &nmsg->lmsg);
else
lwkt_replymsg(&nmsg->lmsg, 0);
}
static void
tcp6_connect(netmsg_t msg)
{
struct tcpcb *tp;
struct socket *so = msg->connect.base.nm_so;
struct sockaddr *nam = msg->connect.nm_nam;
struct thread *td = msg->connect.nm_td;
struct inpcb *inp;
struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)nam;
struct in6_addr *addr6;
#ifdef SMP
lwkt_port_t port;
#endif
int error;
COMMON_START(so, inp, 0);
/*
* Reconnect our pcb if we have to
*/
if (msg->connect.nm_reconnect & NMSG_RECONNECT_RECONNECT) {
msg->connect.nm_reconnect &= ~NMSG_RECONNECT_RECONNECT;
in_pcblink(so->so_pcb, &tcbinfo[mycpu->gd_cpuid]);
}
/*
* Bind if we have to
*/
if (inp->inp_lport == 0) {
error = in6_pcbbind(inp, NULL, td);
if (error)
goto out;
}
/*
* Cannot simply call in_pcbconnect, because there might be an
* earlier incarnation of this same connection still in
* TIME_WAIT state, creating an ADDRINUSE error.
*/
error = in6_pcbladdr(inp, nam, &addr6, td);
if (error)
goto out;
#ifdef SMP
port = tcp6_addrport(); /* XXX hack for now, always cpu0 */
if (port != &curthread->td_msgport) {
struct route *ro = &inp->inp_route;
/*
* in_pcbladdr() may have allocated a route entry for us
* on the current CPU, but we need a route entry on the
* inpcb's owner CPU, so free it here.
*/
if (ro->ro_rt != NULL)
RTFREE(ro->ro_rt);
bzero(ro, sizeof(*ro));
in_pcbunlink(so->so_pcb, &tcbinfo[mycpu->gd_cpuid]);
sosetport(so, port);
msg->connect.nm_reconnect |= NMSG_RECONNECT_RECONNECT;
msg->connect.base.nm_dispatch = tcp6_connect;
lwkt_forwardmsg(port, &msg->connect.base.lmsg);
/* msg invalid now */
return;
}
#endif
error = tcp6_connect_oncpu(tp, msg->connect.nm_flags,
&msg->connect.nm_m, sin6, addr6);
/* nm_m may still be intact */
out:
if (error && (msg->connect.nm_reconnect & NMSG_RECONNECT_FALLBACK)) {
tcp_connect(msg);
/* msg invalid now */
} else {
if (msg->connect.nm_m) {
m_freem(msg->connect.nm_m);
msg->connect.nm_m = NULL;
}
if (msg->connect.nm_reconnect & NMSG_RECONNECT_NAMALLOC) {
kfree(msg->connect.nm_nam, M_LWKTMSG);
msg->connect.nm_nam = NULL;
}
lwkt_replymsg(&msg->connect.base.lmsg, error);
/* msg invalid now */
}
}
/*
* Common subroutine to open a TCP connection to remote host specified
* by struct sockaddr_in in mbuf *nam. Call in_pcbbind to assign a local
* port number if needed. Call in_pcbladdr to do the routing and to choose
* a local host address (interface).
* Initialize connection parameters and enter SYN-SENT state.
*/
static void
tcp_connect(netmsg_t msg)
{
struct socket *so = msg->connect.base.nm_so;
struct sockaddr *nam = msg->connect.nm_nam;
struct thread *td = msg->connect.nm_td;
struct sockaddr_in *sin = (struct sockaddr_in *)nam;
struct sockaddr_in *if_sin;
struct inpcb *inp;
struct tcpcb *tp;
int error, calc_laddr = 1;
#ifdef SMP
lwkt_port_t port;
#endif
COMMON_START(so, inp, 0);
/*
* Reconnect our pcb if we have to
*/
if (msg->connect.nm_reconnect & NMSG_RECONNECT_RECONNECT) {
msg->connect.nm_reconnect &= ~NMSG_RECONNECT_RECONNECT;
in_pcblink(so->so_pcb, &tcbinfo[mycpu->gd_cpuid]);
}
/*
* Bind if we have to
*/
if (inp->inp_lport == 0) {
if (tcp_lport_extension) {
KKASSERT(inp->inp_laddr.s_addr == INADDR_ANY);
error = in_pcbladdr(inp, nam, &if_sin, td);
if (error)
goto out;
inp->inp_laddr.s_addr = if_sin->sin_addr.s_addr;
error = in_pcbconn_bind(inp, nam, td);
if (error)
goto out;
calc_laddr = 0;
} else {
error = in_pcbbind(inp, NULL, td);
if (error)
goto out;
}
}
if (calc_laddr) {
/*
* Calculate the correct protocol processing thread. The
* connect operation must run there. Set the forwarding
* port before we forward the message or it will get bounced
* right back to us.
*/
error = in_pcbladdr(inp, nam, &if_sin, td);
if (error)
goto out;
}
KKASSERT(inp->inp_socket == so);
#ifdef SMP
port = tcp_addrport(sin->sin_addr.s_addr, sin->sin_port,
(inp->inp_laddr.s_addr ?
inp->inp_laddr.s_addr : if_sin->sin_addr.s_addr),
inp->inp_lport);
if (port != &curthread->td_msgport) {
struct route *ro = &inp->inp_route;
/*
* in_pcbladdr() may have allocated a route entry for us
* on the current CPU, but we need a route entry on the
* inpcb's owner CPU, so free it here.
*/
if (ro->ro_rt != NULL)
RTFREE(ro->ro_rt);
bzero(ro, sizeof(*ro));
/*
* We are moving the protocol processing port the socket
* is on, we have to unlink here and re-link on the
* target cpu.
*/
in_pcbunlink(so->so_pcb, &tcbinfo[mycpu->gd_cpuid]);
sosetport(so, port);
msg->connect.nm_reconnect |= NMSG_RECONNECT_RECONNECT;
msg->connect.base.nm_dispatch = tcp_connect;
lwkt_forwardmsg(port, &msg->connect.base.lmsg);
/* msg invalid now */
return;
}
#else
KKASSERT(so->so_port == &curthread->td_msgport);
#endif
error = tcp_connect_oncpu(tp, msg->connect.nm_flags,
msg->connect.nm_m, sin, if_sin);
msg->connect.nm_m = NULL;
out:
if (msg->connect.nm_m) {
m_freem(msg->connect.nm_m);
msg->connect.nm_m = NULL;
}
if (msg->connect.nm_reconnect & NMSG_RECONNECT_NAMALLOC) {
kfree(msg->connect.nm_nam, M_LWKTMSG);
msg->connect.nm_nam = NULL;
}
lwkt_replymsg(&msg->connect.base.lmsg, error);
/* msg invalid now */
}
/*
* Prepare to accept connections.
*/
static void
tcp_usr_listen(netmsg_t msg)
{
struct socket *so = msg->listen.base.nm_so;
struct thread *td = msg->listen.nm_td;
int error = 0;
struct inpcb *inp;
struct tcpcb *tp;
struct netmsg_inswildcard nm;
lwkt_port_t port0 = netisr_cpuport(0);
COMMON_START(so, inp, 0);
if (&curthread->td_msgport != port0) {
lwkt_msg_t lmsg = &msg->listen.base.lmsg;
KASSERT((msg->listen.nm_flags & PRUL_RELINK) == 0,
("already asked to relink"));
in_pcbunlink(so->so_pcb, &tcbinfo[mycpuid]);
msg->listen.nm_flags |= PRUL_RELINK;
/* See the related comment in tcp_connect() */
lwkt_setmsg_receipt(lmsg, tcp_sosetport);
lwkt_forwardmsg(port0, lmsg);
/* msg invalid now */
return;
}
KASSERT(so->so_port == port0, ("so_port is not netisr0"));
if (msg->listen.nm_flags & PRUL_RELINK) {
msg->listen.nm_flags &= ~PRUL_RELINK;
in_pcblink(so->so_pcb, &tcbinfo[mycpuid]);
}
KASSERT(inp->inp_pcbinfo == &tcbinfo[0], ("pcbinfo is not tcbinfo0"));
if (tp->t_flags & TF_LISTEN)
goto out;
if (inp->inp_lport == 0) {
error = in_pcbbind(inp, NULL, td);
if (error)
goto out;
}
tp->t_state = TCPS_LISTEN;
tp->t_flags |= TF_LISTEN;
tp->tt_msg = NULL; /* Catch any invalid timer usage */
if (ncpus2 > 1) {
/*
* Put this inpcb into wildcard hash on other cpus.
*/
ASSERT_INP_NOTINHASH(inp);
netmsg_init(&nm.base, NULL, &curthread->td_msgport,
MSGF_PRIORITY, in_pcbinswildcardhash_handler);
nm.nm_inp = inp;
lwkt_domsg(netisr_cpuport(1), &nm.base.lmsg, 0);
}
in_pcbinswildcardhash(inp);
COMMON_END(PRU_LISTEN);
}