/*
* Concatenate two pkthdr mbuf chains.
*/
void
m_catpkt(struct mbuf *m, struct mbuf *n)
{
M_ASSERTPKTHDR(m);
M_ASSERTPKTHDR(n);
m->m_pkthdr.len += n->m_pkthdr.len;
m_demote(n, 1, 0);
m_cat(m, n);
}
/*
* "Move" mbuf pkthdr from "from" to "to".
* "from" must have M_PKTHDR set, and "to" must be empty.
*/
void
m_move_pkthdr(struct mbuf *to, struct mbuf *from)
{
#if 0
/* see below for why these are not enabled */
M_ASSERTPKTHDR(to);
/* Note: with MAC, this may not be a good assertion. */
KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
("m_move_pkthdr: to has tags"));
#endif
#ifdef MAC
/*
* XXXMAC: It could be this should also occur for non-MAC?
*/
if (to->m_flags & M_PKTHDR)
m_tag_delete_chain(to, NULL);
#endif
to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
if ((to->m_flags & M_EXT) == 0)
to->m_data = to->m_pktdat;
to->m_pkthdr = from->m_pkthdr; /* especially tags */
SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
from->m_flags &= ~M_PKTHDR;
}
static int
do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cpl_iscsi_data *cpl = mtod(m, struct cpl_iscsi_data *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_cxgbei_pdu *icp = toep->ulpcb2;
M_ASSERTPKTHDR(m);
/* Must already have received the header (but not the data). */
MPASS(icp != NULL);
MPASS(icp->pdu_flags == SBUF_ULP_FLAG_HDR_RCVD);
MPASS(icp->ip.ip_data_mbuf == NULL);
MPASS(icp->ip.ip_data_len == 0);
m_adj(m, sizeof(*cpl));
icp->pdu_flags |= SBUF_ULP_FLAG_DATA_RCVD;
icp->ip.ip_data_mbuf = m;
icp->ip.ip_data_len = m->m_pkthdr.len;
#if 0
CTR4(KTR_CXGBE, "%s: tid %u, cpl->len dlen %u, m->m_len dlen %u",
__func__, tid, ntohs(cpl->len), m->m_len);
#endif
return (0);
}
static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_pdu *ip;
struct icl_cxgbei_pdu *icp;
M_ASSERTPKTHDR(m);
ip = icl_cxgbei_new_pdu(M_NOWAIT);
if (ip == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
icp = ip_to_icp(ip);
bcopy(mtod(m, caddr_t) + sizeof(*cpl), icp->ip.ip_bhs, sizeof(struct
iscsi_bhs));
icp->pdu_seq = ntohl(cpl->seq);
icp->pdu_flags = SBUF_ULP_FLAG_HDR_RCVD;
/* This is the start of a new PDU. There should be no old state. */
MPASS(toep->ulpcb2 == NULL);
toep->ulpcb2 = icp;
#if 0
CTR4(KTR_CXGBE, "%s: tid %u, cpl->len hlen %u, m->m_len hlen %u",
__func__, tid, ntohs(cpl->len), m->m_len);
#endif
m_freem(m);
return (0);
}
/*
* Duplicate "from"'s mbuf pkthdr in "to".
* "from" must have M_PKTHDR set, and "to" must be empty.
* In particular, this does a deep copy of the packet tags.
*/
int
m_dup_pkthdr(struct mbuf *to, const struct mbuf *from, int how)
{
#if 0
/*
* The mbuf allocator only initializes the pkthdr
* when the mbuf is allocated with m_gethdr(). Many users
* (e.g. m_copy*, m_prepend) use m_get() and then
* smash the pkthdr as needed causing these
* assertions to trip. For now just disable them.
*/
M_ASSERTPKTHDR(to);
/* Note: with MAC, this may not be a good assertion. */
KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
#endif
MBUF_CHECKSLEEP(how);
#ifdef MAC
if (to->m_flags & M_PKTHDR)
m_tag_delete_chain(to, NULL);
#endif
to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
if ((to->m_flags & M_EXT) == 0)
to->m_data = to->m_pktdat;
to->m_pkthdr = from->m_pkthdr;
SLIST_INIT(&to->m_pkthdr.tags);
return (m_tag_copy_chain(to, from, how));
}
static int
discoutput(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
struct route *ro)
{
u_int32_t af;
M_ASSERTPKTHDR(m);
/* BPF writes need to be handled specially. */
if (dst->sa_family == AF_UNSPEC)
bcopy(dst->sa_data, &af, sizeof(af));
else
af = dst->sa_family;
if (bpf_peers_present(ifp->if_bpf))
bpf_mtap2(ifp->if_bpf, &af, sizeof(af), m);
m->m_pkthdr.rcvif = ifp;
ifp->if_opackets++;
ifp->if_obytes += m->m_pkthdr.len;
m_freem(m);
return (0);
}
/*
* IP output. The packet in mbuf chain m contains a skeletal IP
* header (with len, off, ttl, proto, tos, src, dst).
* The mbuf chain containing the packet will be freed.
* The mbuf opt, if present, will not be freed.
* If route ro is present and has ro_rt initialized, route lookup would be
* skipped and ro->ro_rt would be used. If ro is present but ro->ro_rt is NULL,
* then result of route lookup is stored in ro->ro_rt.
*
* In the IP forwarding case, the packet will arrive with options already
* inserted, so must have a NULL opt pointer.
*/
int
ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
struct ip_moptions *imo, struct inpcb *inp)
{
struct rm_priotracker in_ifa_tracker;
struct ip *ip;
struct ifnet *ifp = NULL; /* keep compiler happy */
struct mbuf *m0;
int hlen = sizeof (struct ip);
int mtu;
int error = 0;
struct sockaddr_in *dst;
const struct sockaddr_in *gw;
struct in_ifaddr *ia;
int isbroadcast;
uint16_t ip_len, ip_off;
struct route iproute;
struct rtentry *rte; /* cache for ro->ro_rt */
uint32_t fibnum;
int have_ia_ref;
#ifdef IPSEC
int no_route_but_check_spd = 0;
#endif
M_ASSERTPKTHDR(m);
if (inp != NULL) {
INP_LOCK_ASSERT(inp);
M_SETFIB(m, inp->inp_inc.inc_fibnum);
if ((flags & IP_NODEFAULTFLOWID) == 0) {
m->m_pkthdr.flowid = inp->inp_flowid;
M_HASHTYPE_SET(m, inp->inp_flowtype);
}
}
if (ro == NULL) {
ro = &iproute;
bzero(ro, sizeof (*ro));
}
#ifdef FLOWTABLE
if (ro->ro_rt == NULL)
(void )flowtable_lookup(AF_INET, m, ro);
#endif
if (opt) {
int len = 0;
m = ip_insertoptions(m, opt, &len);
if (len != 0)
hlen = len; /* ip->ip_hl is updated above */
}
ip = mtod(m, struct ip *);
ip_len = ntohs(ip->ip_len);
ip_off = ntohs(ip->ip_off);
if ((flags & (IP_FORWARDING|IP_RAWOUTPUT)) == 0) {
ip->ip_v = IPVERSION;
ip->ip_hl = hlen >> 2;
ip_fillid(ip);
IPSTAT_INC(ips_localout);
} else {
int
bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
bus_dma_segment_t *segs, int *nsegs, int flags)
{
int error = 0;
M_ASSERTPKTHDR(m0);
*nsegs = 0;
if (m0->m_pkthdr.len <= dmat->maxsize) {
int first = 1;
vm_offset_t lastaddr = 0;
struct mbuf *m;
for (m = m0; m != NULL && error == 0; m = m->m_next) {
if (m->m_len > 0) {
error = bus_dmamap_load_buffer(dmat,
segs, m->m_data, m->m_len, NULL,
flags, &lastaddr, nsegs, first);
first = 0;
}
}
++*nsegs;
} else {
error = EINVAL;
}
return (error);
}
static int
do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cxgbei_data *ci = sc->iscsi_ulp_softc;
struct cpl_iscsi_data *cpl = mtod(m, struct cpl_iscsi_data *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_cxgbei_pdu *icp = toep->ulpcb2;
M_ASSERTPKTHDR(m);
MPASS(m->m_pkthdr.len == be16toh(cpl->len) + sizeof(*cpl));
/* Must already have received the header (but not the data). */
MPASS(icp != NULL);
MPASS(icp->icp_flags == ICPF_RX_HDR);
MPASS(icp->ip.ip_data_mbuf == NULL);
m_adj(m, sizeof(*cpl));
MPASS(icp->ip.ip_data_len == m->m_pkthdr.len);
icp->icp_flags |= ICPF_RX_FLBUF;
icp->ip.ip_data_mbuf = m;
counter_u64_add(ci->fl_pdus, 1);
counter_u64_add(ci->fl_bytes, m->m_pkthdr.len);
#if 0
CTR3(KTR_CXGBE, "%s: tid %u, cpl->len %u", __func__, tid,
be16toh(cpl->len));
#endif
return (0);
}
/*
* Similar to rss_m2cpuid, but designed to be used by the IP NETISR
* on incoming frames.
*
* If an existing RSS hash exists and it matches what the configured
* hashing is, then use it.
*
* If there's an existing RSS hash but the desired hash is different,
* or if there's no useful RSS hash, then calculate it via
* the software path.
*
* XXX TODO: definitely want statistics here!
*/
struct mbuf *
rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
{
uint32_t hash_val, hash_type;
int ret;
M_ASSERTPKTHDR(m);
ret = rss_mbuf_software_hash_v4(m, RSS_HASH_PKT_INGRESS,
&hash_val, &hash_type);
if (ret > 0) {
/* mbuf has a valid hash already; don't need to modify it */
*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
} else if (ret == 0) {
/* hash was done; update */
m->m_pkthdr.flowid = hash_val;
M_HASHTYPE_SET(m, hash_type);
m->m_flags |= M_FLOWID;
*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
} else { /* ret < 0 */
/* no hash was done */
*cpuid = NETISR_CPUID_NONE;
}
return (m);
}
static void
ether_nh_input(struct rte_mbuf *m)
{
M_ASSERTPKTHDR(m);
ether_input_internal(NULL, m);
}
/*
* netisr CPU affinity lookup routine for use by protocols.
*/
struct mbuf *
rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
{
M_ASSERTPKTHDR(m);
*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
return (m);
}
int
rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
{
M_ASSERTPKTHDR(m);
return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
bucket_id));
}
void
m_demote_pkthdr(struct mbuf *m)
{
M_ASSERTPKTHDR(m);
m_tag_delete_chain(m, NULL);
m->m_flags &= ~M_PKTHDR;
bzero(&m->m_pkthdr, sizeof(struct pkthdr));
}
/*
* natmintr: interrupt
*
* Note: we expect a socket pointer in rcvif rather than an interface
* pointer. We can get the interface pointer from the so's PCB if we really
* need it.
*/
void
natmintr(struct mbuf *m)
{
struct socket *so;
struct natmpcb *npcb;
#ifdef DIAGNOSTIC
M_ASSERTPKTHDR(m);
#endif
NATM_LOCK();
npcb = (struct natmpcb *)m->m_pkthdr.rcvif; /* XXX: overloaded */
so = npcb->npcb_socket;
npcb->npcb_inq--;
if (npcb->npcb_flags & NPCB_DRAIN) {
if (npcb->npcb_inq == 0)
free(npcb, M_PCB); /* done! */
NATM_UNLOCK();
m_freem(m);
return;
}
if (npcb->npcb_flags & NPCB_FREE) {
NATM_UNLOCK();
m_freem(m); /* drop */
return;
}
#ifdef NEED_TO_RESTORE_IFP
m->m_pkthdr.rcvif = npcb->npcb_ifp;
#else
#ifdef DIAGNOSTIC
m->m_pkthdr.rcvif = NULL; /* null it out to be safe */
#endif
#endif
if (sbspace(&so->so_rcv) > m->m_pkthdr.len) {
#ifdef NATM_STAT
natm_sookcnt++;
natm_sookbytes += m->m_pkthdr.len;
#endif
sbappendrecord(&so->so_rcv, m);
sorwakeup(so);
NATM_UNLOCK();
} else {
#ifdef NATM_STAT
natm_sodropcnt++;
natm_sodropbytes += m->m_pkthdr.len;
#endif
NATM_UNLOCK();
m_freem(m);
}
}
static int
ifoff_socket_check_deliver(struct socket *so, struct label *solabel,
struct mbuf *m, struct label *mlabel)
{
M_ASSERTPKTHDR(m);
if (m->m_pkthdr.rcvif != NULL)
return (ifnet_check_incoming(m->m_pkthdr.rcvif, 0));
return (0);
}
static int
discoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
struct rtentry *rt)
{
M_ASSERTPKTHDR(m);
/* BPF write needs to be handled specially */
if (dst->sa_family == AF_UNSPEC) {
dst->sa_family = *(mtod(m, int *));
m->m_len -= sizeof(int);
m->m_pkthdr.len -= sizeof(int);
m->m_data += sizeof(int);
}
void
ip_dn_packet_free(struct dn_pkt *pkt)
{
struct netmsg_packet *nmp;
struct mbuf *m = pkt->dn_m;
M_ASSERTPKTHDR(m);
KASSERT(m->m_pkthdr.fw_flags & DUMMYNET_MBUF_TAGGED,
("mbuf is not tagged for dummynet!"));
nmp = &m->m_hdr.mh_netmsg;
netmsg_init(&nmp->base, NULL, &netisr_apanic_rport,
0, ip_dn_freepkt_dispatch);
nmp->nm_packet = m;
lwkt_sendmsg(pkt->msgport, &nmp->base.lmsg);
}
void
ip_dn_queue(struct mbuf *m)
{
struct netmsg_packet *nmp;
lwkt_port_t port;
M_ASSERTPKTHDR(m);
KASSERT(m->m_pkthdr.fw_flags & DUMMYNET_MBUF_TAGGED,
("mbuf is not tagged for dummynet!"));
nmp = &m->m_hdr.mh_netmsg;
netmsg_init(&nmp->base, NULL, &netisr_apanic_rport,
0, ip_dn_dispatch);
nmp->nm_packet = m;
port = netisr_cpuport(ip_dn_cpu);
lwkt_sendmsg(port, &nmp->base.lmsg);
}
static inline void
rqdrop_locked(struct mbufq *q, int plen)
{
struct mbuf *m;
while (plen > 0) {
m = mbufq_dequeue(q);
/* Too many credits. */
MPASS(m != NULL);
M_ASSERTPKTHDR(m);
/* Partial credits. */
MPASS(plen >= m->m_pkthdr.len);
plen -= m->m_pkthdr.len;
m_freem(m);
}
}
static int
looutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
struct rtentry *rt)
{
M_ASSERTPKTHDR(m);
if (rt && rt->rt_flags & (RTF_REJECT|RTF_BLACKHOLE)) {
m_freem(m);
return (rt->rt_flags & RTF_BLACKHOLE ? 0 :
rt->rt_flags & RTF_HOST ? EHOSTUNREACH : ENETUNREACH);
}
ifp->if_opackets++;
ifp->if_obytes += m->m_pkthdr.len;
#if 1 /* XXX */
switch (dst->sa_family) {
case AF_INET:
case AF_INET6:
case AF_IPX:
case AF_NS:
break;
default:
kprintf("looutput: af=%d unexpected\n", dst->sa_family);
m_freem(m);
return (EAFNOSUPPORT);
}
#endif
if (ifp->if_capenable & IFCAP_RXCSUM) {
int csum_flags = 0;
if (m->m_pkthdr.csum_flags & CSUM_IP)
csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID);
if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA)
csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
m->m_pkthdr.csum_flags |= csum_flags;
if (csum_flags & CSUM_DATA_VALID)
m->m_pkthdr.csum_data = 0xffff;
}
return (if_simloop(ifp, m, dst->sa_family, 0));
}
/*
* Like bus_dmamap_load(), but for mbufs.
*/
int
bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
bus_dmamap_callback2_t *callback, void *callback_arg, int flags)
{
#ifdef __CC_SUPPORTS_DYNAMIC_ARRAY_INIT
bus_dma_segment_t dm_segments[dmat->nsegments];
#else
bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS];
#endif
int nsegs = 0, error = 0;
M_ASSERTPKTHDR(m0);
if (m0->m_pkthdr.len <= dmat->maxsize) {
int first = 1;
vm_offset_t lastaddr = 0;
struct mbuf *m;
for (m = m0; m != NULL && error == 0; m = m->m_next) {
if (m->m_len > 0) {
error = bus_dmamap_load_buffer(dmat,
dm_segments, m->m_data, m->m_len, NULL,
flags, &lastaddr, &nsegs, first);
first = 0;
}
}
} else {
error = EINVAL;
}
if (error) {
/*
* force "no valid mappings" on error in callback.
*/
(*callback)(callback_arg, dm_segments, 0, 0, error);
} else {
(*callback)(callback_arg, dm_segments, nsegs+1,
m0->m_pkthdr.len, error);
}
return (error);
}
int
mac_ifnet_check_transmit(struct ifnet *ifp, struct mbuf *m)
{
struct label *label;
int error;
M_ASSERTPKTHDR(m);
if (mac_policy_count == 0)
return (0);
label = mac_mbuf_to_label(m);
MAC_IFNET_LOCK(ifp);
MAC_POLICY_CHECK_NOSLEEP(ifnet_check_transmit, ifp, ifp->if_label, m,
label);
MAC_CHECK_PROBE2(ifnet_check_transmit, error, ifp, m);
MAC_IFNET_UNLOCK(ifp);
return (error);
}
/*
* Load an mbuf chain.
*/
static int
_bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags)
{
struct mbuf *m;
int error;
M_ASSERTPKTHDR(m0);
error = 0;
for (m = m0; m != NULL && error == 0; m = m->m_next) {
if (m->m_len > 0) {
error = _bus_dmamap_load_buffer(dmat, map, m->m_data,
m->m_len, kernel_pmap, flags | BUS_DMA_LOAD_MBUF,
segs, nsegs);
}
}
CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
__func__, dmat, flags, error, *nsegs);
return (error);
}
int
mac_mbuf_init(struct mbuf *m, int flag)
{
struct m_tag *tag;
int error;
M_ASSERTPKTHDR(m);
if (mac_labeled & MPC_OBJECT_MBUF) {
tag = m_tag_get(PACKET_TAG_MACLABEL, sizeof(struct label),
flag);
if (tag == NULL)
return (ENOMEM);
error = mac_mbuf_tag_init(tag, flag);
if (error) {
m_tag_free(tag);
return (error);
}
m_tag_prepend(m, tag);
}
return (0);
}
static int
in_gre_encapcheck(const struct mbuf *m, int off, int proto, void *arg)
{
GRE_RLOCK_TRACKER;
struct gre_softc *sc;
struct ip *ip;
sc = (struct gre_softc *)arg;
if ((GRE2IFP(sc)->if_flags & IFF_UP) == 0)
return (0);
M_ASSERTPKTHDR(m);
/*
* We expect that payload contains at least IPv4
* or IPv6 packet.
*/
if (m->m_pkthdr.len < sizeof(struct greip) + sizeof(struct ip))
return (0);
GRE_RLOCK(sc);
if (sc->gre_family == 0)
goto bad;
KASSERT(sc->gre_family == AF_INET,
("wrong gre_family: %d", sc->gre_family));
ip = mtod(m, struct ip *);
if (sc->gre_oip.ip_src.s_addr != ip->ip_dst.s_addr ||
sc->gre_oip.ip_dst.s_addr != ip->ip_src.s_addr)
goto bad;
GRE_RUNLOCK(sc);
return (32 * 2);
bad:
GRE_RUNLOCK(sc);
return (0);
}
static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_pdu *ip;
struct icl_cxgbei_pdu *icp;
uint16_t len_ddp = be16toh(cpl->pdu_len_ddp);
uint16_t len = be16toh(cpl->len);
M_ASSERTPKTHDR(m);
MPASS(m->m_pkthdr.len == len + sizeof(*cpl));
ip = icl_cxgbei_new_pdu(M_NOWAIT);
if (ip == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
m_copydata(m, sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs);
ip->ip_data_len = G_ISCSI_PDU_LEN(len_ddp) - len;
icp = ip_to_icp(ip);
icp->icp_seq = ntohl(cpl->seq);
icp->icp_flags = ICPF_RX_HDR;
/* This is the start of a new PDU. There should be no old state. */
MPASS(toep->ulpcb2 == NULL);
toep->ulpcb2 = icp;
#if 0
CTR5(KTR_CXGBE, "%s: tid %u, cpl->len %u, pdu_len_ddp 0x%04x, icp %p",
__func__, tid, len, len_ddp, icp);
#endif
m_freem(m);
return (0);
}
int
bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
bus_dmamap_callback2_t *callback, void *callback_arg, int flags)
{
bus_dma_segment_t *segs;
int nsegs, error;
M_ASSERTPKTHDR(m0);
flags |= BUS_DMA_NOWAIT;
nsegs = -1;
error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags);
++nsegs;
segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
if (error)
(*callback)(callback_arg, segs, 0, 0, error);
else
(*callback)(callback_arg, segs, nsegs, m0->m_pkthdr.len, error);
CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
__func__, dmat, flags, error, nsegs);
return (error);
}
/*
* IP output. The packet in mbuf chain m contains a skeletal IP
* header (with len, off, ttl, proto, tos, src, dst).
* The mbuf chain containing the packet will be freed.
* The mbuf opt, if present, will not be freed.
* If route ro is present and has ro_rt initialized, route lookup would be
* skipped and ro->ro_rt would be used. If ro is present but ro->ro_rt is NULL,
* then result of route lookup is stored in ro->ro_rt.
*
* In the IP forwarding case, the packet will arrive with options already
* inserted, so must have a NULL opt pointer.
*/
int
ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
struct ip_moptions *imo, struct inpcb *inp)
{
struct ip *ip;
struct ifnet *ifp = NULL; /* keep compiler happy */
struct mbuf *m0;
int hlen = sizeof (struct ip);
int mtu;
int n; /* scratchpad */
int error = 0;
struct sockaddr_in *dst;
const struct sockaddr_in *gw;
struct in_ifaddr *ia;
int isbroadcast;
uint16_t ip_len, ip_off;
struct route iproute;
struct rtentry *rte; /* cache for ro->ro_rt */
struct in_addr odst;
struct m_tag *fwd_tag = NULL;
int have_ia_ref;
#ifdef IPSEC
int no_route_but_check_spd = 0;
#endif
M_ASSERTPKTHDR(m);
if (inp != NULL) {
INP_LOCK_ASSERT(inp);
M_SETFIB(m, inp->inp_inc.inc_fibnum);
if (inp->inp_flowtype != M_HASHTYPE_NONE) {
m->m_pkthdr.flowid = inp->inp_flowid;
M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
}
}
if (ro == NULL) {
ro = &iproute;
bzero(ro, sizeof (*ro));
}
#ifdef FLOWTABLE
if (ro->ro_rt == NULL)
(void )flowtable_lookup(AF_INET, m, ro);
#endif
if (opt) {
int len = 0;
m = ip_insertoptions(m, opt, &len);
if (len != 0)
hlen = len; /* ip->ip_hl is updated above */
}
ip = mtod(m, struct ip *);
ip_len = ntohs(ip->ip_len);
ip_off = ntohs(ip->ip_off);
/*
* Fill in IP header. If we are not allowing fragmentation,
* then the ip_id field is meaningless, but we don't set it
* to zero. Doing so causes various problems when devices along
* the path (routers, load balancers, firewalls, etc.) illegally
* disable DF on our packet. Note that a 16-bit counter
* will wrap around in less than 10 seconds at 100 Mbit/s on a
* medium with MTU 1500. See Steven M. Bellovin, "A Technique
* for Counting NATted Hosts", Proc. IMW'02, available at
* <http://www.cs.columbia.edu/~smb/papers/fnat.pdf>.
*/
if ((flags & (IP_FORWARDING|IP_RAWOUTPUT)) == 0) {
ip->ip_v = IPVERSION;
ip->ip_hl = hlen >> 2;
ip->ip_id = ip_newid();
IPSTAT_INC(ips_localout);
} else {
/* This function will free m0! */
int
ip_output0(PNATState pData, struct socket *so, struct mbuf *m0, int urg)
{
register struct ip *ip;
register struct mbuf *m = m0;
register int hlen = sizeof(struct ip);
int len, off, error = 0;
struct ethhdr *eh = NULL;
uint8_t eth_dst[ETH_ALEN];
int rc = 1;
STAM_PROFILE_START(&pData->StatIP_output, a);
#ifdef LOG_ENABLED
LogFlowFunc(("ip_output: so = %R[natsock], m0 = %lx\n", so, (long)m0));
#else
NOREF(so);
#endif
M_ASSERTPKTHDR(m);
Assert(m->m_pkthdr.header);
#if 0 /* We do no options */
if (opt)
{
m = ip_insertoptions(m, opt, &len);
hlen = len;
}
#endif
ip = mtod(m, struct ip *);
LogFunc(("ip(src:%RTnaipv4, dst:%RTnaipv4)\n", ip->ip_src, ip->ip_dst));
/*
* Fill in IP header.
*/
ip->ip_v = IPVERSION;
ip->ip_off &= IP_DF;
ip->ip_id = RT_H2N_U16(ip_currid++);
ip->ip_hl = hlen >> 2;
ipstat.ips_localout++;
/* Current TCP/IP stack hasn't routing information at
* all so we need to calculate destination ethernet address
*/
rc = rt_lookup_in_cache(pData, ip->ip_dst.s_addr, eth_dst);
if (RT_FAILURE(rc))
goto exit_drop_package;
eh = (struct ethhdr *)(m->m_data - ETH_HLEN);
/*
* If small enough for interface, can just send directly.
*/
if ((u_int16_t)ip->ip_len <= if_mtu)
{
ip->ip_len = RT_H2N_U16((u_int16_t)ip->ip_len);
ip->ip_off = RT_H2N_U16((u_int16_t)ip->ip_off);
ip->ip_sum = 0;
ip->ip_sum = cksum(m, hlen);
if (!(m->m_flags & M_SKIP_FIREWALL)){
struct m_tag *t;
STAM_PROFILE_START(&pData->StatALIAS_output, b);
if ((t = m_tag_find(m, PACKET_TAG_ALIAS, NULL)) != 0)
rc = LibAliasOut((struct libalias *)&t[1], mtod(m, char *),
m_length(m, NULL));
else
rc = LibAliasOut(pData->proxy_alias, mtod(m, char *),
m_length(m, NULL));
if (rc == PKT_ALIAS_IGNORED)
{
Log(("NAT: packet was droppped\n"));
goto exit_drop_package;
}
STAM_PROFILE_STOP(&pData->StatALIAS_output, b);
}
