OM_uint32
gss_unwrap(OM_uint32 *minor_status,
const gss_ctx_id_t ctx,
const gss_buffer_t input_message_buffer,
gss_buffer_t output_message_buffer,
int *conf_state,
gss_qop_t *qop_state)
{
OM_uint32 maj_stat;
struct mbuf *m;
if (!ctx) {
*minor_status = 0;
return (GSS_S_NO_CONTEXT);
}
MGET(m, M_WAITOK, MT_DATA);
if (input_message_buffer->length > MLEN)
MCLGET(m, M_WAITOK);
m_append(m, input_message_buffer->length, input_message_buffer->value);
maj_stat = KGSS_UNWRAP(ctx, minor_status, &m, conf_state, qop_state);
/*
* On success, m is the wrapped message, on failure, m is
* freed.
*/
if (maj_stat == GSS_S_COMPLETE) {
output_message_buffer->length = m_length(m, NULL);
output_message_buffer->value =
malloc(output_message_buffer->length,
M_GSSAPI, M_WAITOK);
m_copydata(m, 0, output_message_buffer->length,
output_message_buffer->value);
m_freem(m);
}
return (maj_stat);
}
int
cdcef_encap(struct cdcef_softc *sc, struct mbuf *m, int idx)
{
usbf_status err;
m_copydata(m, 0, m->m_pkthdr.len, sc->sc_buffer_in);
/* NO CRC */
usbf_setup_xfer(sc->sc_xfer_in, sc->sc_pipe_in, sc, sc->sc_buffer_in,
m->m_pkthdr.len, USBD_FORCE_SHORT_XFER | USBD_NO_COPY,
10000, cdcef_txeof);
err = usbf_transfer(sc->sc_xfer_in);
if (err && err != USBD_IN_PROGRESS) {
printf("encap error\n");
cdcef_stop(sc);
return (EIO);
}
sc->sc_xmit_mbuf = m;
return (0);
}
static void
ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
void *offset)
{
struct ntb_payload_header *hdr;
CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset);
if (entry->buf != NULL)
m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset);
hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame -
sizeof(struct ntb_payload_header));
hdr->len = entry->len; /* TODO: replace with bus_space_write */
hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */
wmb();
/* TODO: replace with bus_space_write */
hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG;
ntb_ring_sdb(qp->ntb, qp->qp_num);
/*
* The entry length can only be zero if the packet is intended to be a
* "link down" or similar. Since no payload is being sent in these
* cases, there is nothing to add to the completion queue.
*/
if (entry->len > 0) {
qp->tx_bytes += entry->len;
if (qp->tx_handler)
qp->tx_handler(qp, qp->cb_data, entry->cb_data,
entry->len);
}
CTR2(KTR_NTB,
"TX: entry %p sent. hdr->ver = %d, Returning to tx_free_q", entry,
hdr->ver);
ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
}
void
compat_wifi_output (struct mbuf *m)
{
sgIP_memblock *packet;
size_t size = 0;
struct mbuf *l;
/* compute total size of the packet */
for (l = m; l != NULL; l = l->m_next)
size += l->m_len;
/* allocate memblock */
packet = sgIP_memblock_allocHW (14, size - 14);
/* copy data from mbuf to memblock */
m_copydata (m, 0, size, packet->datastart);
/* call TransmitFunction */
hw.TransmitFunction (&hw, packet);
/* free mbuf */
m_freem (m);
}
/*
* Start transfer from internal queue
*/
static int
lgue_start_transfer(struct lgue_softc *sc) {
usbd_status err;
struct lgue_queue_entry *entry;
struct ifnet *ifp;
if (STAILQ_EMPTY(&sc->lgue_tx_queue))
return(0);
ifp = &sc->lgue_arpcom.ac_if;
entry = STAILQ_FIRST(&sc->lgue_tx_queue);
STAILQ_REMOVE_HEAD(&sc->lgue_tx_queue, entry_next);
m_copydata(entry->entry_mbuf, 0, entry->entry_mbuf->m_pkthdr.len,
sc->lgue_tx_buf);
/* Transmit */
usbd_setup_xfer(sc->lgue_tx_xfer, sc->lgue_ep[LGUE_ENDPT_TX], sc,
sc->lgue_tx_buf, entry->entry_mbuf->m_pkthdr.len,
USBD_SHORT_XFER_OK, USBD_NO_TIMEOUT, lgue_txeof);
err = usbd_transfer(sc->lgue_tx_xfer);
if (err != USBD_IN_PROGRESS) {
m_freem(entry->entry_mbuf);
kfree(entry, M_USBDEV);
lgue_stop(sc);
ifq_clr_oactive(&ifp->if_snd);
return(EIO);
}
m_freem(entry->entry_mbuf);
kfree(entry, M_USBDEV);
sc->lgue_tx_cnt++;
ifq_set_oactive(&ifp->if_snd);
ifp->if_timer = 5;
return(0);
}
int
an_mwrite_bap(struct an_softc *sc, int id, int off, struct mbuf *m, int totlen)
{
int error, len, cnt;
if (off == -1)
off = sc->sc_bap_off;
if (id != sc->sc_bap_id || off != sc->sc_bap_off) {
if ((error = an_seek_bap(sc, id, off)) != 0)
return EIO;
}
for (len = 0; m != NULL; m = m->m_next) {
if (m->m_len == 0)
continue;
len = min(m->m_len, totlen);
if ((mtod(m, u_long) & 0x1) || (len & 0x1)) {
m_copydata(m, 0, totlen, (caddr_t)&sc->sc_buf.sc_txbuf);
cnt = (totlen + 1) / 2;
an_swap16((u_int16_t *)&sc->sc_buf.sc_txbuf, cnt);
CSR_WRITE_MULTI_STREAM_2(sc, AN_DATA0,
sc->sc_buf.sc_val, cnt);
off += cnt * 2;
break;
}
cnt = len / 2;
an_swap16((u_int16_t *)mtod(m, u_int16_t *), cnt);
CSR_WRITE_MULTI_STREAM_2(sc, AN_DATA0, mtod(m, u_int16_t *),
cnt);
off += len;
totlen -= len;
}
sc->sc_bap_off = off;
return 0;
}
/*
* m_megapullup() - this function is a big hack.
* Thankfully, it's only used in ng_nat and ipfw+nat.
*
* It allocates an mbuf with cluster and copies the specified part of the chain
* into cluster, so that it is all contiguous and can be accessed via a plain
* (char *) pointer. This is required, because libalias doesn't know how to
* handle mbuf chains.
*
* On success, m_megapullup returns an mbuf (possibly with cluster) containing
* the input packet, on failure NULL. The input packet is always consumed.
*/
struct mbuf *
m_megapullup(struct mbuf *m, int len) {
struct mbuf *mcl;
if (len > m->m_pkthdr.len)
goto bad;
if (m->m_next == NULL && M_WRITABLE(m))
return (m);
mcl = m_get2(len, M_NOWAIT, MT_DATA, M_PKTHDR);
if (mcl == NULL)
goto bad;
m_align(mcl, len);
m_move_pkthdr(mcl, m);
m_copydata(m, 0, len, mtod(mcl, caddr_t));
mcl->m_len = mcl->m_pkthdr.len = len;
m_freem(m);
return (mcl);
bad:
m_freem(m);
return (NULL);
}
static void
btsco_sco_input(void *arg, struct mbuf *m)
{
struct btsco_softc *sc = arg;
int len;
DPRINTFN(10, "%s len=%d\n", sc->sc_name, m->m_pkthdr.len);
mutex_enter(&sc->sc_intr_lock);
if (sc->sc_rx_want == 0) {
m_freem(m);
} else {
KASSERT(sc->sc_rx_intr != NULL);
KASSERT(sc->sc_rx_block != NULL);
len = MIN(sc->sc_rx_want, m->m_pkthdr.len);
m_copydata(m, 0, len, sc->sc_rx_block);
sc->sc_rx_want -= len;
sc->sc_rx_block += len;
if (len > m->m_pkthdr.len) {
if (sc->sc_rx_mbuf != NULL)
m_freem(sc->sc_rx_mbuf);
m_adj(m, len);
sc->sc_rx_mbuf = m;
} else {
m_freem(m);
}
if (sc->sc_rx_want == 0)
(*sc->sc_rx_intr)(sc->sc_rx_intrarg);
}
mutex_exit(&sc->sc_intr_lock);
}
static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_pdu *ip;
struct icl_cxgbei_pdu *icp;
uint16_t len_ddp = be16toh(cpl->pdu_len_ddp);
uint16_t len = be16toh(cpl->len);
M_ASSERTPKTHDR(m);
MPASS(m->m_pkthdr.len == len + sizeof(*cpl));
ip = icl_cxgbei_new_pdu(M_NOWAIT);
if (ip == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
m_copydata(m, sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs);
ip->ip_data_len = G_ISCSI_PDU_LEN(len_ddp) - len;
icp = ip_to_icp(ip);
icp->icp_seq = ntohl(cpl->seq);
icp->icp_flags = ICPF_RX_HDR;
/* This is the start of a new PDU. There should be no old state. */
MPASS(toep->ulpcb2 == NULL);
toep->ulpcb2 = icp;
#if 0
CTR5(KTR_CXGBE, "%s: tid %u, cpl->len %u, pdu_len_ddp 0x%04x, icp %p",
__func__, tid, len, len_ddp, icp);
#endif
m_freem(m);
return (0);
}
/*
* ipsec_common_input gets called when an IPsec-protected packet
* is received by IPv4 or IPv6. Its job is to find the right SA
* and call the appropriate transform. The transform callback
* takes care of further processing (like ingress filtering).
*/
int
ipsec_common_input(struct mbuf *m, int skip, int protoff, int af, int sproto)
{
char buf[INET6_ADDRSTRLEN];
union sockaddr_union dst_address;
struct secasvar *sav;
u_int32_t spi;
int error;
#ifdef INET
#ifdef IPSEC_NAT_T
struct m_tag *tag;
#endif
#endif
IPSEC_ISTAT(sproto, input);
IPSEC_ASSERT(m != NULL, ("null packet"));
IPSEC_ASSERT(sproto == IPPROTO_ESP || sproto == IPPROTO_AH ||
sproto == IPPROTO_IPCOMP,
("unexpected security protocol %u", sproto));
if ((sproto == IPPROTO_ESP && !V_esp_enable) ||
(sproto == IPPROTO_AH && !V_ah_enable) ||
(sproto == IPPROTO_IPCOMP && !V_ipcomp_enable)) {
m_freem(m);
IPSEC_ISTAT(sproto, pdrops);
return EOPNOTSUPP;
}
if (m->m_pkthdr.len - skip < 2 * sizeof (u_int32_t)) {
m_freem(m);
IPSEC_ISTAT(sproto, hdrops);
DPRINTF(("%s: packet too small\n", __func__));
return EINVAL;
}
/* Retrieve the SPI from the relevant IPsec header */
if (sproto == IPPROTO_ESP)
m_copydata(m, skip, sizeof(u_int32_t), (caddr_t) &spi);
else if (sproto == IPPROTO_AH)
m_copydata(m, skip + sizeof(u_int32_t), sizeof(u_int32_t),
(caddr_t) &spi);
else if (sproto == IPPROTO_IPCOMP) {
u_int16_t cpi;
m_copydata(m, skip + sizeof(u_int16_t), sizeof(u_int16_t),
(caddr_t) &cpi);
spi = ntohl(htons(cpi));
}
/*
* Find the SA and (indirectly) call the appropriate
* kernel crypto routine. The resulting mbuf chain is a valid
* IP packet ready to go through input processing.
*/
bzero(&dst_address, sizeof (dst_address));
dst_address.sa.sa_family = af;
switch (af) {
#ifdef INET
case AF_INET:
dst_address.sin.sin_len = sizeof(struct sockaddr_in);
m_copydata(m, offsetof(struct ip, ip_dst),
sizeof(struct in_addr),
(caddr_t) &dst_address.sin.sin_addr);
#ifdef IPSEC_NAT_T
/* Find the source port for NAT-T; see udp*_espdecap. */
tag = m_tag_find(m, PACKET_TAG_IPSEC_NAT_T_PORTS, NULL);
if (tag != NULL)
dst_address.sin.sin_port = ((u_int16_t *)(tag + 1))[1];
#endif /* IPSEC_NAT_T */
break;
#endif /* INET */
#ifdef INET6
case AF_INET6:
dst_address.sin6.sin6_len = sizeof(struct sockaddr_in6);
m_copydata(m, offsetof(struct ip6_hdr, ip6_dst),
sizeof(struct in6_addr),
(caddr_t) &dst_address.sin6.sin6_addr);
/* We keep addresses in SADB without embedded scope id */
if (IN6_IS_SCOPE_LINKLOCAL(&dst_address.sin6.sin6_addr)) {
/* XXX: sa6_recoverscope() */
dst_address.sin6.sin6_scope_id =
ntohs(dst_address.sin6.sin6_addr.s6_addr16[1]);
dst_address.sin6.sin6_addr.s6_addr16[1] = 0;
}
break;
#endif /* INET6 */
default:
DPRINTF(("%s: unsupported protocol family %u\n", __func__, af));
m_freem(m);
IPSEC_ISTAT(sproto, nopf);
return EPFNOSUPPORT;
}
/* NB: only pass dst since key_allocsa follows RFC2401 */
sav = KEY_ALLOCSA(&dst_address, sproto, spi);
if (sav == NULL) {
DPRINTF(("%s: no key association found for SA %s/%08lx/%u\n",
__func__, ipsec_address(&dst_address, buf, sizeof(buf)),
(u_long) ntohl(spi), sproto));
IPSEC_ISTAT(sproto, notdb);
m_freem(m);
return ENOENT;
}
if (sav->tdb_xform == NULL) {
DPRINTF(("%s: attempted to use uninitialized SA %s/%08lx/%u\n",
__func__, ipsec_address(&dst_address, buf, sizeof(buf)),
(u_long) ntohl(spi), sproto));
IPSEC_ISTAT(sproto, noxform);
KEY_FREESAV(&sav);
m_freem(m);
return ENXIO;
}
/*
* Call appropriate transform and return -- callback takes care of
* everything else.
*/
error = (*sav->tdb_xform->xf_input)(m, sav, skip, protoff);
KEY_FREESAV(&sav);
return error;
}
void
mec_start(struct ifnet *ifp)
{
struct mec_softc *sc = ifp->if_softc;
struct mbuf *m0;
struct mec_txdesc *txd;
struct mec_txsoft *txs;
bus_dmamap_t dmamap;
bus_space_tag_t st = sc->sc_st;
bus_space_handle_t sh = sc->sc_sh;
uint64_t txdaddr;
int error, firsttx, nexttx, opending;
int len, bufoff, buflen, unaligned, txdlen;
if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
return;
/*
* Remember the previous txpending and the first transmit descriptor.
*/
opending = sc->sc_txpending;
firsttx = MEC_NEXTTX(sc->sc_txlast);
DPRINTF(MEC_DEBUG_START,
("mec_start: opending = %d, firsttx = %d\n", opending, firsttx));
for (;;) {
/* Grab a packet off the queue. */
IFQ_POLL(&ifp->if_snd, m0);
if (m0 == NULL)
break;
if (sc->sc_txpending == MEC_NTXDESC) {
break;
}
/*
* Get the next available transmit descriptor.
*/
nexttx = MEC_NEXTTX(sc->sc_txlast);
txd = &sc->sc_txdesc[nexttx];
txs = &sc->sc_txsoft[nexttx];
buflen = 0;
bufoff = 0;
txdaddr = 0; /* XXX gcc */
txdlen = 0; /* XXX gcc */
len = m0->m_pkthdr.len;
DPRINTF(MEC_DEBUG_START,
("mec_start: len = %d, nexttx = %d\n", len, nexttx));
IFQ_DEQUEUE(&ifp->if_snd, m0);
if (len < ETHER_PAD_LEN) {
/*
* I don't know if MEC chip does auto padding,
* so if the packet is small enough,
* just copy it to the buffer in txdesc.
* Maybe this is the simple way.
*/
DPRINTF(MEC_DEBUG_START, ("mec_start: short packet\n"));
bufoff = MEC_TXD_BUFSTART(ETHER_PAD_LEN);
m_copydata(m0, 0, m0->m_pkthdr.len,
txd->txd_buf + bufoff);
memset(txd->txd_buf + bufoff + len, 0,
ETHER_PAD_LEN - len);
len = buflen = ETHER_PAD_LEN;
txs->txs_flags = MEC_TXS_TXDBUF | buflen;
} else {
/*
* If the packet won't fit the buffer in txdesc,
* we have to use concatenate pointer to handle it.
* While MEC can handle up to three segments to
* concatenate, MEC requires that both the second and
* third segments have to be 8 byte aligned.
* Since it's unlikely for mbuf clusters, we use
* only the first concatenate pointer. If the packet
* doesn't fit in one DMA segment, allocate new mbuf
* and copy the packet to it.
*
* Besides, if the start address of the first segments
* is not 8 byte aligned, such part have to be copied
* to the txdesc buffer. (XXX see below comments)
*/
DPRINTF(MEC_DEBUG_START, ("mec_start: long packet\n"));
dmamap = txs->txs_dmamap;
if (bus_dmamap_load_mbuf(sc->sc_dmat, dmamap, m0,
BUS_DMA_WRITE | BUS_DMA_NOWAIT) != 0) {
struct mbuf *m;
DPRINTF(MEC_DEBUG_START,
("mec_start: re-allocating mbuf\n"));
MGETHDR(m, M_DONTWAIT, MT_DATA);
if (m == NULL) {
printf("%s: unable to allocate "
"TX mbuf\n", sc->sc_dev.dv_xname);
break;
}
if (len > (MHLEN - ETHER_ALIGN)) {
MCLGET(m, M_DONTWAIT);
if ((m->m_flags & M_EXT) == 0) {
printf("%s: unable to allocate "
"TX cluster\n",
sc->sc_dev.dv_xname);
m_freem(m);
break;
}
}
/*
* Each packet has the Ethernet header, so
* in many cases the header isn't 4-byte aligned
* and data after the header is 4-byte aligned.
* Thus adding 2-byte offset before copying to
* new mbuf avoids unaligned copy and this may
* improve performance.
* As noted above, unaligned part has to be
* copied to txdesc buffer so this may cause
* extra copy ops, but for now MEC always
* requires some data in txdesc buffer,
* so we always have to copy some data anyway.
*/
m->m_data += ETHER_ALIGN;
m_copydata(m0, 0, len, mtod(m, caddr_t));
m->m_pkthdr.len = m->m_len = len;
m_freem(m0);
m0 = m;
error = bus_dmamap_load_mbuf(sc->sc_dmat,
dmamap, m, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
if (error) {
printf("%s: unable to load TX buffer, "
"error = %d\n",
sc->sc_dev.dv_xname, error);
m_freem(m);
break;
}
}
/* Handle unaligned part. */
txdaddr = MEC_TXD_ROUNDUP(dmamap->dm_segs[0].ds_addr);
txs->txs_flags = MEC_TXS_TXDPTR1;
unaligned =
dmamap->dm_segs[0].ds_addr & (MEC_TXD_ALIGN - 1);
DPRINTF(MEC_DEBUG_START,
("mec_start: ds_addr = 0x%x, unaligned = %d\n",
(u_int)dmamap->dm_segs[0].ds_addr, unaligned));
if (unaligned != 0) {
buflen = MEC_TXD_ALIGN - unaligned;
bufoff = MEC_TXD_BUFSTART(buflen);
DPRINTF(MEC_DEBUG_START,
("mec_start: unaligned, "
"buflen = %d, bufoff = %d\n",
buflen, bufoff));
memcpy(txd->txd_buf + bufoff,
mtod(m0, caddr_t), buflen);
txs->txs_flags |= MEC_TXS_TXDBUF | buflen;
}
#if 1
else {
/*
* XXX needs hardware info XXX
* It seems MEC always requires some data
* in txd_buf[] even if buffer is
* 8-byte aligned otherwise DMA abort error
* occurs later...
*/
buflen = MEC_TXD_ALIGN;
bufoff = MEC_TXD_BUFSTART(buflen);
memcpy(txd->txd_buf + bufoff,
mtod(m0, caddr_t), buflen);
DPRINTF(MEC_DEBUG_START,
("mec_start: aligned, "
"buflen = %d, bufoff = %d\n",
buflen, bufoff));
txs->txs_flags |= MEC_TXS_TXDBUF | buflen;
txdaddr += MEC_TXD_ALIGN;
}
#endif
txdlen = len - buflen;
DPRINTF(MEC_DEBUG_START,
("mec_start: txdaddr = 0x%llx, txdlen = %d\n",
txdaddr, txdlen));
/*
* Sync the DMA map for TX mbuf.
*
* XXX unaligned part doesn't have to be sync'ed,
* but it's harmless...
*/
bus_dmamap_sync(sc->sc_dmat, dmamap, 0,
dmamap->dm_mapsize, BUS_DMASYNC_PREWRITE);
}
#if NBPFILTER > 0
/*
* Pass packet to bpf if there is a listener.
*/
if (ifp->if_bpf)
bpf_mtap(ifp->if_bpf, m0, BPF_DIRECTION_OUT);
#endif
/*
* Setup the transmit descriptor.
*/
/* TXINT bit will be set later on the last packet. */
txd->txd_cmd = (len - 1);
/* But also set TXINT bit on a half of TXDESC. */
if (sc->sc_txpending == (MEC_NTXDESC / 2))
txd->txd_cmd |= MEC_TXCMD_TXINT;
if (txs->txs_flags & MEC_TXS_TXDBUF)
txd->txd_cmd |= TXCMD_BUFSTART(MEC_TXDESCSIZE - buflen);
if (txs->txs_flags & MEC_TXS_TXDPTR1) {
txd->txd_cmd |= MEC_TXCMD_PTR1;
txd->txd_ptr[0] = TXPTR_LEN(txdlen - 1) | txdaddr;
/*
* Store a pointer to the packet so we can
* free it later.
*/
txs->txs_mbuf = m0;
} else {
txd->txd_ptr[0] = 0;
/*
* In this case all data are copied to buffer in txdesc,
* we can free TX mbuf here.
*/
m_freem(m0);
}
DPRINTF(MEC_DEBUG_START,
("mec_start: txd_cmd = 0x%llx, txd_ptr = 0x%llx\n",
txd->txd_cmd, txd->txd_ptr[0]));
DPRINTF(MEC_DEBUG_START,
("mec_start: len = %d (0x%04x), buflen = %d (0x%02x)\n",
len, len, buflen, buflen));
/* Sync TX descriptor. */
MEC_TXDESCSYNC(sc, nexttx,
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
/* Advance the TX pointer. */
sc->sc_txpending++;
sc->sc_txlast = nexttx;
}
if (sc->sc_txpending == MEC_NTXDESC) {
/* No more slots; notify upper layer. */
ifp->if_flags |= IFF_OACTIVE;
}
if (sc->sc_txpending != opending) {
/*
* Cause a TX interrupt to happen on the last packet
* we enqueued.
*/
sc->sc_txdesc[sc->sc_txlast].txd_cmd |= MEC_TXCMD_TXINT;
MEC_TXCMDSYNC(sc, sc->sc_txlast,
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
/* Start TX. */
bus_space_write_8(st, sh, MEC_TX_RING_PTR,
MEC_NEXTTX(sc->sc_txlast));
/*
* If the transmitter was idle,
* reset the txdirty pointer and re-enable TX interrupt.
*/
if (opending == 0) {
sc->sc_txdirty = firsttx;
bus_space_write_8(st, sh, MEC_TX_ALIAS,
MEC_TX_ALIAS_INT_ENABLE);
}
/* Set a watchdog timer in case the chip flakes out. */
ifp->if_timer = 5;
}
}
static
void
ip6_input(netmsg_t msg)
{
struct mbuf *m = msg->packet.nm_packet;
struct ip6_hdr *ip6;
int off = sizeof(struct ip6_hdr), nest;
u_int32_t plen;
u_int32_t rtalert = ~0;
int nxt, ours = 0, rh_present = 0;
struct ifnet *deliverifp = NULL;
struct in6_addr odst;
int srcrt = 0;
#ifdef IPSEC
/*
* should the inner packet be considered authentic?
* see comment in ah4_input().
*/
if (m) {
m->m_flags &= ~M_AUTHIPHDR;
m->m_flags &= ~M_AUTHIPDGM;
}
#endif
/*
* make sure we don't have onion peering information into m_aux.
*/
ip6_delaux(m);
/*
* mbuf statistics
*/
if (m->m_flags & M_EXT) {
if (m->m_next)
ip6stat.ip6s_mext2m++;
else
ip6stat.ip6s_mext1++;
} else {
#define M2MMAX NELEM(ip6stat.ip6s_m2m)
if (m->m_next) {
if (m->m_flags & M_LOOP) {
ip6stat.ip6s_m2m[loif[0].if_index]++; /* XXX */
} else if (m->m_pkthdr.rcvif->if_index < M2MMAX)
ip6stat.ip6s_m2m[m->m_pkthdr.rcvif->if_index]++;
else
ip6stat.ip6s_m2m[0]++;
} else
ip6stat.ip6s_m1++;
#undef M2MMAX
}
in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_receive);
ip6stat.ip6s_total++;
#ifndef PULLDOWN_TEST
/*
* L2 bridge code and some other code can return mbuf chain
* that does not conform to KAME requirement. too bad.
* XXX: fails to join if interface MTU > MCLBYTES. jumbogram?
*/
if (m && m->m_next != NULL && m->m_pkthdr.len < MCLBYTES) {
struct mbuf *n;
n = m_getb(m->m_pkthdr.len, MB_DONTWAIT, MT_HEADER, M_PKTHDR);
if (n == NULL)
goto bad;
M_MOVE_PKTHDR(n, m);
m_copydata(m, 0, n->m_pkthdr.len, mtod(n, caddr_t));
n->m_len = n->m_pkthdr.len;
m_freem(m);
m = n;
}
IP6_EXTHDR_CHECK_VOIDRET(m, 0, sizeof(struct ip6_hdr));
#endif
if (m->m_len < sizeof(struct ip6_hdr)) {
struct ifnet *inifp;
inifp = m->m_pkthdr.rcvif;
if ((m = m_pullup(m, sizeof(struct ip6_hdr))) == NULL) {
ip6stat.ip6s_toosmall++;
in6_ifstat_inc(inifp, ifs6_in_hdrerr);
goto bad2;
}
}
ip6 = mtod(m, struct ip6_hdr *);
if ((ip6->ip6_vfc & IPV6_VERSION_MASK) != IPV6_VERSION) {
ip6stat.ip6s_badvers++;
in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_hdrerr);
goto bad;
}
/*
* Run through list of hooks for input packets.
*
* NB: Beware of the destination address changing
* (e.g. by NAT rewriting). When this happens,
* tell ip6_forward to do the right thing.
*/
if (pfil_has_hooks(&inet6_pfil_hook)) {
odst = ip6->ip6_dst;
if (pfil_run_hooks(&inet6_pfil_hook, &m,
m->m_pkthdr.rcvif, PFIL_IN)) {
goto bad2;
}
if (m == NULL) /* consumed by filter */
goto bad2;
ip6 = mtod(m, struct ip6_hdr *);
srcrt = !IN6_ARE_ADDR_EQUAL(&odst, &ip6->ip6_dst);
}
static void
usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error)
{
struct usie_softc *sc = usbd_xfer_softc(xfer);
struct ifnet *ifp = sc->sc_ifp;
struct mbuf *m0;
struct mbuf *m = NULL;
struct usie_desc *rxd;
uint32_t actlen;
uint16_t err;
uint16_t pkt;
uint16_t ipl;
uint16_t len;
uint16_t diff;
uint8_t pad;
uint8_t ipv;
usbd_xfer_status(xfer, &actlen, NULL, NULL, NULL);
switch (USB_GET_STATE(xfer)) {
case USB_ST_TRANSFERRED:
DPRINTFN(15, "rx done, actlen=%u\n", actlen);
if (actlen < sizeof(struct usie_hip)) {
DPRINTF("data too short %u\n", actlen);
goto tr_setup;
}
m = sc->sc_rxm;
sc->sc_rxm = NULL;
/* fall though */
case USB_ST_SETUP:
tr_setup:
if (sc->sc_rxm == NULL) {
sc->sc_rxm = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR,
MJUMPAGESIZE /* could be bigger than MCLBYTES */ );
}
if (sc->sc_rxm == NULL) {
DPRINTF("could not allocate Rx mbuf\n");
ifp->if_ierrors++;
usbd_xfer_set_stall(xfer);
usbd_xfer_set_frames(xfer, 0);
} else {
/*
* Directly loading a mbuf cluster into DMA to
* save some data copying. This works because
* there is only one cluster.
*/
usbd_xfer_set_frame_data(xfer, 0,
mtod(sc->sc_rxm, caddr_t), MIN(MJUMPAGESIZE, USIE_RXSZ_MAX));
usbd_xfer_set_frames(xfer, 1);
}
usbd_transfer_submit(xfer);
break;
default: /* Error */
DPRINTF("USB transfer error, %s\n", usbd_errstr(error));
if (error != USB_ERR_CANCELLED) {
/* try to clear stall first */
usbd_xfer_set_stall(xfer);
ifp->if_ierrors++;
goto tr_setup;
}
if (sc->sc_rxm != NULL) {
m_freem(sc->sc_rxm);
sc->sc_rxm = NULL;
}
break;
}
if (m == NULL)
return;
mtx_unlock(&sc->sc_mtx);
m->m_pkthdr.len = m->m_len = actlen;
err = pkt = 0;
/* HW can aggregate multiple frames in a single USB xfer */
for (;;) {
rxd = mtod(m, struct usie_desc *);
len = be16toh(rxd->hip.len) & USIE_HIP_IP_LEN_MASK;
pad = (rxd->hip.id & USIE_HIP_PAD) ? 1 : 0;
ipl = (len - pad - ETHER_HDR_LEN);
if (ipl >= len) {
DPRINTF("Corrupt frame\n");
m_freem(m);
break;
}
diff = sizeof(struct usie_desc) + ipl + pad;
if (((rxd->hip.id & USIE_HIP_MASK) != USIE_HIP_IP) ||
(be16toh(rxd->desc_type) & USIE_TYPE_MASK) != USIE_IP_RX) {
DPRINTF("received wrong type of packet\n");
m->m_data += diff;
m->m_pkthdr.len = (m->m_len -= diff);
err++;
if (m->m_pkthdr.len > 0)
continue;
m_freem(m);
break;
}
switch (be16toh(rxd->ethhdr.ether_type)) {
case ETHERTYPE_IP:
ipv = NETISR_IP;
break;
#ifdef INET6
case ETHERTYPE_IPV6:
ipv = NETISR_IPV6;
break;
#endif
default:
DPRINTF("unsupported ether type\n");
err++;
break;
}
/* the last packet */
if (m->m_pkthdr.len <= diff) {
m->m_data += (sizeof(struct usie_desc) + pad);
m->m_pkthdr.len = m->m_len = ipl;
m->m_pkthdr.rcvif = ifp;
BPF_MTAP(sc->sc_ifp, m);
netisr_dispatch(ipv, m);
break;
}
/* copy aggregated frames to another mbuf */
m0 = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
if (__predict_false(m0 == NULL)) {
DPRINTF("could not allocate mbuf\n");
err++;
m_freem(m);
break;
}
m_copydata(m, sizeof(struct usie_desc) + pad, ipl, mtod(m0, caddr_t));
m0->m_pkthdr.rcvif = ifp;
m0->m_pkthdr.len = m0->m_len = ipl;
BPF_MTAP(sc->sc_ifp, m0);
netisr_dispatch(ipv, m0);
m->m_data += diff;
m->m_pkthdr.len = (m->m_len -= diff);
}
mtx_lock(&sc->sc_mtx);
ifp->if_ierrors += err;
ifp->if_ipackets += pkt;
}
static
#ifndef GPROF
__inline
#endif
int
tcp_build_datapkt(struct tcpcb *tp, struct socket *so, int off,
long len, int hdrlen, struct mbuf **mp)
{
struct mbuf *m, *m0;
if (tp->t_force && len == 1)
tcpstat.tcps_sndprobe++;
else if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
tcpstat.tcps_sndrexmitpack++;
tcpstat.tcps_sndrexmitbyte += len;
} else {
tcpstat.tcps_sndpack++;
tcpstat.tcps_sndbyte += len;
}
#ifdef notyet
if ((m = m_copypack(so->so_snd.sb_mb, off,
(int)len, max_linkhdr + hdrlen)) == 0)
return (ENOBUFS);
/*
* m_copypack left space for our hdr; use it.
*/
m->m_len += hdrlen;
m->m_data -= hdrlen;
#else
MGETHDR(m, M_DONTWAIT, MT_HEADER);
if (__predict_false(m == NULL))
return (ENOBUFS);
MCLAIM(m, &tcp_tx_mowner);
/*
* XXX Because other code assumes headers will fit in
* XXX one header mbuf.
*
* (This code should almost *never* be run.)
*/
if (__predict_false((max_linkhdr + hdrlen) > MHLEN)) {
TCP_OUTPUT_COUNTER_INCR(&tcp_output_bigheader);
MCLGET(m, M_DONTWAIT);
if ((m->m_flags & M_EXT) == 0) {
m_freem(m);
return (ENOBUFS);
}
}
m->m_data += max_linkhdr;
m->m_len = hdrlen;
/*
* To avoid traversing the whole sb_mb chain for correct
* data to send, remember last sent mbuf, its offset and
* the sent size. When called the next time, see if the
* data to send is directly following the previous transfer.
* This is important for large TCP windows.
*/
if (off == 0 || tp->t_lastm == NULL ||
(tp->t_lastoff + tp->t_lastlen) != off) {
TCP_OUTPUT_COUNTER_INCR(&tcp_output_predict_miss);
/*
* Either a new packet or a retransmit.
* Start from the beginning.
*/
tp->t_lastm = so->so_snd.sb_mb;
tp->t_inoff = off;
} else {
TCP_OUTPUT_COUNTER_INCR(&tcp_output_predict_hit);
tp->t_inoff += tp->t_lastlen;
}
/* Traverse forward to next packet */
while (tp->t_inoff > 0) {
if (tp->t_lastm == NULL)
panic("tp->t_lastm == NULL");
if (tp->t_inoff < tp->t_lastm->m_len)
break;
tp->t_inoff -= tp->t_lastm->m_len;
tp->t_lastm = tp->t_lastm->m_next;
}
tp->t_lastoff = off;
tp->t_lastlen = len;
m0 = tp->t_lastm;
off = tp->t_inoff;
if (len <= M_TRAILINGSPACE(m)) {
m_copydata(m0, off, (int) len, mtod(m, caddr_t) + hdrlen);
m->m_len += len;
TCP_OUTPUT_COUNTER_INCR(&tcp_output_copysmall);
} else {
m->m_next = m_copy(m0, off, (int) len);
if (m->m_next == NULL) {
m_freem(m);
return (ENOBUFS);
}
#ifdef TCP_OUTPUT_COUNTERS
if (m->m_next->m_flags & M_EXT)
TCP_OUTPUT_COUNTER_INCR(&tcp_output_refbig);
else
TCP_OUTPUT_COUNTER_INCR(&tcp_output_copybig);
#endif /* TCP_OUTPUT_COUNTERS */
}
#endif
*mp = m;
return (0);
}
/*
* Defragment a mbuf chain, returning the shortest possible
* chain of mbufs and clusters. If allocation fails and
* this cannot be completed, NULL will be returned, but
* the passed in chain will be unchanged. Upon success,
* the original chain will be freed, and the new chain
* will be returned.
*
* If a non-packet header is passed in, the original
* mbuf (chain?) will be returned unharmed.
*/
struct mbuf *
m_defrag(struct mbuf *m0, int how)
{
struct mbuf *m_new = NULL, *m_final = NULL;
int progress = 0, length;
MBUF_CHECKSLEEP(how);
if (!(m0->m_flags & M_PKTHDR))
return (m0);
m_fixhdr(m0); /* Needed sanity check */
#ifdef MBUF_STRESS_TEST
if (m_defragrandomfailures) {
int temp = arc4random() & 0xff;
if (temp == 0xba)
goto nospace;
}
#endif
if (m0->m_pkthdr.len > MHLEN)
m_final = m_getcl(how, MT_DATA, M_PKTHDR);
else
m_final = m_gethdr(how, MT_DATA);
if (m_final == NULL)
goto nospace;
if (m_dup_pkthdr(m_final, m0, how) == 0)
goto nospace;
m_new = m_final;
while (progress < m0->m_pkthdr.len) {
length = m0->m_pkthdr.len - progress;
if (length > MCLBYTES)
length = MCLBYTES;
if (m_new == NULL) {
if (length > MLEN)
m_new = m_getcl(how, MT_DATA, 0);
else
m_new = m_get(how, MT_DATA);
if (m_new == NULL)
goto nospace;
}
m_copydata(m0, progress, length, mtod(m_new, caddr_t));
progress += length;
m_new->m_len = length;
if (m_new != m_final)
m_cat(m_final, m_new);
m_new = NULL;
}
#ifdef MBUF_STRESS_TEST
if (m0->m_next == NULL)
m_defraguseless++;
#endif
m_freem(m0);
m0 = m_final;
#ifdef MBUF_STRESS_TEST
m_defragpackets++;
m_defragbytes += m0->m_pkthdr.len;
#endif
return (m0);
nospace:
#ifdef MBUF_STRESS_TEST
m_defragfailure++;
#endif
if (m_final)
m_freem(m_final);
return (NULL);
}
/*
* [remote_input] Packet, without data, from peer. Restore pkt->info,
* return 0 on success
*/
int
remote_input(struct mbuf *m)
{
u_int32_t ip_len;
struct remote_packet *info;
struct packet * pkt = NULL;
int err = 0, once = 1;
mn.stats.mc_recv++;
Dprintf(("remote_input: received foreign packet\n"), MN_P_PKTS);
ASSERT(m);
if (udp_validate(m)==0) {
printf("UDP fail\n");
return EINVAL;
}
ip_len = m->m_pkthdr.len;
m_adj(m, sizeof(struct udpiphdr));
while(m->m_pkthdr.len && once){
if (once > 46){
printf("remote_input: pkthdrlen(%d) once (%d)\n",m->m_pkthdr.len,once);
err = 1; /* too many 32 byte remote pkts in
1500-28 byte packet */
goto error;
}
once++;
if (m->m_pkthdr.len - sizeof(struct remote_packet) < 0){
printf("remote_input: m->m_pkthdr.len(%d) < sizeof(remote_packet)(%d)\n",
m->m_pkthdr.len, sizeof(struct remote_packet));
err = 1;
goto error;
}
info = mtod(m, struct remote_packet *);
pkt = malloc(sizeof(struct packet), M_MN_PKTS, M_NOWAIT);
if (!pkt) {
err = ENOBUFS;
goto error;
}
mn.stats.pkt_alloc++;
pkt->cachehost = 0;
pkt->info.id = (unsigned long) NULL;
pkt->m = NULL;
m_copydata(m,0,sizeof(struct remote_packet),(caddr_t)&(pkt->info));
m_adj(m, sizeof(struct remote_packet));
if (MC_PKT_PCACHED(pkt)) {
/* with aggregation we could have many */
if (!(mn.state & MN_AGGREGATE) && (ip_len !=
sizeof(struct remote_packet) + sizeof(struct udpiphdr))) {
printf("modelnet: got udp of wrong size (%u != %d!)\n", ip_len,
sizeof(struct remote_packet) + sizeof(struct udpiphdr));
err = EINVAL;
goto error;
}
} else { /* unencapsulate non-cached packets */
/* XXX untested ! */
/*pkt->m = m_split(pkt->m, sizeof(struct remote_packet), M_DONTWAIT);*/
printf("remote_input: aggregation with non pcached pkts\n");
pkt->m = m;
m = NULL;
once = 0; /* we take the rest */
}
if (info->ttl-- <= 0) {
DDB(printf("modelnet: ttl expired on pkt 0x%x\n", (int) info->id);
)
mn.stats.mc_ttldrops++;
MN_LOG_PACKET(info->src, info->dst, info->hop, 1);
err = EINVAL;
#ifdef DEBUG_REMOTE_INPUT
goto error;
#else
MN_FREE_PKT(pkt);
continue;
#endif
}
/*
* Apply a symmetric encryption/decryption algorithm.
*/
static int
swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
int flags)
{
unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat;
unsigned char *ivp, piv[EALG_MAX_BLOCK_LEN];
struct enc_xform *exf;
int i, k, j, blks;
exf = sw->sw_exf;
blks = exf->blocksize;
/* Check for non-padded data */
if (crd->crd_len % blks)
return EINVAL;
/* Initialize the IV */
if (crd->crd_flags & CRD_F_ENCRYPT) {
/* IV explicitly provided ? */
if (crd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(crd->crd_iv, iv, blks);
else
arc4rand(iv, blks, 0);
/* Do we need to write the IV */
if (!(crd->crd_flags & CRD_F_IV_PRESENT))
crypto_copyback(flags, buf, crd->crd_inject, blks, iv);
} else { /* Decryption */
/* IV explicitly provided ? */
if (crd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(crd->crd_iv, iv, blks);
else {
/* Get IV off buf */
crypto_copydata(flags, buf, crd->crd_inject, blks, iv);
}
}
if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
int error;
if (sw->sw_kschedule)
exf->zerokey(&(sw->sw_kschedule));
error = exf->setkey(&sw->sw_kschedule,
crd->crd_key, crd->crd_klen / 8);
if (error)
return (error);
}
ivp = iv;
/*
* xforms that provide a reinit method perform all IV
* handling themselves.
*/
if (exf->reinit)
exf->reinit(sw->sw_kschedule, iv);
if (flags & CRYPTO_F_IMBUF) {
struct mbuf *m = (struct mbuf *) buf;
/* Find beginning of data */
m = m_getptr(m, crd->crd_skip, &k);
if (m == NULL)
return EINVAL;
i = crd->crd_len;
while (i > 0) {
/*
* If there's insufficient data at the end of
* an mbuf, we have to do some copying.
*/
if (m->m_len < k + blks && m->m_len != k) {
m_copydata(m, k, blks, blk);
/* Actual encryption/decryption */
if (exf->reinit) {
if (crd->crd_flags & CRD_F_ENCRYPT) {
exf->encrypt(sw->sw_kschedule,
blk);
} else {
exf->decrypt(sw->sw_kschedule,
blk);
}
} else if (crd->crd_flags & CRD_F_ENCRYPT) {
/* XOR with previous block */
for (j = 0; j < blks; j++)
blk[j] ^= ivp[j];
exf->encrypt(sw->sw_kschedule, blk);
/*
* Keep encrypted block for XOR'ing
* with next block
*/
bcopy(blk, iv, blks);
ivp = iv;
} else { /* decrypt */
/*
* Keep encrypted block for XOR'ing
* with next block
*/
if (ivp == iv)
bcopy(blk, piv, blks);
else
bcopy(blk, iv, blks);
exf->decrypt(sw->sw_kschedule, blk);
/* XOR with previous block */
for (j = 0; j < blks; j++)
blk[j] ^= ivp[j];
if (ivp == iv)
bcopy(piv, iv, blks);
else
ivp = iv;
}
/* Copy back decrypted block */
m_copyback(m, k, blks, blk);
/* Advance pointer */
m = m_getptr(m, k + blks, &k);
if (m == NULL)
return EINVAL;
i -= blks;
/* Could be done... */
if (i == 0)
break;
}
/* Skip possibly empty mbufs */
if (k == m->m_len) {
for (m = m->m_next; m && m->m_len == 0;
m = m->m_next)
;
k = 0;
}
/* Sanity check */
if (m == NULL)
return EINVAL;
/*
* Warning: idat may point to garbage here, but
* we only use it in the while() loop, only if
* there are indeed enough data.
*/
idat = mtod(m, unsigned char *) + k;
while (m->m_len >= k + blks && i > 0) {
if (exf->reinit) {
if (crd->crd_flags & CRD_F_ENCRYPT) {
exf->encrypt(sw->sw_kschedule,
idat);
} else {
exf->decrypt(sw->sw_kschedule,
idat);
}
} else if (crd->crd_flags & CRD_F_ENCRYPT) {
/* XOR with previous block/IV */
for (j = 0; j < blks; j++)
idat[j] ^= ivp[j];
exf->encrypt(sw->sw_kschedule, idat);
ivp = idat;
} else { /* decrypt */
/*
* Keep encrypted block to be used
* in next block's processing.
*/
if (ivp == iv)
bcopy(idat, piv, blks);
else
bcopy(idat, iv, blks);
exf->decrypt(sw->sw_kschedule, idat);
/* XOR with previous block/IV */
for (j = 0; j < blks; j++)
idat[j] ^= ivp[j];
if (ivp == iv)
bcopy(piv, iv, blks);
else
ivp = iv;
}
idat += blks;
k += blks;
i -= blks;
}
}
return 0; /* Done with mbuf encryption/decryption */
} else if (flags & CRYPTO_F_IOV) {
/*
* ESP input processing, called (eventually) through the protocol switch.
*/
static int
esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
{
char buf[128];
struct auth_hash *esph;
struct enc_xform *espx;
struct tdb_crypto *tc;
uint8_t *ivp;
int plen, alen, hlen;
struct newesp *esp;
struct cryptodesc *crde;
struct cryptop *crp;
IPSEC_ASSERT(sav != NULL, ("null SA"));
IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
/* Valid IP Packet length ? */
if ( (skip&3) || (m->m_pkthdr.len&3) ){
DPRINTF(("%s: misaligned packet, skip %u pkt len %u",
__func__, skip, m->m_pkthdr.len));
ESPSTAT_INC(esps_badilen);
m_freem(m);
return EINVAL;
}
/* XXX don't pullup, just copy header */
IP6_EXTHDR_GET(esp, struct newesp *, m, skip, sizeof (struct newesp));
esph = sav->tdb_authalgxform;
espx = sav->tdb_encalgxform;
/* Determine the ESP header and auth length */
if (sav->flags & SADB_X_EXT_OLD)
hlen = sizeof (struct esp) + sav->ivlen;
else
hlen = sizeof (struct newesp) + sav->ivlen;
alen = xform_ah_authsize(esph);
/*
* Verify payload length is multiple of encryption algorithm
* block size.
*
* NB: This works for the null algorithm because the blocksize
* is 4 and all packets must be 4-byte aligned regardless
* of the algorithm.
*/
plen = m->m_pkthdr.len - (skip + hlen + alen);
if ((plen & (espx->blocksize - 1)) || (plen <= 0)) {
DPRINTF(("%s: payload of %d octets not a multiple of %d octets,"
" SA %s/%08lx\n", __func__, plen, espx->blocksize,
ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
(u_long)ntohl(sav->spi)));
ESPSTAT_INC(esps_badilen);
m_freem(m);
return EINVAL;
}
/*
* Check sequence number.
*/
if (esph != NULL && sav->replay != NULL &&
!ipsec_chkreplay(ntohl(esp->esp_seq), sav)) {
DPRINTF(("%s: packet replay check for %s\n", __func__,
ipsec_logsastr(sav, buf, sizeof(buf)))); /*XXX*/
ESPSTAT_INC(esps_replay);
m_freem(m);
return ENOBUFS; /*XXX*/
}
/* Update the counters */
ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen));
/* Get crypto descriptors */
crp = crypto_getreq(esph && espx ? 2 : 1);
if (crp == NULL) {
DPRINTF(("%s: failed to acquire crypto descriptors\n",
__func__));
ESPSTAT_INC(esps_crypto);
m_freem(m);
return ENOBUFS;
}
/* Get IPsec-specific opaque pointer */
tc = (struct tdb_crypto *) malloc(sizeof(struct tdb_crypto) + alen,
M_XDATA, M_NOWAIT | M_ZERO);
if (tc == NULL) {
crypto_freereq(crp);
DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
ESPSTAT_INC(esps_crypto);
m_freem(m);
return ENOBUFS;
}
if (esph != NULL) {
struct cryptodesc *crda = crp->crp_desc;
IPSEC_ASSERT(crda != NULL, ("null ah crypto descriptor"));
/* Authentication descriptor */
crda->crd_skip = skip;
if (SAV_ISGCM(sav))
crda->crd_len = 8; /* RFC4106 5, SPI + SN */
else
crda->crd_len = m->m_pkthdr.len - (skip + alen);
crda->crd_inject = m->m_pkthdr.len - alen;
crda->crd_alg = esph->type;
/* Copy the authenticator */
m_copydata(m, m->m_pkthdr.len - alen, alen,
(caddr_t) (tc + 1));
/* Chain authentication request */
crde = crda->crd_next;
} else {
crde = crp->crp_desc;
}
/* Crypto operation descriptor */
crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
crp->crp_buf = (caddr_t) m;
crp->crp_callback = esp_input_cb;
crp->crp_sid = sav->tdb_cryptoid;
crp->crp_opaque = (caddr_t) tc;
/* These are passed as-is to the callback */
tc->tc_spi = sav->spi;
tc->tc_dst = sav->sah->saidx.dst;
tc->tc_proto = sav->sah->saidx.proto;
tc->tc_protoff = protoff;
tc->tc_skip = skip;
KEY_ADDREFSA(sav);
tc->tc_sav = sav;
/* Decryption descriptor */
IPSEC_ASSERT(crde != NULL, ("null esp crypto descriptor"));
crde->crd_skip = skip + hlen;
crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
crde->crd_inject = skip + hlen - sav->ivlen;
if (SAV_ISCTRORGCM(sav)) {
ivp = &crde->crd_iv[0];
/* GCM IV Format: RFC4106 4 */
/* CTR IV Format: RFC3686 4 */
/* Salt is last four bytes of key, RFC4106 8.1 */
/* Nonce is last four bytes of key, RFC3686 5.1 */
memcpy(ivp, sav->key_enc->key_data +
_KEYLEN(sav->key_enc) - 4, 4);
if (SAV_ISCTR(sav)) {
/* Initial block counter is 1, RFC3686 4 */
be32enc(&ivp[sav->ivlen + 4], 1);
}
m_copydata(m, skip + hlen - sav->ivlen, sav->ivlen, &ivp[4]);
crde->crd_flags |= CRD_F_IV_EXPLICIT;
}
crde->crd_alg = espx->type;
return (crypto_dispatch(crp));
}
static int
do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cxgbei_data *ci = sc->iscsi_ulp_softc;
const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
struct socket *so;
struct sockbuf *sb;
struct tcpcb *tp;
struct icl_cxgbei_conn *icc;
struct icl_conn *ic;
struct icl_cxgbei_pdu *icp = toep->ulpcb2;
struct icl_pdu *ip;
u_int pdu_len, val;
MPASS(m == NULL);
/* Must already be assembling a PDU. */
MPASS(icp != NULL);
MPASS(icp->icp_flags & ICPF_RX_HDR); /* Data is optional. */
MPASS((icp->icp_flags & ICPF_RX_STATUS) == 0);
pdu_len = be16toh(cpl->len); /* includes everything. */
val = be32toh(cpl->ddpvld);
#if 0
CTR5(KTR_CXGBE,
"%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp_flags 0x%08x",
__func__, tid, pdu_len, val, icp->icp_flags);
#endif
icp->icp_flags |= ICPF_RX_STATUS;
ip = &icp->ip;
if (val & F_DDP_PADDING_ERR)
icp->icp_flags |= ICPF_PAD_ERR;
if (val & F_DDP_HDRCRC_ERR)
icp->icp_flags |= ICPF_HCRC_ERR;
if (val & F_DDP_DATACRC_ERR)
icp->icp_flags |= ICPF_DCRC_ERR;
if (val & F_DDP_PDU && ip->ip_data_mbuf == NULL) {
MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0);
MPASS(ip->ip_data_len > 0);
icp->icp_flags |= ICPF_RX_DDP;
counter_u64_add(ci->ddp_pdus, 1);
counter_u64_add(ci->ddp_bytes, ip->ip_data_len);
}
INP_WLOCK(inp);
if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
__func__, tid, pdu_len, inp->inp_flags);
INP_WUNLOCK(inp);
icl_cxgbei_conn_pdu_free(NULL, ip);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
tp = intotcpcb(inp);
MPASS(icp->icp_seq == tp->rcv_nxt);
MPASS(tp->rcv_wnd >= pdu_len);
tp->rcv_nxt += pdu_len;
tp->rcv_wnd -= pdu_len;
tp->t_rcvtime = ticks;
/* update rx credits */
toep->rx_credits += pdu_len;
t4_rcvd(&toep->td->tod, tp); /* XXX: sc->tom_softc.tod */
so = inp->inp_socket;
sb = &so->so_rcv;
SOCKBUF_LOCK(sb);
icc = toep->ulpcb;
if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) {
CTR5(KTR_CXGBE,
"%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x",
__func__, tid, pdu_len, icc, sb->sb_state);
SOCKBUF_UNLOCK(sb);
INP_WUNLOCK(inp);
INP_INFO_RLOCK(&V_tcbinfo);
INP_WLOCK(inp);
tp = tcp_drop(tp, ECONNRESET);
if (tp)
INP_WUNLOCK(inp);
INP_INFO_RUNLOCK(&V_tcbinfo);
icl_cxgbei_conn_pdu_free(NULL, ip);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
ic = &icc->ic;
icl_cxgbei_new_pdu_set_conn(ip, ic);
MPASS(m == NULL); /* was unused, we'll use it now. */
m = sbcut_locked(sb, sbused(sb)); /* XXXNP: toep->sb_cc accounting? */
if (__predict_false(m != NULL)) {
int len = m_length(m, NULL);
/*
* PDUs were received before the tid transitioned to ULP mode.
* Convert them to icl_cxgbei_pdus and send them to ICL before
* the PDU in icp/ip.
*/
CTR3(KTR_CXGBE, "%s: tid %u, %u bytes in so_rcv", __func__, tid,
len);
/* XXXNP: needs to be rewritten. */
if (len == sizeof(struct iscsi_bhs) || len == 4 + sizeof(struct
iscsi_bhs)) {
struct icl_cxgbei_pdu *icp0;
struct icl_pdu *ip0;
ip0 = icl_cxgbei_new_pdu(M_NOWAIT);
icl_cxgbei_new_pdu_set_conn(ip0, ic);
if (ip0 == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
icp0 = ip_to_icp(ip0);
icp0->icp_seq = 0; /* XXX */
icp0->icp_flags = ICPF_RX_HDR | ICPF_RX_STATUS;
m_copydata(m, 0, sizeof(struct iscsi_bhs), (void *)ip0->ip_bhs);
STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip0, ip_next);
}
m_freem(m);
}
STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
if ((icc->rx_flags & RXF_ACTIVE) == 0) {
struct cxgbei_worker_thread_softc *cwt = &cwt_softc[icc->cwt];
mtx_lock(&cwt->cwt_lock);
icc->rx_flags |= RXF_ACTIVE;
TAILQ_INSERT_TAIL(&cwt->rx_head, icc, rx_link);
if (cwt->cwt_state == CWT_SLEEPING) {
cwt->cwt_state = CWT_RUNNING;
cv_signal(&cwt->cwt_cv);
}
mtx_unlock(&cwt->cwt_lock);
}
SOCKBUF_UNLOCK(sb);
INP_WUNLOCK(inp);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
static void
aes_decrypt(const struct krb5_key_state *ks, struct mbuf *inout,
size_t skip, size_t len, void *ivec, size_t ivlen)
{
size_t blocklen = 16, plen;
struct {
uint8_t cn_1[16], cn[16];
} last2;
int i, off, t;
/*
* AES decryption with cyphertext stealing:
*
* CTSencrypt(C[0], ..., C[n], IV, K):
* len = length(C[n])
* E[n] = C[n-1]
* X = decrypt(E[n], K)
* P[n] = (X ^ C[n]){0..len-1}
* E[n-1] = {C[n,0],...,C[n,len-1],X[len],...,X[blocksize-1]}
* (P[0],...,P[n-1]) = CBCdecrypt(C[0],...,C[n-2],E[n-1], IV, K)
*/
plen = len % blocklen;
if (len == blocklen) {
/*
* Note: caller will ensure len >= blocklen.
*/
aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec, 0);
} else if (plen == 0) {
/*
* This is equivalent to CBC mode followed by swapping
* the last two blocks.
*/
off = skip + len - 2 * blocklen;
m_copydata(inout, off, 2 * blocklen, (void*) &last2);
m_copyback(inout, off, blocklen, last2.cn);
m_copyback(inout, off + blocklen, blocklen, last2.cn_1);
aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec, 0);
} else {
/*
* This is the difficult case. We first decrypt the
* second to last block with a zero IV to make X. The
* plaintext for the last block is the XOR of X and
* the last cyphertext block.
*
* We derive a new cypher text for the second to last
* block by mixing the unused bytes of X with the last
* cyphertext block. The result of that can be
* decrypted with the rest in CBC mode.
*/
off = skip + len - plen - blocklen;
aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, off, blocklen,
NULL, 0);
m_copydata(inout, off, blocklen + plen, (void*) &last2);
for (i = 0; i < plen; i++) {
t = last2.cn[i];
last2.cn[i] ^= last2.cn_1[i];
last2.cn_1[i] = t;
}
m_copyback(inout, off, blocklen + plen, (void*) &last2);
aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len - plen,
ivec, 0);
}
}
static void
aes_encrypt(const struct krb5_key_state *ks, struct mbuf *inout,
size_t skip, size_t len, void *ivec, size_t ivlen)
{
size_t blocklen = 16, plen;
struct {
uint8_t cn_1[16], cn[16];
} last2;
int i, off;
/*
* AES encryption with cyphertext stealing:
*
* CTSencrypt(P[0], ..., P[n], IV, K):
* len = length(P[n])
* (C[0], ..., C[n-2], E[n-1]) =
* CBCencrypt(P[0], ..., P[n-1], IV, K)
* P = pad(P[n], 0, blocksize)
* E[n] = CBCencrypt(P, E[n-1], K);
* C[n-1] = E[n]
* C[n] = E[n-1]{0..len-1}
*/
plen = len % blocklen;
if (len == blocklen) {
/*
* Note: caller will ensure len >= blocklen.
*/
aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec,
CRD_F_ENCRYPT);
} else if (plen == 0) {
/*
* This is equivalent to CBC mode followed by swapping
* the last two blocks. We assume that neither of the
* last two blocks cross iov boundaries.
*/
aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec,
CRD_F_ENCRYPT);
off = skip + len - 2 * blocklen;
m_copydata(inout, off, 2 * blocklen, (void*) &last2);
m_copyback(inout, off, blocklen, last2.cn);
m_copyback(inout, off + blocklen, blocklen, last2.cn_1);
} else {
/*
* This is the difficult case. We encrypt all but the
* last partial block first. We then create a padded
* copy of the last block and encrypt that using the
* second to last encrypted block as IV. Once we have
* the encrypted versions of the last two blocks, we
* reshuffle to create the final result.
*/
aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len - plen,
ivec, CRD_F_ENCRYPT);
/*
* Copy out the last two blocks, pad the last block
* and encrypt it. Rearrange to get the final
* result. The cyphertext for cn_1 is in cn. The
* cyphertext for cn is the first plen bytes of what
* is in cn_1 now.
*/
off = skip + len - blocklen - plen;
m_copydata(inout, off, blocklen + plen, (void*) &last2);
for (i = plen; i < blocklen; i++)
last2.cn[i] = 0;
aes_encrypt_1(ks, 0, last2.cn, 0, blocklen, last2.cn_1,
CRD_F_ENCRYPT);
m_copyback(inout, off, blocklen, last2.cn);
m_copyback(inout, off + blocklen, plen, last2.cn_1);
}
}
static void
ngc_send(netmsg_t netmsg)
{
struct socket *so = netmsg->send.base.nm_so;
struct mbuf *m = netmsg->send.nm_m;
struct sockaddr *addr = netmsg->send.nm_addr;
struct mbuf *control = netmsg->send.nm_control;
struct ngpcb *const pcbp = sotongpcb(so);
struct sockaddr_ng *const sap = (struct sockaddr_ng *) addr;
struct ng_mesg *msg;
struct mbuf *m0;
item_p item;
char *path = NULL;
int len, error = 0;
struct ng_apply_info *apply;
#ifdef NOTYET
if (control && (error = ng_internalize(control, td))) {
if (pcbp->sockdata == NULL) {
error = ENOTCONN;
goto release;
}
}
#else /* NOTYET */
if (control) {
error = EINVAL;
goto release;
}
#endif /* NOTYET */
/* Require destination as there may be >= 1 hooks on this node. */
if (addr == NULL) {
error = EDESTADDRREQ;
goto release;
}
/*
* Allocate an expendable buffer for the path, chop off
* the sockaddr header, and make sure it's NUL terminated.
*/
len = sap->sg_len - 2;
path = kmalloc(len + 1, M_NETGRAPH_PATH, M_WAITOK);
bcopy(sap->sg_data, path, len);
path[len] = '\0';
/*
* Move the actual message out of mbufs into a linear buffer.
* Start by adding up the size of the data. (could use mh_len?)
*/
for (len = 0, m0 = m; m0 != NULL; m0 = m0->m_next)
len += m0->m_len;
/*
* Move the data into a linear buffer as well.
* Messages are not delivered in mbufs.
*/
msg = kmalloc(len + 1, M_NETGRAPH_MSG, M_WAITOK);
m_copydata(m, 0, len, (char *)msg);
if (msg->header.version != NG_VERSION) {
kfree(msg, M_NETGRAPH_MSG);
error = EINVAL;
goto release;
}
/*
* Hack alert!
* We look into the message and if it mkpeers a node of unknown type, we
* try to load it. We need to do this now, in syscall thread, because if
* message gets queued and applied later we will get panic.
*/
if (msg->header.typecookie == NGM_GENERIC_COOKIE &&
msg->header.cmd == NGM_MKPEER) {
struct ngm_mkpeer *const mkp = (struct ngm_mkpeer *) msg->data;
struct ng_type *type;
if ((type = ng_findtype(mkp->type)) == NULL) {
char filename[NG_TYPESIZ + 3];
linker_file_t fileid;
if (!linker_api_available()) {
error = ENXIO;
goto done;
}
/* Not found, try to load it as a loadable module. */
ksnprintf(filename, sizeof(filename), "ng_%s.ko",
mkp->type);
error = linker_load_file(filename, &fileid);
if (error != 0) {
kfree(msg, M_NETGRAPH_MSG);
goto release;
}
/* See if type has been loaded successfully. */
if ((type = ng_findtype(mkp->type)) == NULL) {
kfree(msg, M_NETGRAPH_MSG);
(void)linker_file_unload(fileid);
error = ENXIO;
goto release;
}
}
}
item = ng_package_msg(msg, NG_WAITOK);
if ((error = ng_address_path((pcbp->sockdata->node), item, path, 0))
!= 0) {
#ifdef TRACE_MESSAGES
kprintf("ng_address_path: errx=%d\n", error);
#endif
goto release;
}
#ifdef TRACE_MESSAGES
kprintf("[%x]:<---------[socket]: c=<%d>cmd=%x(%s) f=%x #%d (%s)\n",
item->el_dest->nd_ID,
msg->header.typecookie,
msg->header.cmd,
msg->header.cmdstr,
msg->header.flags,
msg->header.token,
item->el_dest->nd_type->name);
#endif
SAVE_LINE(item);
/*
* We do not want the user thread to return from syscall until the
* item is processed by destination node. We register callback
* on the item, which will reply to the user thread when item
* was applied.
*/
apply = ng_alloc_apply();
bzero(apply, sizeof(*apply));
apply->apply = ng_socket_item_applied;
apply->context = &netmsg->send.base.lmsg;
item->apply = apply;
error = ng_snd_item(item, NG_PROGRESS);
release:
if (path != NULL)
kfree(path, M_NETGRAPH_PATH);
if (control != NULL)
m_freem(control);
if (m != NULL)
m_freem(m);
done:
if (error != EINPROGRESS)
lwkt_replymsg(&netmsg->send.base.lmsg, error);
}
/*
* IPsec input callback, called by the transform callback. Takes care of
* filtering and other sanity checks on the processed packet.
*/
int
ipsec_common_input_cb(struct mbuf *m, struct tdb *tdbp, int skip, int protoff,
struct m_tag *mt)
{
int af, sproto;
u_char prot;
#if NBPFILTER > 0
struct ifnet *encif;
#endif
#ifdef INET
struct ip *ip, ipn;
#endif /* INET */
#ifdef INET6
struct ip6_hdr *ip6, ip6n;
#endif /* INET6 */
struct m_tag *mtag;
struct tdb_ident *tdbi;
af = tdbp->tdb_dst.sa.sa_family;
sproto = tdbp->tdb_sproto;
tdbp->tdb_last_used = time_second;
/* Sanity check */
if (m == NULL) {
/* The called routine will print a message if necessary */
IPSEC_ISTAT(espstat.esps_badkcr, ahstat.ahs_badkcr,
ipcompstat.ipcomps_badkcr);
return EINVAL;
}
#ifdef INET
/* Fix IPv4 header */
if (af == AF_INET) {
if ((m->m_len < skip) && ((m = m_pullup(m, skip)) == NULL)) {
DPRINTF(("ipsec_common_input_cb(): processing failed "
"for SA %s/%08x\n", ipsp_address(tdbp->tdb_dst),
ntohl(tdbp->tdb_spi)));
IPSEC_ISTAT(espstat.esps_hdrops, ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
return ENOBUFS;
}
ip = mtod(m, struct ip *);
ip->ip_len = htons(m->m_pkthdr.len);
ip->ip_sum = 0;
ip->ip_sum = in_cksum(m, ip->ip_hl << 2);
prot = ip->ip_p;
/* IP-in-IP encapsulation */
if (prot == IPPROTO_IPIP) {
if (m->m_pkthdr.len - skip < sizeof(struct ip)) {
m_freem(m);
IPSEC_ISTAT(espstat.esps_hdrops,
ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
return EINVAL;
}
/* ipn will now contain the inner IPv4 header */
m_copydata(m, skip, sizeof(struct ip),
(caddr_t) &ipn);
/*
* Check that the inner source address is the same as
* the proxy address, if available.
*/
if ((tdbp->tdb_proxy.sa.sa_family == AF_INET &&
tdbp->tdb_proxy.sin.sin_addr.s_addr !=
INADDR_ANY &&
ipn.ip_src.s_addr !=
tdbp->tdb_proxy.sin.sin_addr.s_addr) ||
(tdbp->tdb_proxy.sa.sa_family != AF_INET &&
tdbp->tdb_proxy.sa.sa_family != 0)) {
#if ENCDEBUG
char addr[INET_ADDRSTRLEN];
#endif
DPRINTF(("ipsec_common_input_cb(): inner "
"source address %s doesn't correspond to "
"expected proxy source %s, SA %s/%08x\n",
inet_ntop(AF_INET, &ipn.ip_src,
addr, sizeof(addr)),
ipsp_address(tdbp->tdb_proxy),
ipsp_address(tdbp->tdb_dst),
ntohl(tdbp->tdb_spi)));
m_freem(m);
IPSEC_ISTAT(espstat.esps_pdrops,
ahstat.ahs_pdrops,
ipcompstat.ipcomps_pdrops);
return EACCES;
}
}
#ifdef INET6
/* IPv6-in-IP encapsulation. */
if (prot == IPPROTO_IPV6) {
if (m->m_pkthdr.len - skip < sizeof(struct ip6_hdr)) {
m_freem(m);
IPSEC_ISTAT(espstat.esps_hdrops,
ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
return EINVAL;
}
/* ip6n will now contain the inner IPv6 header. */
m_copydata(m, skip, sizeof(struct ip6_hdr),
(caddr_t) &ip6n);
/*
* Check that the inner source address is the same as
* the proxy address, if available.
*/
if ((tdbp->tdb_proxy.sa.sa_family == AF_INET6 &&
!IN6_IS_ADDR_UNSPECIFIED(&tdbp->tdb_proxy.sin6.sin6_addr) &&
!IN6_ARE_ADDR_EQUAL(&ip6n.ip6_src,
&tdbp->tdb_proxy.sin6.sin6_addr)) ||
(tdbp->tdb_proxy.sa.sa_family != AF_INET6 &&
tdbp->tdb_proxy.sa.sa_family != 0)) {
#if ENCDEBUG
char addr[INET6_ADDRSTRLEN];
#endif
DPRINTF(("ipsec_common_input_cb(): inner "
"source address %s doesn't correspond to "
"expected proxy source %s, SA %s/%08x\n",
inet_ntop(AF_INET6, &ip6n.ip6_src,
addr, sizeof(addr)),
ipsp_address(tdbp->tdb_proxy),
ipsp_address(tdbp->tdb_dst),
ntohl(tdbp->tdb_spi)));
m_freem(m);
IPSEC_ISTAT(espstat.esps_pdrops,
ahstat.ahs_pdrops,
ipcompstat.ipcomps_pdrops);
return EACCES;
}
}
#endif /* INET6 */
}
#endif /* INET */
#ifdef INET6
/* Fix IPv6 header */
if (af == AF_INET6)
{
if (m->m_len < sizeof(struct ip6_hdr) &&
(m = m_pullup(m, sizeof(struct ip6_hdr))) == NULL) {
DPRINTF(("ipsec_common_input_cb(): processing failed "
"for SA %s/%08x\n", ipsp_address(tdbp->tdb_dst),
ntohl(tdbp->tdb_spi)));
IPSEC_ISTAT(espstat.esps_hdrops, ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
return EACCES;
}
ip6 = mtod(m, struct ip6_hdr *);
ip6->ip6_plen = htons(m->m_pkthdr.len - skip);
/* Save protocol */
m_copydata(m, protoff, 1, (caddr_t) &prot);
#ifdef INET
/* IP-in-IP encapsulation */
if (prot == IPPROTO_IPIP) {
if (m->m_pkthdr.len - skip < sizeof(struct ip)) {
m_freem(m);
IPSEC_ISTAT(espstat.esps_hdrops,
ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
return EINVAL;
}
/* ipn will now contain the inner IPv4 header */
m_copydata(m, skip, sizeof(struct ip), (caddr_t) &ipn);
/*
* Check that the inner source address is the same as
* the proxy address, if available.
*/
if ((tdbp->tdb_proxy.sa.sa_family == AF_INET &&
tdbp->tdb_proxy.sin.sin_addr.s_addr !=
INADDR_ANY &&
ipn.ip_src.s_addr !=
tdbp->tdb_proxy.sin.sin_addr.s_addr) ||
(tdbp->tdb_proxy.sa.sa_family != AF_INET &&
tdbp->tdb_proxy.sa.sa_family != 0)) {
#if ENCDEBUG
char addr[INET_ADDRSTRLEN];
#endif
DPRINTF(("ipsec_common_input_cb(): inner "
"source address %s doesn't correspond to "
"expected proxy source %s, SA %s/%08x\n",
inet_ntop(AF_INET, &ipn.ip_src,
addr, sizeof(addr)),
ipsp_address(tdbp->tdb_proxy),
ipsp_address(tdbp->tdb_dst),
ntohl(tdbp->tdb_spi)));
m_freem(m);
IPSEC_ISTAT(espstat.esps_pdrops,
ahstat.ahs_pdrops,
ipcompstat.ipcomps_pdrops);
return EACCES;
}
}
#endif /* INET */
/* IPv6-in-IP encapsulation */
if (prot == IPPROTO_IPV6) {
if (m->m_pkthdr.len - skip < sizeof(struct ip6_hdr)) {
m_freem(m);
IPSEC_ISTAT(espstat.esps_hdrops,
ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
return EINVAL;
}
/* ip6n will now contain the inner IPv6 header. */
m_copydata(m, skip, sizeof(struct ip6_hdr),
(caddr_t) &ip6n);
/*
* Check that the inner source address is the same as
* the proxy address, if available.
*/
if ((tdbp->tdb_proxy.sa.sa_family == AF_INET6 &&
!IN6_IS_ADDR_UNSPECIFIED(&tdbp->tdb_proxy.sin6.sin6_addr) &&
!IN6_ARE_ADDR_EQUAL(&ip6n.ip6_src,
&tdbp->tdb_proxy.sin6.sin6_addr)) ||
(tdbp->tdb_proxy.sa.sa_family != AF_INET6 &&
tdbp->tdb_proxy.sa.sa_family != 0)) {
#if ENCDEBUG
char addr[INET6_ADDRSTRLEN];
#endif
DPRINTF(("ipsec_common_input_cb(): inner "
"source address %s doesn't correspond to "
"expected proxy source %s, SA %s/%08x\n",
inet_ntop(AF_INET6, &ip6n.ip6_src,
addr, sizeof(addr)),
ipsp_address(tdbp->tdb_proxy),
ipsp_address(tdbp->tdb_dst),
ntohl(tdbp->tdb_spi)));
m_freem(m);
IPSEC_ISTAT(espstat.esps_pdrops,
ahstat.ahs_pdrops,
ipcompstat.ipcomps_pdrops);
return EACCES;
}
}
}
#endif /* INET6 */
/*
* Fix TCP/UDP checksum of UDP encapsulated transport mode ESP packet.
* (RFC3948 3.1.2)
*/
if ((af == AF_INET || af == AF_INET6) &&
(tdbp->tdb_flags & TDBF_UDPENCAP) &&
(tdbp->tdb_flags & TDBF_TUNNELING) == 0) {
u_int16_t cksum;
switch (prot) {
case IPPROTO_UDP:
if (m->m_pkthdr.len < skip + sizeof(struct udphdr)) {
m_freem(m);
IPSEC_ISTAT(espstat.esps_hdrops,
ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
return EINVAL;
}
cksum = 0;
m_copyback(m, skip + offsetof(struct udphdr, uh_sum),
sizeof(cksum), &cksum, M_NOWAIT);
#ifdef INET6
if (af == AF_INET6) {
cksum = in6_cksum(m, IPPROTO_UDP, skip,
m->m_pkthdr.len - skip);
m_copyback(m, skip + offsetof(struct udphdr,
uh_sum), sizeof(cksum), &cksum, M_NOWAIT);
}
#endif
break;
case IPPROTO_TCP:
if (m->m_pkthdr.len < skip + sizeof(struct tcphdr)) {
m_freem(m);
IPSEC_ISTAT(espstat.esps_hdrops,
ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
return EINVAL;
}
cksum = 0;
m_copyback(m, skip + offsetof(struct tcphdr, th_sum),
sizeof(cksum), &cksum, M_NOWAIT);
if (af == AF_INET)
cksum = in4_cksum(m, IPPROTO_TCP, skip,
m->m_pkthdr.len - skip);
#ifdef INET6
else if (af == AF_INET6)
cksum = in6_cksum(m, IPPROTO_TCP, skip,
m->m_pkthdr.len - skip);
#endif
m_copyback(m, skip + offsetof(struct tcphdr, th_sum),
sizeof(cksum), &cksum, M_NOWAIT);
break;
}
/*
* ipsec_common_input() gets called when we receive an IPsec-protected packet
* in IPv4 or IPv6. All it does is find the right TDB and call the appropriate
* transform. The callback takes care of further processing (like ingress
* filtering).
*/
int
ipsec_common_input(struct mbuf *m, int skip, int protoff, int af, int sproto,
int udpencap)
{
#define IPSEC_ISTAT(x,y,z) (sproto == IPPROTO_ESP ? (x)++ : \
sproto == IPPROTO_AH ? (y)++ : (z)++)
union sockaddr_union dst_address;
struct timeval tv;
struct tdb *tdbp;
struct ifnet *encif;
u_int32_t spi;
u_int16_t cpi;
int s, error;
IPSEC_ISTAT(espstat.esps_input, ahstat.ahs_input,
ipcompstat.ipcomps_input);
if (m == 0) {
DPRINTF(("ipsec_common_input(): NULL packet received\n"));
IPSEC_ISTAT(espstat.esps_hdrops, ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
return EINVAL;
}
if ((sproto == IPPROTO_ESP && !esp_enable) ||
(sproto == IPPROTO_AH && !ah_enable) ||
#if NPF > 0
(m->m_pkthdr.pf.flags & PF_TAG_DIVERTED) ||
#endif
(sproto == IPPROTO_IPCOMP && !ipcomp_enable)) {
switch (af) {
#ifdef INET
case AF_INET:
rip_input(m, skip, sproto);
break;
#endif /* INET */
#ifdef INET6
case AF_INET6:
rip6_input(&m, &skip, sproto);
break;
#endif /* INET6 */
default:
DPRINTF(("ipsec_common_input(): unsupported protocol "
"family %d\n", af));
m_freem(m);
IPSEC_ISTAT(espstat.esps_nopf, ahstat.ahs_nopf,
ipcompstat.ipcomps_nopf);
return EPFNOSUPPORT;
}
return 0;
}
if ((sproto == IPPROTO_IPCOMP) && (m->m_flags & M_COMP)) {
m_freem(m);
ipcompstat.ipcomps_pdrops++;
DPRINTF(("ipsec_common_input(): repeated decompression\n"));
return EINVAL;
}
if (m->m_pkthdr.len - skip < 2 * sizeof(u_int32_t)) {
m_freem(m);
IPSEC_ISTAT(espstat.esps_hdrops, ahstat.ahs_hdrops,
ipcompstat.ipcomps_hdrops);
DPRINTF(("ipsec_common_input(): packet too small\n"));
return EINVAL;
}
/* Retrieve the SPI from the relevant IPsec header */
if (sproto == IPPROTO_ESP)
m_copydata(m, skip, sizeof(u_int32_t), (caddr_t) &spi);
else if (sproto == IPPROTO_AH)
m_copydata(m, skip + sizeof(u_int32_t), sizeof(u_int32_t),
(caddr_t) &spi);
else if (sproto == IPPROTO_IPCOMP) {
m_copydata(m, skip + sizeof(u_int16_t), sizeof(u_int16_t),
(caddr_t) &cpi);
spi = ntohl(htons(cpi));
}
/*
* Find tunnel control block and (indirectly) call the appropriate
* kernel crypto routine. The resulting mbuf chain is a valid
* IP packet ready to go through input processing.
*/
memset(&dst_address, 0, sizeof(dst_address));
dst_address.sa.sa_family = af;
switch (af) {
#ifdef INET
case AF_INET:
dst_address.sin.sin_len = sizeof(struct sockaddr_in);
m_copydata(m, offsetof(struct ip, ip_dst),
sizeof(struct in_addr),
(caddr_t) &(dst_address.sin.sin_addr));
break;
#endif /* INET */
#ifdef INET6
case AF_INET6:
dst_address.sin6.sin6_len = sizeof(struct sockaddr_in6);
m_copydata(m, offsetof(struct ip6_hdr, ip6_dst),
sizeof(struct in6_addr),
(caddr_t) &(dst_address.sin6.sin6_addr));
in6_recoverscope(&dst_address.sin6, &dst_address.sin6.sin6_addr,
NULL);
break;
#endif /* INET6 */
default:
DPRINTF(("ipsec_common_input(): unsupported protocol "
"family %d\n", af));
m_freem(m);
IPSEC_ISTAT(espstat.esps_nopf, ahstat.ahs_nopf,
ipcompstat.ipcomps_nopf);
return EPFNOSUPPORT;
}
s = splsoftnet();
tdbp = gettdb(rtable_l2(m->m_pkthdr.ph_rtableid),
spi, &dst_address, sproto);
if (tdbp == NULL) {
splx(s);
DPRINTF(("ipsec_common_input(): could not find SA for "
"packet to %s, spi %08x\n",
ipsp_address(dst_address), ntohl(spi)));
m_freem(m);
IPSEC_ISTAT(espstat.esps_notdb, ahstat.ahs_notdb,
ipcompstat.ipcomps_notdb);
return ENOENT;
}
if (tdbp->tdb_flags & TDBF_INVALID) {
splx(s);
DPRINTF(("ipsec_common_input(): attempted to use invalid SA %s/%08x/%u\n", ipsp_address(dst_address), ntohl(spi), tdbp->tdb_sproto));
m_freem(m);
IPSEC_ISTAT(espstat.esps_invalid, ahstat.ahs_invalid,
ipcompstat.ipcomps_invalid);
return EINVAL;
}
if (udpencap && !(tdbp->tdb_flags & TDBF_UDPENCAP)) {
splx(s);
DPRINTF(("ipsec_common_input(): attempted to use non-udpencap SA %s/%08x/%u\n", ipsp_address(dst_address), ntohl(spi), tdbp->tdb_sproto));
m_freem(m);
espstat.esps_udpinval++;
return EINVAL;
}
if (tdbp->tdb_xform == NULL) {
splx(s);
DPRINTF(("ipsec_common_input(): attempted to use uninitialized SA %s/%08x/%u\n", ipsp_address(dst_address), ntohl(spi), tdbp->tdb_sproto));
m_freem(m);
IPSEC_ISTAT(espstat.esps_noxform, ahstat.ahs_noxform,
ipcompstat.ipcomps_noxform);
return ENXIO;
}
if (sproto != IPPROTO_IPCOMP) {
if ((encif = enc_getif(tdbp->tdb_rdomain,
tdbp->tdb_tap)) == NULL) {
splx(s);
DPRINTF(("ipsec_common_input(): "
"no enc%u interface for SA %s/%08x/%u\n",
tdbp->tdb_tap, ipsp_address(dst_address),
ntohl(spi), tdbp->tdb_sproto));
m_freem(m);
IPSEC_ISTAT(espstat.esps_pdrops,
ahstat.ahs_pdrops,
ipcompstat.ipcomps_pdrops);
return EACCES;
}
/* XXX This conflicts with the scoped nature of IPv6 */
m->m_pkthdr.rcvif = encif;
}
/* Register first use, setup expiration timer. */
if (tdbp->tdb_first_use == 0) {
tdbp->tdb_first_use = time_second;
tv.tv_usec = 0;
tv.tv_sec = tdbp->tdb_exp_first_use + tdbp->tdb_first_use;
if (tdbp->tdb_flags & TDBF_FIRSTUSE)
timeout_add(&tdbp->tdb_first_tmo, hzto(&tv));
tv.tv_sec = tdbp->tdb_first_use + tdbp->tdb_soft_first_use;
if (tdbp->tdb_flags & TDBF_SOFT_FIRSTUSE)
timeout_add(&tdbp->tdb_sfirst_tmo, hzto(&tv));
}
/*
* Call appropriate transform and return -- callback takes care of
* everything else.
*/
error = (*(tdbp->tdb_xform->xf_input))(m, tdbp, skip, protoff);
splx(s);
return error;
}
/*
* Synchronize an ISA DMA map.
*/
void
isadma_bounce_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
bus_size_t len, int ops)
{
struct isadma_bounce_cookie *cookie = map->_dm_cookie;
/*
* Mixing PRE and POST operations is not allowed.
*/
if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 &&
(ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0)
panic("isadma_bounce_dmamap_sync: mix PRE and POST");
#ifdef DIAGNOSTIC
if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) != 0) {
if (offset >= map->dm_mapsize)
panic("isadma_bounce_dmamap_sync: bad offset");
if (len == 0 || (offset + len) > map->dm_mapsize)
panic("isadma_bounce_dmamap_sync: bad length");
}
#endif
/*
* If we're not bouncing, just do the normal sync operation
* and return.
*/
if ((cookie->id_flags & ID_IS_BOUNCING) == 0) {
_bus_dmamap_sync(t, map, offset, len, ops);
return;
}
/*
* Flush data cache for PREREAD. This has the side-effect
* of invalidating the cache. Done at PREREAD since it
* causes the cache line(s) to be written back to memory.
*
* Copy the original buffer to the bounce buffer and flush
* the data cache for PREWRITE, so that the contents
* of the data buffer in memory reflect reality.
*
* Copy the bounce buffer to the original buffer in POSTREAD.
*/
switch (cookie->id_buftype) {
case ID_BUFTYPE_LINEAR:
/*
* Nothing to do for pre-read.
*/
if (ops & BUS_DMASYNC_PREWRITE) {
/*
* Copy the caller's buffer to the bounce buffer.
*/
memcpy((char *)cookie->id_bouncebuf + offset,
(char *)cookie->id_origbuf + offset, len);
wbflush();
}
if (ops & BUS_DMASYNC_POSTREAD) {
/*
* Copy the bounce buffer to the caller's buffer.
*/
memcpy((char *)cookie->id_origbuf + offset,
(char *)cookie->id_bouncebuf + offset, len);
}
/*
* Nothing to do for post-write.
*/
break;
case ID_BUFTYPE_MBUF:
{
struct mbuf *m, *m0 = cookie->id_origbuf;
bus_size_t minlen, moff;
/*
* Nothing to do for pre-read.
*/
if (ops & BUS_DMASYNC_PREWRITE) {
/*
* Copy the caller's buffer to the bounce buffer.
*/
m_copydata(m0, offset, len,
(char *)cookie->id_bouncebuf + offset);
}
if (ops & BUS_DMASYNC_POSTREAD) {
/*
* Copy the bounce buffer to the caller's buffer.
*/
for (moff = offset, m = m0; m != NULL && len != 0;
m = m->m_next) {
/* Find the beginning mbuf. */
if (moff >= m->m_len) {
moff -= m->m_len;
continue;
}
/*
* Now at the first mbuf to sync; nail
* each one until we have exhausted the
* length.
*/
minlen = len < m->m_len - moff ?
len : m->m_len - moff;
memcpy(mtod(m, caddr_t) + moff,
(char *)cookie->id_bouncebuf + offset,
minlen);
moff = 0;
len -= minlen;
offset += minlen;
}
}
/*
* Nothing to do for post-write.
*/
break;
}
case ID_BUFTYPE_UIO:
panic("isadma_bounce_dmamap_sync: ID_BUFTYPE_UIO");
break;
case ID_BUFTYPE_RAW:
panic("isadma_bounce_dmamap_sync: ID_BUFTYPE_RAW");
break;
case ID_BUFTYPE_INVALID:
panic("isadma_bounce_dmamap_sync: ID_BUFTYPE_INVALID");
break;
default:
printf("unknown buffer type %d\n", cookie->id_buftype);
panic("isadma_bounce_dmamap_sync");
}
/* Drain the write buffer. */
wbflush();
/* XXXJRT */
if (ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE))
mips_dcache_wbinv_range((vaddr_t)cookie->id_bouncebuf + offset,
len);
}
/*
* Massage IPv4/IPv6 headers for AH processing.
*/
int
ah_massage_headers(struct mbuf **m0, int proto, int skip, int alg, int out)
{
struct mbuf *m = *m0;
unsigned char *ptr;
int off, count;
#ifdef INET
struct ip *ip;
#endif /* INET */
#ifdef INET6
struct ip6_ext *ip6e;
struct ip6_hdr ip6;
int ad, alloc, nxt;
#endif /* INET6 */
switch (proto) {
#ifdef INET
case AF_INET:
/*
* This is the least painful way of dealing with IPv4 header
* and option processing -- just make sure they're in
* contiguous memory.
*/
*m0 = m = m_pullup(m, skip);
if (m == NULL) {
DPRINTF(("ah_massage_headers(): m_pullup() failed\n"));
ahstat.ahs_hdrops++;
return ENOBUFS;
}
/* Fix the IP header */
ip = mtod(m, struct ip *);
ip->ip_tos = 0;
ip->ip_ttl = 0;
ip->ip_sum = 0;
/*
* On input, fix ip_len which has been byte-swapped
* at ip_input().
*/
if (alg == CRYPTO_MD5_KPDK || alg == CRYPTO_SHA1_KPDK)
ip->ip_off &= htons(IP_DF);
else
ip->ip_off = 0;
ptr = mtod(m, unsigned char *) + sizeof(struct ip);
/* IPv4 option processing */
for (off = sizeof(struct ip); off < skip;) {
if (ptr[off] == IPOPT_EOL || ptr[off] == IPOPT_NOP ||
off + 1 < skip)
;
else {
DPRINTF(("ah_massage_headers(): illegal IPv4 "
"option length for option %d\n",
ptr[off]));
ahstat.ahs_hdrops++;
m_freem(m);
return EINVAL;
}
switch (ptr[off]) {
case IPOPT_EOL:
off = skip; /* End the loop. */
break;
case IPOPT_NOP:
off++;
break;
case IPOPT_SECURITY: /* 0x82 */
case 0x85: /* Extended security. */
case 0x86: /* Commercial security. */
case 0x94: /* Router alert */
case 0x95: /* RFC1770 */
/* Sanity check for option length. */
if (ptr[off + 1] < 2) {
DPRINTF(("ah_massage_headers(): "
"illegal IPv4 option length for "
"option %d\n", ptr[off]));
ahstat.ahs_hdrops++;
m_freem(m);
return EINVAL;
}
off += ptr[off + 1];
break;
case IPOPT_LSRR:
case IPOPT_SSRR:
/* Sanity check for option length. */
if (ptr[off + 1] < 2) {
DPRINTF(("ah_massage_headers(): "
"illegal IPv4 option length for "
"option %d\n", ptr[off]));
ahstat.ahs_hdrops++;
m_freem(m);
return EINVAL;
}
/*
* On output, if we have either of the
* source routing options, we should
* swap the destination address of the
* IP header with the last address
* specified in the option, as that is
* what the destination's IP header
* will look like.
*/
if (out)
bcopy(ptr + off + ptr[off + 1] -
sizeof(struct in_addr),
&(ip->ip_dst), sizeof(struct in_addr));
/* FALLTHROUGH */
default:
/* Sanity check for option length. */
if (ptr[off + 1] < 2) {
DPRINTF(("ah_massage_headers(): "
"illegal IPv4 option length for "
"option %d\n", ptr[off]));
ahstat.ahs_hdrops++;
m_freem(m);
return EINVAL;
}
/* Zeroize all other options. */
count = ptr[off + 1];
bcopy(ipseczeroes, ptr, count);
off += count;
break;
}
/* Sanity check. */
if (off > skip) {
DPRINTF(("ah_massage_headers(): malformed "
"IPv4 options header\n"));
ahstat.ahs_hdrops++;
m_freem(m);
return EINVAL;
}
}
break;
#endif /* INET */
#ifdef INET6
case AF_INET6: /* Ugly... */
/* Copy and "cook" the IPv6 header. */
m_copydata(m, 0, sizeof(ip6), (caddr_t) &ip6);
/* We don't do IPv6 Jumbograms. */
if (ip6.ip6_plen == 0) {
DPRINTF(("ah_massage_headers(): unsupported IPv6 "
"jumbogram"));
ahstat.ahs_hdrops++;
m_freem(m);
return EMSGSIZE;
}
ip6.ip6_flow = 0;
ip6.ip6_hlim = 0;
ip6.ip6_vfc &= ~IPV6_VERSION_MASK;
ip6.ip6_vfc |= IPV6_VERSION;
/* Scoped address handling. */
if (IN6_IS_SCOPE_EMBED(&ip6.ip6_src))
ip6.ip6_src.s6_addr16[1] = 0;
if (IN6_IS_SCOPE_EMBED(&ip6.ip6_dst))
ip6.ip6_dst.s6_addr16[1] = 0;
/* Done with IPv6 header. */
m_copyback(m, 0, sizeof(struct ip6_hdr), &ip6, M_NOWAIT);
/* Let's deal with the remaining headers (if any). */
if (skip - sizeof(struct ip6_hdr) > 0) {
if (m->m_len <= skip) {
ptr = malloc(skip - sizeof(struct ip6_hdr),
M_XDATA, M_NOWAIT);
if (ptr == NULL) {
DPRINTF(("ah_massage_headers(): failed to allocate memory for IPv6 headers\n"));
ahstat.ahs_hdrops++;
m_freem(m);
return ENOBUFS;
}
/*
* Copy all the protocol headers after
* the IPv6 header.
*/
m_copydata(m, sizeof(struct ip6_hdr),
skip - sizeof(struct ip6_hdr), ptr);
alloc = 1;
} else {
/* No need to allocate memory. */
ptr = mtod(m, unsigned char *) +
sizeof(struct ip6_hdr);
alloc = 0;
}
} else
break;
nxt = ip6.ip6_nxt & 0xff; /* Next header type. */
for (off = 0; off < skip - sizeof(struct ip6_hdr);) {
switch (nxt) {
case IPPROTO_HOPOPTS:
case IPPROTO_DSTOPTS:
ip6e = (struct ip6_ext *) (ptr + off);
/*
* Process the mutable/immutable
* options -- borrows heavily from the
* KAME code.
*/
for (count = off + sizeof(struct ip6_ext);
count < off + ((ip6e->ip6e_len + 1) << 3);) {
if (ptr[count] == IP6OPT_PAD1) {
count++;
continue; /* Skip padding. */
}
/* Sanity check. */
if (count > off +
((ip6e->ip6e_len + 1) << 3)) {
ahstat.ahs_hdrops++;
m_freem(m);
/* Free, if we allocated. */
if (alloc)
free(ptr, M_XDATA);
return EINVAL;
}
ad = ptr[count + 1];
/* If mutable option, zeroize. */
if (ptr[count] & IP6OPT_MUTABLE)
bcopy(ipseczeroes, ptr + count,
ptr[count + 1]);
count += ad;
/* Sanity check. */
if (count >
skip - sizeof(struct ip6_hdr)) {
ahstat.ahs_hdrops++;
m_freem(m);
/* Free, if we allocated. */
if (alloc)
free(ptr, M_XDATA);
return EINVAL;
}
}
/* Advance. */
off += ((ip6e->ip6e_len + 1) << 3);
nxt = ip6e->ip6e_nxt;
break;
case IPPROTO_ROUTING:
/*
* Always include routing headers in
* computation.
*/
{
struct ip6_rthdr *rh;
ip6e = (struct ip6_ext *) (ptr + off);
rh = (struct ip6_rthdr *)(ptr + off);
/*
* must adjust content to make it look like
* its final form (as seen at the final
* destination).
* we only know how to massage type 0 routing
* header.
*/
if (out && rh->ip6r_type == IPV6_RTHDR_TYPE_0) {
struct ip6_rthdr0 *rh0;
struct in6_addr *addr, finaldst;
int i;
rh0 = (struct ip6_rthdr0 *)rh;
addr = (struct in6_addr *)(rh0 + 1);
for (i = 0; i < rh0->ip6r0_segleft; i++)
if (IN6_IS_SCOPE_EMBED(&addr[i]))
addr[i].s6_addr16[1] = 0;
finaldst = addr[rh0->ip6r0_segleft - 1];
memmove(&addr[1], &addr[0],
sizeof(struct in6_addr) *
(rh0->ip6r0_segleft - 1));
m_copydata(m, 0, sizeof(ip6),
(caddr_t)&ip6);
addr[0] = ip6.ip6_dst;
ip6.ip6_dst = finaldst;
m_copyback(m, 0, sizeof(ip6), &ip6,
M_NOWAIT);
rh0->ip6r0_segleft = 0;
}
/* advance */
off += ((ip6e->ip6e_len + 1) << 3);
nxt = ip6e->ip6e_nxt;
break;
}
default:
DPRINTF(("ah_massage_headers(): unexpected "
"IPv6 header type %d\n", off));
if (alloc)
free(ptr, M_XDATA);
ahstat.ahs_hdrops++;
m_freem(m);
return EINVAL;
}
}
/* Copyback and free, if we allocated. */
if (alloc) {
m_copyback(m, sizeof(struct ip6_hdr),
skip - sizeof(struct ip6_hdr), ptr, M_NOWAIT);
free(ptr, M_XDATA);
}
break;
#endif /* INET6 */
}
/*
* Copy data from m to n, where n cannot fit all the data we might
* want from m.
*
* Prioritize data like this:
* 1. TCP header
* 2. IP header
* 3. Data
*/
static void
tcp_pcap_copy_bestfit(struct tcphdr *th, struct mbuf *m, struct mbuf *n)
{
struct mbuf *m_cur = m;
int bytes_to_copy=0, trailing_data, skip=0, tcp_off;
/* Below, we assume these will be non-NULL. */
KASSERT(th, ("%s: called with th == NULL", __func__));
KASSERT(m, ("%s: called with m == NULL", __func__));
KASSERT(n, ("%s: called with n == NULL", __func__));
/* We assume this initialization occurred elsewhere. */
KASSERT(n->m_len == 0, ("%s: called with n->m_len=%d (expected 0)",
__func__, n->m_len));
KASSERT(n->m_data == M_START(n),
("%s: called with n->m_data != M_START(n)", __func__));
/*
* Calculate the size of the TCP header. We use this often
* enough that it is worth just calculating at the start.
*/
tcp_off = th->th_off << 2;
/* Trim off leading empty mbufs. */
while (m && m->m_len == 0)
m = m->m_next;
if (m) {
m_cur = m;
}
else {
/*
* No data? Highly unusual. We would expect to at
* least see a TCP header in the mbuf.
* As we have a pointer to the TCP header, I guess
* we should just copy that. (???)
*/
fallback:
bytes_to_copy = tcp_off;
if (bytes_to_copy > M_SIZE(n))
bytes_to_copy = M_SIZE(n);
bcopy(th, n->m_data, bytes_to_copy);
n->m_len = bytes_to_copy;
return;
}
/*
* Find TCP header. Record the total number of bytes up to,
* and including, the TCP header.
*/
while (m_cur) {
if ((caddr_t) th >= (caddr_t) m_cur->m_data &&
(caddr_t) th < (caddr_t) (m_cur->m_data + m_cur->m_len))
break;
bytes_to_copy += m_cur->m_len;
m_cur = m_cur->m_next;
}
if (m_cur)
bytes_to_copy += (caddr_t) th - (caddr_t) m_cur->m_data;
else
goto fallback;
bytes_to_copy += tcp_off;
/*
* If we already want to copy more bytes than we can hold
* in the destination mbuf, skip leading bytes and copy
* what we can.
*
* Otherwise, consider trailing data.
*/
if (bytes_to_copy > M_SIZE(n)) {
skip = bytes_to_copy - M_SIZE(n);
bytes_to_copy = M_SIZE(n);
}
else {
/*
* Determine how much trailing data is in the chain.
* We start with the length of this mbuf (the one
* containing th) and subtract the size of the TCP
* header (tcp_off) and the size of the data prior
* to th (th - m_cur->m_data).
*
* This *should not* be negative, as the TCP code
* should put the whole TCP header in a single
* mbuf. But, it isn't a problem if it is. We will
* simple work off our negative balance as we look
* at subsequent mbufs.
*/
trailing_data = m_cur->m_len - tcp_off;
trailing_data -= (caddr_t) th - (caddr_t) m_cur->m_data;
m_cur = m_cur->m_next;
while (m_cur) {
trailing_data += m_cur->m_len;
m_cur = m_cur->m_next;
}
if ((bytes_to_copy + trailing_data) > M_SIZE(n))
bytes_to_copy = M_SIZE(n);
else
bytes_to_copy += trailing_data;
}
m_copydata(m, skip, bytes_to_copy, n->m_data);
n->m_len = bytes_to_copy;
}
void
udp_input(struct mbuf *m, ...)
{
struct ip *ip;
struct udphdr *uh;
struct inpcb *inp = NULL;
struct mbuf *opts = NULL;
struct ip save_ip;
int iphlen, len;
va_list ap;
u_int16_t savesum;
union {
struct sockaddr sa;
struct sockaddr_in sin;
#ifdef INET6
struct sockaddr_in6 sin6;
#endif /* INET6 */
} srcsa, dstsa;
#ifdef INET6
struct ip6_hdr *ip6;
#endif /* INET6 */
#ifdef IPSEC
struct m_tag *mtag;
struct tdb_ident *tdbi;
struct tdb *tdb;
int error, s;
#endif /* IPSEC */
va_start(ap, m);
iphlen = va_arg(ap, int);
va_end(ap);
udpstat.udps_ipackets++;
switch (mtod(m, struct ip *)->ip_v) {
case 4:
ip = mtod(m, struct ip *);
#ifdef INET6
ip6 = NULL;
#endif /* INET6 */
srcsa.sa.sa_family = AF_INET;
break;
#ifdef INET6
case 6:
ip = NULL;
ip6 = mtod(m, struct ip6_hdr *);
srcsa.sa.sa_family = AF_INET6;
break;
#endif /* INET6 */
default:
goto bad;
}
IP6_EXTHDR_GET(uh, struct udphdr *, m, iphlen, sizeof(struct udphdr));
if (!uh) {
udpstat.udps_hdrops++;
return;
}
/* Check for illegal destination port 0 */
if (uh->uh_dport == 0) {
udpstat.udps_noport++;
goto bad;
}
/*
* Make mbuf data length reflect UDP length.
* If not enough data to reflect UDP length, drop.
*/
len = ntohs((u_int16_t)uh->uh_ulen);
if (ip) {
if (m->m_pkthdr.len - iphlen != len) {
if (len > (m->m_pkthdr.len - iphlen) ||
len < sizeof(struct udphdr)) {
udpstat.udps_badlen++;
goto bad;
}
m_adj(m, len - (m->m_pkthdr.len - iphlen));
}
}
#ifdef INET6
else if (ip6) {
/* jumbograms */
if (len == 0 && m->m_pkthdr.len - iphlen > 0xffff)
len = m->m_pkthdr.len - iphlen;
if (len != m->m_pkthdr.len - iphlen) {
udpstat.udps_badlen++;
goto bad;
}
}
#endif
else /* shouldn't happen */
goto bad;
/*
* Save a copy of the IP header in case we want restore it
* for sending an ICMP error message in response.
*/
if (ip)
save_ip = *ip;
/*
* Checksum extended UDP header and data.
* from W.R.Stevens: check incoming udp cksums even if
* udpcksum is not set.
*/
savesum = uh->uh_sum;
#ifdef INET6
if (ip6) {
/* Be proactive about malicious use of IPv4 mapped address */
if (IN6_IS_ADDR_V4MAPPED(&ip6->ip6_src) ||
IN6_IS_ADDR_V4MAPPED(&ip6->ip6_dst)) {
/* XXX stat */
goto bad;
}
/*
* In IPv6, the UDP checksum is ALWAYS used.
*/
if (uh->uh_sum == 0) {
udpstat.udps_nosum++;
goto bad;
}
if ((m->m_pkthdr.csum_flags & M_UDP_CSUM_IN_OK) == 0) {
if (m->m_pkthdr.csum_flags & M_UDP_CSUM_IN_BAD) {
udpstat.udps_badsum++;
udpstat.udps_inhwcsum++;
goto bad;
}
if ((uh->uh_sum = in6_cksum(m, IPPROTO_UDP,
iphlen, len))) {
udpstat.udps_badsum++;
goto bad;
}
} else {
m->m_pkthdr.csum_flags &= ~M_UDP_CSUM_IN_OK;
udpstat.udps_inhwcsum++;
}
} else
#endif /* INET6 */
if (uh->uh_sum) {
if ((m->m_pkthdr.csum_flags & M_UDP_CSUM_IN_OK) == 0) {
if (m->m_pkthdr.csum_flags & M_UDP_CSUM_IN_BAD) {
udpstat.udps_badsum++;
udpstat.udps_inhwcsum++;
m_freem(m);
return;
}
if ((uh->uh_sum = in4_cksum(m, IPPROTO_UDP,
iphlen, len))) {
udpstat.udps_badsum++;
m_freem(m);
return;
}
} else {
m->m_pkthdr.csum_flags &= ~M_UDP_CSUM_IN_OK;
udpstat.udps_inhwcsum++;
}
} else
udpstat.udps_nosum++;
#ifdef IPSEC
if (udpencap_enable && udpencap_port &&
uh->uh_dport == htons(udpencap_port)) {
u_int32_t spi;
int skip = iphlen + sizeof(struct udphdr);
if (m->m_pkthdr.len - skip < sizeof(u_int32_t)) {
/* packet too short */
m_freem(m);
return;
}
m_copydata(m, skip, sizeof(u_int32_t), (caddr_t) &spi);
/*
* decapsulate if the SPI is not zero, otherwise pass
* to userland
*/
if (spi != 0) {
if ((m = m_pullup2(m, skip)) == NULL) {
udpstat.udps_hdrops++;
return;
}
/* remove the UDP header */
bcopy(mtod(m, u_char *),
mtod(m, u_char *) + sizeof(struct udphdr), iphlen);
m_adj(m, sizeof(struct udphdr));
skip -= sizeof(struct udphdr);
espstat.esps_udpencin++;
ipsec_common_input(m, skip, offsetof(struct ip, ip_p),
srcsa.sa.sa_family, IPPROTO_ESP, 1);
return;
}
}
static void
if_netmap_send(void *arg)
{
struct mbuf *m;
struct if_netmap_softc *sc = (struct if_netmap_softc *)arg;
struct ifnet *ifp = sc->ifp;
struct uhi_pollfd pfd;
uint32_t avail;
uint32_t cur;
u_int pktlen;
int rv;
int done;
int pkts_sent;
if (sc->cfg->cpu >= 0)
sched_bind(sc->tx_thread.thr, sc->cfg->cpu);
rv = if_netmap_txsync(sc->nm_host_ctx, NULL, NULL);
if (rv == -1) {
printf("could not sync tx descriptors before transmit\n");
}
avail = if_netmap_txavail(sc->nm_host_ctx);
sc->tx_thread.last_stop_check = ticks;
done = 0;
pkts_sent = 0;
do {
mtx_lock(&sc->tx_lock);
sc->tx_pkts_to_send -= pkts_sent;
while ((sc->tx_pkts_to_send == 0) && !done)
if (EWOULDBLOCK == cv_timedwait(&sc->tx_cv, &sc->tx_lock, sc->stop_check_ticks))
done = if_netmap_stoppable_thread_check(&sc->tx_thread);
mtx_unlock(&sc->tx_lock);
if (done)
break;
pkts_sent = 0;
IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
while (m) {
while (0 == avail && !done) {
memset(&pfd, 0, sizeof(pfd));
pfd.fd = sc->fd;
pfd.events = UHI_POLLOUT;
rv = uhi_poll(&pfd, 1, IF_NETMAP_THREAD_STOP_CHECK_MS);
if (rv == 0)
done = if_netmap_stoppable_thread_check(&sc->tx_thread);
else if (rv == -1)
printf("error from poll for transmit\n");
avail = if_netmap_txavail(sc->nm_host_ctx);
}
if (ticks - sc->tx_thread.last_stop_check >= sc->stop_check_ticks)
done = if_netmap_stoppable_thread_check(&sc->tx_thread);
if (done)
break;
cur = if_netmap_txcur(sc->nm_host_ctx);
while (m && avail) {
ifp->if_ocopies++;
ifp->if_opackets++;
avail--;
pkts_sent++;
pktlen = m_length(m, NULL);
m_copydata(m, 0, pktlen,
if_netmap_txslot(sc->nm_host_ctx, &cur, pktlen));
m_freem(m);
IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
}
rv = if_netmap_txsync(sc->nm_host_ctx, &avail, &cur);
if (rv == -1) {
printf("could not sync tx descriptors after transmit\n");
}
avail = if_netmap_txavail(sc->nm_host_ctx);
}
} while (!done);
if_netmap_stoppable_thread_done(&sc->tx_thread);
}
/*
* ESP input callback from the crypto driver.
*/
static int
esp_input_cb(struct cryptop *crp)
{
char buf[128];
u_int8_t lastthree[3], aalg[AH_HMAC_MAXHASHLEN];
int hlen, skip, protoff, error, alen;
struct mbuf *m;
struct cryptodesc *crd;
struct auth_hash *esph;
struct enc_xform *espx;
struct tdb_crypto *tc;
struct secasvar *sav;
struct secasindex *saidx;
caddr_t ptr;
crd = crp->crp_desc;
IPSEC_ASSERT(crd != NULL, ("null crypto descriptor!"));
tc = (struct tdb_crypto *) crp->crp_opaque;
IPSEC_ASSERT(tc != NULL, ("null opaque crypto data area!"));
skip = tc->tc_skip;
protoff = tc->tc_protoff;
m = (struct mbuf *) crp->crp_buf;
sav = tc->tc_sav;
IPSEC_ASSERT(sav != NULL, ("null SA!"));
saidx = &sav->sah->saidx;
IPSEC_ASSERT(saidx->dst.sa.sa_family == AF_INET ||
saidx->dst.sa.sa_family == AF_INET6,
("unexpected protocol family %u", saidx->dst.sa.sa_family));
esph = sav->tdb_authalgxform;
espx = sav->tdb_encalgxform;
/* Check for crypto errors */
if (crp->crp_etype) {
/* Reset the session ID */
if (sav->tdb_cryptoid != 0)
sav->tdb_cryptoid = crp->crp_sid;
if (crp->crp_etype == EAGAIN)
return (crypto_dispatch(crp));
ESPSTAT_INC(esps_noxform);
DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
error = crp->crp_etype;
goto bad;
}
/* Shouldn't happen... */
if (m == NULL) {
ESPSTAT_INC(esps_crypto);
DPRINTF(("%s: bogus returned buffer from crypto\n", __func__));
error = EINVAL;
goto bad;
}
ESPSTAT_INC(esps_hist[sav->alg_enc]);
/* If authentication was performed, check now. */
if (esph != NULL) {
alen = xform_ah_authsize(esph);
AHSTAT_INC(ahs_hist[sav->alg_auth]);
/* Copy the authenticator from the packet */
m_copydata(m, m->m_pkthdr.len - alen, alen, aalg);
ptr = (caddr_t) (tc + 1);
/* Verify authenticator */
if (timingsafe_bcmp(ptr, aalg, alen) != 0) {
DPRINTF(("%s: authentication hash mismatch for "
"packet in SA %s/%08lx\n", __func__,
ipsec_address(&saidx->dst, buf, sizeof(buf)),
(u_long) ntohl(sav->spi)));
ESPSTAT_INC(esps_badauth);
error = EACCES;
goto bad;
}
/* Remove trailing authenticator */
m_adj(m, -alen);
}
/* Release the crypto descriptors */
free(tc, M_XDATA), tc = NULL;
crypto_freereq(crp), crp = NULL;
/*
* Packet is now decrypted.
*/
m->m_flags |= M_DECRYPTED;
/*
* Update replay sequence number, if appropriate.
*/
if (sav->replay) {
u_int32_t seq;
m_copydata(m, skip + offsetof(struct newesp, esp_seq),
sizeof (seq), (caddr_t) &seq);
if (ipsec_updatereplay(ntohl(seq), sav)) {
DPRINTF(("%s: packet replay check for %s\n", __func__,
ipsec_logsastr(sav, buf, sizeof(buf))));
ESPSTAT_INC(esps_replay);
error = ENOBUFS;
goto bad;
}
}
