/*
* ESP input processing, called (eventually) through the protocol switch.
*/
static int
esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
{
char buf[128];
struct auth_hash *esph;
struct enc_xform *espx;
struct tdb_crypto *tc;
uint8_t *ivp;
int plen, alen, hlen;
struct newesp *esp;
struct cryptodesc *crde;
struct cryptop *crp;
IPSEC_ASSERT(sav != NULL, ("null SA"));
IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
/* Valid IP Packet length ? */
if ( (skip&3) || (m->m_pkthdr.len&3) ){
DPRINTF(("%s: misaligned packet, skip %u pkt len %u",
__func__, skip, m->m_pkthdr.len));
ESPSTAT_INC(esps_badilen);
m_freem(m);
return EINVAL;
}
/* XXX don't pullup, just copy header */
IP6_EXTHDR_GET(esp, struct newesp *, m, skip, sizeof (struct newesp));
esph = sav->tdb_authalgxform;
espx = sav->tdb_encalgxform;
/* Determine the ESP header and auth length */
if (sav->flags & SADB_X_EXT_OLD)
hlen = sizeof (struct esp) + sav->ivlen;
else
hlen = sizeof (struct newesp) + sav->ivlen;
alen = xform_ah_authsize(esph);
/*
* Verify payload length is multiple of encryption algorithm
* block size.
*
* NB: This works for the null algorithm because the blocksize
* is 4 and all packets must be 4-byte aligned regardless
* of the algorithm.
*/
plen = m->m_pkthdr.len - (skip + hlen + alen);
if ((plen & (espx->blocksize - 1)) || (plen <= 0)) {
DPRINTF(("%s: payload of %d octets not a multiple of %d octets,"
" SA %s/%08lx\n", __func__, plen, espx->blocksize,
ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
(u_long)ntohl(sav->spi)));
ESPSTAT_INC(esps_badilen);
m_freem(m);
return EINVAL;
}
/*
* Check sequence number.
*/
if (esph != NULL && sav->replay != NULL &&
!ipsec_chkreplay(ntohl(esp->esp_seq), sav)) {
DPRINTF(("%s: packet replay check for %s\n", __func__,
ipsec_logsastr(sav, buf, sizeof(buf)))); /*XXX*/
ESPSTAT_INC(esps_replay);
m_freem(m);
return ENOBUFS; /*XXX*/
}
/* Update the counters */
ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen));
/* Get crypto descriptors */
crp = crypto_getreq(esph && espx ? 2 : 1);
if (crp == NULL) {
DPRINTF(("%s: failed to acquire crypto descriptors\n",
__func__));
ESPSTAT_INC(esps_crypto);
m_freem(m);
return ENOBUFS;
}
/* Get IPsec-specific opaque pointer */
tc = (struct tdb_crypto *) malloc(sizeof(struct tdb_crypto) + alen,
M_XDATA, M_NOWAIT | M_ZERO);
if (tc == NULL) {
crypto_freereq(crp);
DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
ESPSTAT_INC(esps_crypto);
m_freem(m);
return ENOBUFS;
}
if (esph != NULL) {
struct cryptodesc *crda = crp->crp_desc;
IPSEC_ASSERT(crda != NULL, ("null ah crypto descriptor"));
/* Authentication descriptor */
crda->crd_skip = skip;
if (SAV_ISGCM(sav))
crda->crd_len = 8; /* RFC4106 5, SPI + SN */
else
crda->crd_len = m->m_pkthdr.len - (skip + alen);
crda->crd_inject = m->m_pkthdr.len - alen;
crda->crd_alg = esph->type;
/* Copy the authenticator */
m_copydata(m, m->m_pkthdr.len - alen, alen,
(caddr_t) (tc + 1));
/* Chain authentication request */
crde = crda->crd_next;
} else {
crde = crp->crp_desc;
}
/* Crypto operation descriptor */
crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
crp->crp_buf = (caddr_t) m;
crp->crp_callback = esp_input_cb;
crp->crp_sid = sav->tdb_cryptoid;
crp->crp_opaque = (caddr_t) tc;
/* These are passed as-is to the callback */
tc->tc_spi = sav->spi;
tc->tc_dst = sav->sah->saidx.dst;
tc->tc_proto = sav->sah->saidx.proto;
tc->tc_protoff = protoff;
tc->tc_skip = skip;
KEY_ADDREFSA(sav);
tc->tc_sav = sav;
/* Decryption descriptor */
IPSEC_ASSERT(crde != NULL, ("null esp crypto descriptor"));
crde->crd_skip = skip + hlen;
crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
crde->crd_inject = skip + hlen - sav->ivlen;
if (SAV_ISCTRORGCM(sav)) {
ivp = &crde->crd_iv[0];
/* GCM IV Format: RFC4106 4 */
/* CTR IV Format: RFC3686 4 */
/* Salt is last four bytes of key, RFC4106 8.1 */
/* Nonce is last four bytes of key, RFC3686 5.1 */
memcpy(ivp, sav->key_enc->key_data +
_KEYLEN(sav->key_enc) - 4, 4);
if (SAV_ISCTR(sav)) {
/* Initial block counter is 1, RFC3686 4 */
be32enc(&ivp[sav->ivlen + 4], 1);
}
m_copydata(m, skip + hlen - sav->ivlen, sav->ivlen, &ivp[4]);
crde->crd_flags |= CRD_F_IV_EXPLICIT;
}
crde->crd_alg = espx->type;
return (crypto_dispatch(crp));
}
/*
* ESP input callback from the crypto driver.
*/
static int
esp_input_cb(struct cryptop *crp)
{
char buf[128];
u_int8_t lastthree[3], aalg[AH_HMAC_MAXHASHLEN];
int hlen, skip, protoff, error, alen;
struct mbuf *m;
struct cryptodesc *crd;
struct auth_hash *esph;
struct enc_xform *espx;
struct tdb_crypto *tc;
struct secasvar *sav;
struct secasindex *saidx;
caddr_t ptr;
crd = crp->crp_desc;
IPSEC_ASSERT(crd != NULL, ("null crypto descriptor!"));
tc = (struct tdb_crypto *) crp->crp_opaque;
IPSEC_ASSERT(tc != NULL, ("null opaque crypto data area!"));
skip = tc->tc_skip;
protoff = tc->tc_protoff;
m = (struct mbuf *) crp->crp_buf;
sav = tc->tc_sav;
IPSEC_ASSERT(sav != NULL, ("null SA!"));
saidx = &sav->sah->saidx;
IPSEC_ASSERT(saidx->dst.sa.sa_family == AF_INET ||
saidx->dst.sa.sa_family == AF_INET6,
("unexpected protocol family %u", saidx->dst.sa.sa_family));
esph = sav->tdb_authalgxform;
espx = sav->tdb_encalgxform;
/* Check for crypto errors */
if (crp->crp_etype) {
/* Reset the session ID */
if (sav->tdb_cryptoid != 0)
sav->tdb_cryptoid = crp->crp_sid;
if (crp->crp_etype == EAGAIN)
return (crypto_dispatch(crp));
ESPSTAT_INC(esps_noxform);
DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
error = crp->crp_etype;
goto bad;
}
/* Shouldn't happen... */
if (m == NULL) {
ESPSTAT_INC(esps_crypto);
DPRINTF(("%s: bogus returned buffer from crypto\n", __func__));
error = EINVAL;
goto bad;
}
ESPSTAT_INC(esps_hist[sav->alg_enc]);
/* If authentication was performed, check now. */
if (esph != NULL) {
alen = xform_ah_authsize(esph);
AHSTAT_INC(ahs_hist[sav->alg_auth]);
/* Copy the authenticator from the packet */
m_copydata(m, m->m_pkthdr.len - alen, alen, aalg);
ptr = (caddr_t) (tc + 1);
/* Verify authenticator */
if (timingsafe_bcmp(ptr, aalg, alen) != 0) {
DPRINTF(("%s: authentication hash mismatch for "
"packet in SA %s/%08lx\n", __func__,
ipsec_address(&saidx->dst, buf, sizeof(buf)),
(u_long) ntohl(sav->spi)));
ESPSTAT_INC(esps_badauth);
error = EACCES;
goto bad;
}
/* Remove trailing authenticator */
m_adj(m, -alen);
}
/* Release the crypto descriptors */
free(tc, M_XDATA), tc = NULL;
crypto_freereq(crp), crp = NULL;
/*
* Packet is now decrypted.
*/
m->m_flags |= M_DECRYPTED;
/*
* Update replay sequence number, if appropriate.
*/
if (sav->replay) {
u_int32_t seq;
m_copydata(m, skip + offsetof(struct newesp, esp_seq),
sizeof (seq), (caddr_t) &seq);
if (ipsec_updatereplay(ntohl(seq), sav)) {
DPRINTF(("%s: packet replay check for %s\n", __func__,
ipsec_logsastr(sav, buf, sizeof(buf))));
ESPSTAT_INC(esps_replay);
error = ENOBUFS;
goto bad;
}
}
/*
* ESP input processing, called (eventually) through the protocol switch.
*/
static int
esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
{
struct auth_hash *esph;
struct enc_xform *espx;
struct tdb_ident *tdbi;
struct tdb_crypto *tc;
int plen, alen, hlen;
struct m_tag *mtag;
struct newesp *esp;
struct cryptodesc *crde;
struct cryptop *crp;
IPSEC_ASSERT(sav != NULL, ("null SA"));
IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
/* Valid IP Packet length ? */
if ( (skip&3) || (m->m_pkthdr.len&3) ){
DPRINTF(("%s: misaligned packet, skip %u pkt len %u",
__func__, skip, m->m_pkthdr.len));
ESPSTAT_INC(esps_badilen);
m_freem(m);
return EINVAL;
}
/* XXX don't pullup, just copy header */
IP6_EXTHDR_GET(esp, struct newesp *, m, skip, sizeof (struct newesp));
esph = sav->tdb_authalgxform;
espx = sav->tdb_encalgxform;
/* Determine the ESP header length */
if (sav->flags & SADB_X_EXT_OLD)
hlen = sizeof (struct esp) + sav->ivlen;
else
hlen = sizeof (struct newesp) + sav->ivlen;
/* Authenticator hash size */
if (esph != NULL) {
switch (esph->type) {
case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:
case CRYPTO_SHA2_512_HMAC:
alen = esph->hashsize/2;
break;
default:
alen = AH_HMAC_HASHLEN;
break;
}
}else
alen = 0;
/*
* Verify payload length is multiple of encryption algorithm
* block size.
*
* NB: This works for the null algorithm because the blocksize
* is 4 and all packets must be 4-byte aligned regardless
* of the algorithm.
*/
plen = m->m_pkthdr.len - (skip + hlen + alen);
if ((plen & (espx->blocksize - 1)) || (plen <= 0)) {
DPRINTF(("%s: payload of %d octets not a multiple of %d octets,"
" SA %s/%08lx\n", __func__,
plen, espx->blocksize,
ipsec_address(&sav->sah->saidx.dst),
(u_long) ntohl(sav->spi)));
ESPSTAT_INC(esps_badilen);
m_freem(m);
return EINVAL;
}
/*
* Check sequence number.
*/
if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) {
DPRINTF(("%s: packet replay check for %s\n", __func__,
ipsec_logsastr(sav))); /*XXX*/
ESPSTAT_INC(esps_replay);
m_freem(m);
return ENOBUFS; /*XXX*/
}
/* Update the counters */
ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen));
/* Find out if we've already done crypto */
for (mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_CRYPTO_DONE, NULL);
mtag != NULL;
mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_CRYPTO_DONE, mtag)) {
tdbi = (struct tdb_ident *) (mtag + 1);
if (tdbi->proto == sav->sah->saidx.proto &&
tdbi->spi == sav->spi &&
!bcmp(&tdbi->dst, &sav->sah->saidx.dst,
sizeof(union sockaddr_union)))
break;
}
/* Get crypto descriptors */
crp = crypto_getreq(esph && espx ? 2 : 1);
if (crp == NULL) {
DPRINTF(("%s: failed to acquire crypto descriptors\n",
__func__));
ESPSTAT_INC(esps_crypto);
m_freem(m);
return ENOBUFS;
}
/* Get IPsec-specific opaque pointer */
if (esph == NULL || mtag != NULL)
tc = (struct tdb_crypto *) malloc(sizeof(struct tdb_crypto),
M_XDATA, M_NOWAIT|M_ZERO);
else
tc = (struct tdb_crypto *) malloc(sizeof(struct tdb_crypto) + alen,
M_XDATA, M_NOWAIT|M_ZERO);
if (tc == NULL) {
crypto_freereq(crp);
DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
ESPSTAT_INC(esps_crypto);
m_freem(m);
return ENOBUFS;
}
tc->tc_ptr = (caddr_t) mtag;
if (esph) {
struct cryptodesc *crda = crp->crp_desc;
IPSEC_ASSERT(crda != NULL, ("null ah crypto descriptor"));
/* Authentication descriptor */
crda->crd_skip = skip;
crda->crd_len = m->m_pkthdr.len - (skip + alen);
crda->crd_inject = m->m_pkthdr.len - alen;
crda->crd_alg = esph->type;
crda->crd_key = sav->key_auth->key_data;
crda->crd_klen = _KEYBITS(sav->key_auth);
/* Copy the authenticator */
if (mtag == NULL)
m_copydata(m, m->m_pkthdr.len - alen, alen,
(caddr_t) (tc + 1));
/* Chain authentication request */
crde = crda->crd_next;
} else {
crde = crp->crp_desc;
}
/* Crypto operation descriptor */
crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
crp->crp_buf = (caddr_t) m;
crp->crp_callback = esp_input_cb;
crp->crp_sid = sav->tdb_cryptoid;
crp->crp_opaque = (caddr_t) tc;
/* These are passed as-is to the callback */
tc->tc_spi = sav->spi;
tc->tc_dst = sav->sah->saidx.dst;
tc->tc_proto = sav->sah->saidx.proto;
tc->tc_protoff = protoff;
tc->tc_skip = skip;
KEY_ADDREFSA(sav);
tc->tc_sav = sav;
/* Decryption descriptor */
if (espx) {
IPSEC_ASSERT(crde != NULL, ("null esp crypto descriptor"));
crde->crd_skip = skip + hlen;
crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
crde->crd_inject = skip + hlen - sav->ivlen;
crde->crd_alg = espx->type;
crde->crd_key = sav->key_enc->key_data;
crde->crd_klen = _KEYBITS(sav->key_enc);
/* XXX Rounds ? */
}
if (mtag == NULL)
return crypto_dispatch(crp);
else
return esp_input_cb(crp);
}
int
esp4_input(struct mbuf **mp, int *offp, int proto)
{
int off;
struct ip *ip;
struct esp *esp;
struct esptail esptail;
struct mbuf *m;
u_int32_t spi;
struct secasvar *sav = NULL;
size_t taillen;
u_int16_t nxt;
const struct esp_algorithm *algo;
int ivlen;
size_t hlen;
size_t esplen;
off = *offp;
m = *mp;
*mp = NULL;
/* sanity check for alignment. */
if (off % 4 != 0 || m->m_pkthdr.len % 4 != 0) {
ipseclog((LOG_ERR, "IPv4 ESP input: packet alignment problem "
"(off=%d, pktlen=%d)\n", off, m->m_pkthdr.len));
ipsecstat.in_inval++;
goto bad;
}
if (m->m_len < off + ESPMAXLEN) {
m = m_pullup(m, off + ESPMAXLEN);
if (!m) {
ipseclog((LOG_DEBUG,
"IPv4 ESP input: can't pullup in esp4_input\n"));
ipsecstat.in_inval++;
goto bad;
}
}
ip = mtod(m, struct ip *);
esp = (struct esp *)(((u_int8_t *)ip) + off);
#ifdef _IP_VHL
hlen = IP_VHL_HL(ip->ip_vhl) << 2;
#else
hlen = ip->ip_hl << 2;
#endif
/* find the sassoc. */
spi = esp->esp_spi;
if ((sav = key_allocsa(AF_INET,
(caddr_t)&ip->ip_src, (caddr_t)&ip->ip_dst,
IPPROTO_ESP, spi)) == 0) {
ipseclog((LOG_WARNING,
"IPv4 ESP input: no key association found for spi %u\n",
(u_int32_t)ntohl(spi)));
ipsecstat.in_nosa++;
goto bad;
}
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
kprintf("DP esp4_input called to allocate SA:%p\n", sav));
if (sav->state != SADB_SASTATE_MATURE
&& sav->state != SADB_SASTATE_DYING) {
ipseclog((LOG_DEBUG,
"IPv4 ESP input: non-mature/dying SA found for spi %u\n",
(u_int32_t)ntohl(spi)));
ipsecstat.in_badspi++;
goto bad;
}
algo = esp_algorithm_lookup(sav->alg_enc);
if (!algo) {
ipseclog((LOG_DEBUG, "IPv4 ESP input: "
"unsupported encryption algorithm for spi %u\n",
(u_int32_t)ntohl(spi)));
ipsecstat.in_badspi++;
goto bad;
}
/* check if we have proper ivlen information */
ivlen = sav->ivlen;
if (ivlen < 0) {
ipseclog((LOG_ERR, "improper ivlen in IPv4 ESP input: %s %s\n",
ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
ipsecstat.in_inval++;
goto bad;
}
if (!((sav->flags & SADB_X_EXT_OLD) == 0 && sav->replay
&& (sav->alg_auth && sav->key_auth)))
goto noreplaycheck;
if (sav->alg_auth == SADB_X_AALG_NULL ||
sav->alg_auth == SADB_AALG_NONE)
goto noreplaycheck;
/*
* check for sequence number.
*/
if (ipsec_chkreplay(ntohl(((struct newesp *)esp)->esp_seq), sav))
; /* okey */
else {
ipsecstat.in_espreplay++;
ipseclog((LOG_WARNING,
"replay packet in IPv4 ESP input: %s %s\n",
ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
goto bad;
}
/* check ICV */
{
u_char sum0[AH_MAXSUMSIZE];
u_char sum[AH_MAXSUMSIZE];
const struct ah_algorithm *sumalgo;
size_t siz;
sumalgo = ah_algorithm_lookup(sav->alg_auth);
if (!sumalgo)
goto noreplaycheck;
siz = (((*sumalgo->sumsiz)(sav) + 3) & ~(4 - 1));
if (m->m_pkthdr.len < off + ESPMAXLEN + siz) {
ipsecstat.in_inval++;
goto bad;
}
if (AH_MAXSUMSIZE < siz) {
ipseclog((LOG_DEBUG,
"internal error: AH_MAXSUMSIZE must be larger than %lu\n",
(u_long)siz));
ipsecstat.in_inval++;
goto bad;
}
m_copydata(m, m->m_pkthdr.len - siz, siz, &sum0[0]);
if (esp_auth(m, off, m->m_pkthdr.len - off - siz, sav, sum)) {
ipseclog((LOG_WARNING, "auth fail in IPv4 ESP input: %s %s\n",
ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
ipsecstat.in_espauthfail++;
goto bad;
}
if (bcmp(sum0, sum, siz) != 0) {
ipseclog((LOG_WARNING, "auth fail in IPv4 ESP input: %s %s\n",
ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
ipsecstat.in_espauthfail++;
goto bad;
}
/* strip off the authentication data */
m_adj(m, -siz);
ip = mtod(m, struct ip *);
#ifdef IPLEN_FLIPPED
ip->ip_len = ip->ip_len - siz;
#else
ip->ip_len = htons(ntohs(ip->ip_len) - siz);
#endif
m->m_flags |= M_AUTHIPDGM;
ipsecstat.in_espauthsucc++;
}
/*
* update sequence number.
*/
if ((sav->flags & SADB_X_EXT_OLD) == 0 && sav->replay) {
if (ipsec_updatereplay(ntohl(((struct newesp *)esp)->esp_seq), sav)) {
ipsecstat.in_espreplay++;
goto bad;
}
}
noreplaycheck:
/* process main esp header. */
if (sav->flags & SADB_X_EXT_OLD) {
/* RFC 1827 */
esplen = sizeof(struct esp);
} else {
/* RFC 2406 */
if (sav->flags & SADB_X_EXT_DERIV)
esplen = sizeof(struct esp);
else
esplen = sizeof(struct newesp);
}
if (m->m_pkthdr.len < off + esplen + ivlen + sizeof(esptail)) {
ipseclog((LOG_WARNING,
"IPv4 ESP input: packet too short\n"));
ipsecstat.in_inval++;
goto bad;
}
if (m->m_len < off + esplen + ivlen) {
m = m_pullup(m, off + esplen + ivlen);
if (!m) {
ipseclog((LOG_DEBUG,
"IPv4 ESP input: can't pullup in esp4_input\n"));
ipsecstat.in_inval++;
goto bad;
}
}
/*
* pre-compute and cache intermediate key
*/
if (esp_schedule(algo, sav) != 0) {
ipsecstat.in_inval++;
goto bad;
}
/*
* decrypt the packet.
*/
if (!algo->decrypt)
panic("internal error: no decrypt function");
if ((*algo->decrypt)(m, off, sav, algo, ivlen)) {
/* m is already freed */
m = NULL;
ipseclog((LOG_ERR, "decrypt fail in IPv4 ESP input: %s\n",
ipsec_logsastr(sav)));
ipsecstat.in_inval++;
goto bad;
}
ipsecstat.in_esphist[sav->alg_enc]++;
m->m_flags |= M_DECRYPTED;
/*
* find the trailer of the ESP.
*/
m_copydata(m, m->m_pkthdr.len - sizeof(esptail), sizeof(esptail),
(caddr_t)&esptail);
nxt = esptail.esp_nxt;
taillen = esptail.esp_padlen + sizeof(esptail);
if (m->m_pkthdr.len < taillen ||
m->m_pkthdr.len - taillen < off + esplen + ivlen + sizeof(esptail)) {
ipseclog((LOG_WARNING,
"bad pad length in IPv4 ESP input: %s %s\n",
ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
ipsecstat.in_inval++;
goto bad;
}
/* strip off the trailing pad area. */
m_adj(m, -taillen);
#ifdef IPLEN_FLIPPED
ip->ip_len = ip->ip_len - taillen;
#else
ip->ip_len = htons(ntohs(ip->ip_len) - taillen);
#endif
/* was it transmitted over the IPsec tunnel SA? */
if (ipsec4_tunnel_validate(m, off + esplen + ivlen, nxt, sav)) {
/*
* strip off all the headers that precedes ESP header.
* IP4 xx ESP IP4' payload -> IP4' payload
*
* XXX more sanity checks
* XXX relationship with gif?
*/
u_int8_t tos;
tos = ip->ip_tos;
m_adj(m, off + esplen + ivlen);
if (m->m_len < sizeof(*ip)) {
m = m_pullup(m, sizeof(*ip));
if (!m) {
ipsecstat.in_inval++;
goto bad;
}
}
ip = mtod(m, struct ip *);
/* ECN consideration. */
ip_ecn_egress(ip4_ipsec_ecn, &tos, &ip->ip_tos);
if (!key_checktunnelsanity(sav, AF_INET,
(caddr_t)&ip->ip_src, (caddr_t)&ip->ip_dst)) {
ipseclog((LOG_ERR, "ipsec tunnel address mismatch "
"in IPv4 ESP input: %s %s\n",
ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
ipsecstat.in_inval++;
goto bad;
}
key_sa_recordxfer(sav, m);
if (ipsec_addhist(m, IPPROTO_ESP, spi) != 0 ||
ipsec_addhist(m, IPPROTO_IPV4, 0) != 0) {
ipsecstat.in_nomem++;
goto bad;
}
if (netisr_queue(NETISR_IP, m)) {
ipsecstat.in_inval++;
m = NULL;
goto bad;
}
nxt = IPPROTO_DONE;
} else {
/*
* ESP input callback from the crypto driver.
*/
static int
esp_input_cb(struct cryptop *crp)
{
u_int8_t lastthree[3], aalg[AH_HMAC_MAXHASHLEN];
int hlen, skip, protoff, error, alen;
struct mbuf *m;
struct cryptodesc *crd;
struct auth_hash *esph;
struct enc_xform *espx;
struct tdb_crypto *tc;
struct m_tag *mtag;
struct secasvar *sav;
struct secasindex *saidx;
caddr_t ptr;
crd = crp->crp_desc;
IPSEC_ASSERT(crd != NULL, ("null crypto descriptor!"));
tc = (struct tdb_crypto *) crp->crp_opaque;
IPSEC_ASSERT(tc != NULL, ("null opaque crypto data area!"));
skip = tc->tc_skip;
protoff = tc->tc_protoff;
mtag = (struct m_tag *) tc->tc_ptr;
m = (struct mbuf *) crp->crp_buf;
sav = tc->tc_sav;
IPSEC_ASSERT(sav != NULL, ("null SA!"));
saidx = &sav->sah->saidx;
IPSEC_ASSERT(saidx->dst.sa.sa_family == AF_INET ||
saidx->dst.sa.sa_family == AF_INET6,
("unexpected protocol family %u", saidx->dst.sa.sa_family));
esph = sav->tdb_authalgxform;
espx = sav->tdb_encalgxform;
/* Check for crypto errors */
if (crp->crp_etype) {
/* Reset the session ID */
if (sav->tdb_cryptoid != 0)
sav->tdb_cryptoid = crp->crp_sid;
if (crp->crp_etype == EAGAIN)
return (crypto_dispatch(crp));
V_espstat.esps_noxform++;
DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
error = crp->crp_etype;
goto bad;
}
/* Shouldn't happen... */
if (m == NULL) {
V_espstat.esps_crypto++;
DPRINTF(("%s: bogus returned buffer from crypto\n", __func__));
error = EINVAL;
goto bad;
}
V_espstat.esps_hist[sav->alg_enc]++;
/* If authentication was performed, check now. */
if (esph != NULL) {
switch (esph->type) {
case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:
case CRYPTO_SHA2_512_HMAC:
alen = esph->hashsize/2;
break;
default:
alen = AH_HMAC_HASHLEN;
break;
}
/*
* If we have a tag, it means an IPsec-aware NIC did
* the verification for us. Otherwise we need to
* check the authentication calculation.
*/
V_ahstat.ahs_hist[sav->alg_auth]++;
if (mtag == NULL) {
/* Copy the authenticator from the packet */
m_copydata(m, m->m_pkthdr.len - alen,
alen, aalg);
ptr = (caddr_t) (tc + 1);
/* Verify authenticator */
if (bcmp(ptr, aalg, alen) != 0) {
DPRINTF(("%s: "
"authentication hash mismatch for packet in SA %s/%08lx\n",
__func__,
ipsec_address(&saidx->dst),
(u_long) ntohl(sav->spi)));
V_espstat.esps_badauth++;
error = EACCES;
goto bad;
}
}
/* Remove trailing authenticator */
m_adj(m, -alen);
}
/* Release the crypto descriptors */
free(tc, M_XDATA), tc = NULL;
crypto_freereq(crp), crp = NULL;
/*
* Packet is now decrypted.
*/
m->m_flags |= M_DECRYPTED;
/*
* Update replay sequence number, if appropriate.
*/
if (sav->replay) {
u_int32_t seq;
m_copydata(m, skip + offsetof(struct newesp, esp_seq),
sizeof (seq), (caddr_t) &seq);
if (ipsec_updatereplay(ntohl(seq), sav)) {
DPRINTF(("%s: packet replay check for %s\n", __func__,
ipsec_logsastr(sav)));
V_espstat.esps_replay++;
error = ENOBUFS;
goto bad;
}
}
/*
* ESP input callback from the crypto driver.
*/
static int
esp_input_cb(struct cryptop *crp)
{
u_int8_t lastthree[3], aalg[AH_ALEN_MAX];
int s, hlen, skip, protoff, error;
struct mbuf *m;
struct cryptodesc *crd;
const struct auth_hash *esph;
const struct enc_xform *espx;
struct tdb_crypto *tc;
struct m_tag *mtag;
struct secasvar *sav;
struct secasindex *saidx;
void *ptr;
u_int16_t dport;
u_int16_t sport;
crd = crp->crp_desc;
IPSEC_ASSERT(crd != NULL, ("esp_input_cb: null crypto descriptor!"));
tc = (struct tdb_crypto *) crp->crp_opaque;
IPSEC_ASSERT(tc != NULL, ("esp_input_cb: null opaque crypto data area!"));
skip = tc->tc_skip;
protoff = tc->tc_protoff;
mtag = (struct m_tag *) tc->tc_ptr;
m = (struct mbuf *) crp->crp_buf;
/* find the source port for NAT-T */
nat_t_ports_get(m, &dport, &sport);
s = splsoftnet();
mutex_enter(softnet_lock);
sav = KEY_ALLOCSA(&tc->tc_dst, tc->tc_proto, tc->tc_spi, sport, dport);
if (sav == NULL) {
ESP_STATINC(ESP_STAT_NOTDB);
DPRINTF(("esp_input_cb: SA expired while in crypto "
"(SA %s/%08lx proto %u)\n", ipsec_address(&tc->tc_dst),
(u_long) ntohl(tc->tc_spi), tc->tc_proto));
error = ENOBUFS; /*XXX*/
goto bad;
}
saidx = &sav->sah->saidx;
IPSEC_ASSERT(saidx->dst.sa.sa_family == AF_INET ||
saidx->dst.sa.sa_family == AF_INET6,
("esp_input_cb: unexpected protocol family %u",
saidx->dst.sa.sa_family));
esph = sav->tdb_authalgxform;
espx = sav->tdb_encalgxform;
/* Check for crypto errors */
if (crp->crp_etype) {
/* Reset the session ID */
if (sav->tdb_cryptoid != 0)
sav->tdb_cryptoid = crp->crp_sid;
if (crp->crp_etype == EAGAIN) {
KEY_FREESAV(&sav);
mutex_exit(softnet_lock);
splx(s);
return crypto_dispatch(crp);
}
ESP_STATINC(ESP_STAT_NOXFORM);
DPRINTF(("esp_input_cb: crypto error %d\n", crp->crp_etype));
error = crp->crp_etype;
goto bad;
}
/* Shouldn't happen... */
if (m == NULL) {
ESP_STATINC(ESP_STAT_CRYPTO);
DPRINTF(("esp_input_cb: bogus returned buffer from crypto\n"));
error = EINVAL;
goto bad;
}
ESP_STATINC(ESP_STAT_HIST + sav->alg_enc);
/* If authentication was performed, check now. */
if (esph != NULL) {
/*
* If we have a tag, it means an IPsec-aware NIC did
* the verification for us. Otherwise we need to
* check the authentication calculation.
*/
AH_STATINC(AH_STAT_HIST + sav->alg_auth);
if (mtag == NULL) {
/* Copy the authenticator from the packet */
m_copydata(m, m->m_pkthdr.len - esph->authsize,
esph->authsize, aalg);
ptr = (tc + 1);
/* Verify authenticator */
if (!consttime_memequal(ptr, aalg, esph->authsize)) {
DPRINTF(("esp_input_cb: "
"authentication hash mismatch for packet in SA %s/%08lx\n",
ipsec_address(&saidx->dst),
(u_long) ntohl(sav->spi)));
ESP_STATINC(ESP_STAT_BADAUTH);
error = EACCES;
goto bad;
}
}
/* Remove trailing authenticator */
m_adj(m, -(esph->authsize));
}
/* Release the crypto descriptors */
free(tc, M_XDATA), tc = NULL;
crypto_freereq(crp), crp = NULL;
/*
* Packet is now decrypted.
*/
m->m_flags |= M_DECRYPTED;
/*
* Update replay sequence number, if appropriate.
*/
if (sav->replay) {
u_int32_t seq;
m_copydata(m, skip + offsetof(struct newesp, esp_seq),
sizeof (seq), &seq);
if (ipsec_updatereplay(ntohl(seq), sav)) {
DPRINTF(("%s: packet replay check for %s\n", __func__,
ipsec_logsastr(sav)));
ESP_STATINC(ESP_STAT_REPLAY);
error = ENOBUFS;
goto bad;
}
}
/*
* ESP input processing, called (eventually) through the protocol switch.
*/
static int
esp_input(struct mbuf *m, const struct secasvar *sav, int skip, int protoff)
{
const struct auth_hash *esph;
const struct enc_xform *espx;
struct tdb_ident *tdbi;
struct tdb_crypto *tc;
int plen, alen, hlen, error;
struct m_tag *mtag;
struct newesp *esp;
struct cryptodesc *crde;
struct cryptop *crp;
IPSEC_SPLASSERT_SOFTNET("esp_input");
IPSEC_ASSERT(sav != NULL, ("esp_input: null SA"));
IPSEC_ASSERT(sav->tdb_encalgxform != NULL,
("esp_input: null encoding xform"));
IPSEC_ASSERT((skip&3) == 0 && (m->m_pkthdr.len&3) == 0,
("esp_input: misaligned packet, skip %u pkt len %u",
skip, m->m_pkthdr.len));
/* XXX don't pullup, just copy header */
IP6_EXTHDR_GET(esp, struct newesp *, m, skip, sizeof (struct newesp));
esph = sav->tdb_authalgxform;
espx = sav->tdb_encalgxform;
/* Determine the ESP header length */
if (sav->flags & SADB_X_EXT_OLD)
hlen = sizeof (struct esp) + sav->ivlen;
else
hlen = sizeof (struct newesp) + sav->ivlen;
/* Authenticator hash size */
alen = esph ? esph->authsize : 0;
/*
* Verify payload length is multiple of encryption algorithm
* block size.
*
* NB: This works for the null algorithm because the blocksize
* is 4 and all packets must be 4-byte aligned regardless
* of the algorithm.
*/
plen = m->m_pkthdr.len - (skip + hlen + alen);
if ((plen & (espx->blocksize - 1)) || (plen <= 0)) {
DPRINTF(("esp_input: "
"payload of %d octets not a multiple of %d octets,"
" SA %s/%08lx\n",
plen, espx->blocksize,
ipsec_address(&sav->sah->saidx.dst),
(u_long) ntohl(sav->spi)));
ESP_STATINC(ESP_STAT_BADILEN);
m_freem(m);
return EINVAL;
}
/*
* Check sequence number.
*/
if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) {
DPRINTF(("esp_input: packet replay check for %s\n",
ipsec_logsastr(sav))); /*XXX*/
ESP_STATINC(ESP_STAT_REPLAY);
m_freem(m);
return ENOBUFS; /*XXX*/
}
/* Update the counters */
ESP_STATADD(ESP_STAT_IBYTES, m->m_pkthdr.len - skip - hlen - alen);
/* Find out if we've already done crypto */
for (mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_CRYPTO_DONE, NULL);
mtag != NULL;
mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_CRYPTO_DONE, mtag)) {
tdbi = (struct tdb_ident *) (mtag + 1);
if (tdbi->proto == sav->sah->saidx.proto &&
tdbi->spi == sav->spi &&
!memcmp(&tdbi->dst, &sav->sah->saidx.dst,
sizeof(union sockaddr_union)))
break;
}
/* Get crypto descriptors */
crp = crypto_getreq(esph && espx ? 2 : 1);
if (crp == NULL) {
DPRINTF(("esp_input: failed to acquire crypto descriptors\n"));
ESP_STATINC(ESP_STAT_CRYPTO);
m_freem(m);
return ENOBUFS;
}
/* Get IPsec-specific opaque pointer */
if (esph == NULL || mtag != NULL)
tc = (struct tdb_crypto *) malloc(sizeof(struct tdb_crypto),
M_XDATA, M_NOWAIT|M_ZERO);
else
tc = (struct tdb_crypto *) malloc(sizeof(struct tdb_crypto) + alen,
M_XDATA, M_NOWAIT|M_ZERO);
if (tc == NULL) {
crypto_freereq(crp);
DPRINTF(("esp_input: failed to allocate tdb_crypto\n"));
ESP_STATINC(ESP_STAT_CRYPTO);
m_freem(m);
return ENOBUFS;
}
error = m_makewritable(&m, 0, m->m_pkthdr.len, M_NOWAIT);
if (error) {
m_freem(m);
free(tc, M_XDATA);
crypto_freereq(crp);
DPRINTF(("esp_input: m_makewritable failed\n"));
ESP_STATINC(ESP_STAT_CRYPTO);
return error;
}
tc->tc_ptr = mtag;
if (esph) {
struct cryptodesc *crda = crp->crp_desc;
IPSEC_ASSERT(crda != NULL, ("esp_input: null ah crypto descriptor"));
/* Authentication descriptor */
crda->crd_skip = skip;
if (espx && espx->type == CRYPTO_AES_GCM_16)
crda->crd_len = hlen - sav->ivlen;
else
crda->crd_len = m->m_pkthdr.len - (skip + alen);
crda->crd_inject = m->m_pkthdr.len - alen;
crda->crd_alg = esph->type;
if (espx && (espx->type == CRYPTO_AES_GCM_16 ||
espx->type == CRYPTO_AES_GMAC)) {
crda->crd_key = _KEYBUF(sav->key_enc);
crda->crd_klen = _KEYBITS(sav->key_enc);
} else {
crda->crd_key = _KEYBUF(sav->key_auth);
crda->crd_klen = _KEYBITS(sav->key_auth);
}
/* Copy the authenticator */
if (mtag == NULL)
m_copydata(m, m->m_pkthdr.len - alen, alen,
(tc + 1));
/* Chain authentication request */
crde = crda->crd_next;
} else {
crde = crp->crp_desc;
}
/* Crypto operation descriptor */
crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
crp->crp_flags = CRYPTO_F_IMBUF;
crp->crp_buf = m;
crp->crp_callback = esp_input_cb;
crp->crp_sid = sav->tdb_cryptoid;
crp->crp_opaque = tc;
/* These are passed as-is to the callback */
tc->tc_spi = sav->spi;
tc->tc_dst = sav->sah->saidx.dst;
tc->tc_proto = sav->sah->saidx.proto;
tc->tc_protoff = protoff;
tc->tc_skip = skip;
/* Decryption descriptor */
if (espx) {
IPSEC_ASSERT(crde != NULL, ("esp_input: null esp crypto descriptor"));
crde->crd_skip = skip + hlen;
if (espx->type == CRYPTO_AES_GMAC)
crde->crd_len = 0;
else
crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
crde->crd_inject = skip + hlen - sav->ivlen;
crde->crd_alg = espx->type;
crde->crd_key = _KEYBUF(sav->key_enc);
crde->crd_klen = _KEYBITS(sav->key_enc);
/* XXX Rounds ? */
}
if (mtag == NULL)
return crypto_dispatch(crp);
else
return esp_input_cb(crp);
}
