void
crypto_copyback(int flags, caddr_t buf, int off, int size, caddr_t in)
{
if ((flags & CRYPTO_F_IMBUF) != 0)
m_copyback((struct mbuf *)buf, off, size, in);
else if ((flags & CRYPTO_F_IOV) != 0)
cuio_copyback((struct uio *)buf, off, size, in);
else
bcopy(in, buf + off, size);
}
int
via_padlock_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd,
struct via_padlock_session *ses, struct via_padlock_softc *sc, void *buf)
{
uint32_t *key;
int err = 0;
if ((crd->crd_len % 16) != 0) {
err = EINVAL;
return (err);
}
sc->op_buf = malloc(crd->crd_len, M_DEVBUF, M_NOWAIT);
if (sc->op_buf == NULL) {
err = ENOMEM;
return (err);
}
if (crd->crd_flags & CRD_F_ENCRYPT) {
sc->op_cw[0] = ses->ses_cw0 | C3_CRYPT_CWLO_ENCRYPT;
key = ses->ses_ekey;
if (crd->crd_flags & CRD_F_IV_EXPLICIT)
memcpy(sc->op_iv, crd->crd_iv, 16);
else
memcpy(sc->op_iv, ses->ses_iv, 16);
if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
if (crp->crp_flags & CRYPTO_F_IMBUF)
m_copyback((struct mbuf *)crp->crp_buf,
crd->crd_inject, 16, sc->op_iv);
else if (crp->crp_flags & CRYPTO_F_IOV)
cuio_copyback((struct uio *)crp->crp_buf,
crd->crd_inject, 16, sc->op_iv);
else
memcpy((char *)crp->crp_buf + crd->crd_inject,
sc->op_iv, 16);
}
} else {
sc->op_cw[0] = ses->ses_cw0 | C3_CRYPT_CWLO_DECRYPT;
key = ses->ses_dkey;
if (crd->crd_flags & CRD_F_IV_EXPLICIT)
memcpy(sc->op_iv, crd->crd_iv, 16);
else {
if (crp->crp_flags & CRYPTO_F_IMBUF)
m_copydata((struct mbuf *)crp->crp_buf,
crd->crd_inject, 16, sc->op_iv);
else if (crp->crp_flags & CRYPTO_F_IOV)
cuio_copydata((struct uio *)crp->crp_buf,
crd->crd_inject, 16, sc->op_iv);
else
memcpy(sc->op_iv, (char *)crp->crp_buf +
crd->crd_inject, 16);
}
}
if (crp->crp_flags & CRYPTO_F_IMBUF)
m_copydata((struct mbuf *)crp->crp_buf,
crd->crd_skip, crd->crd_len, sc->op_buf);
else if (crp->crp_flags & CRYPTO_F_IOV)
cuio_copydata((struct uio *)crp->crp_buf,
crd->crd_skip, crd->crd_len, sc->op_buf);
else
memcpy(sc->op_buf, (char *)crp->crp_buf + crd->crd_skip,
crd->crd_len);
sc->op_cw[1] = sc->op_cw[2] = sc->op_cw[3] = 0;
via_padlock_cbc(&sc->op_cw, sc->op_buf, sc->op_buf, key,
crd->crd_len / 16, sc->op_iv);
if (crp->crp_flags & CRYPTO_F_IMBUF)
m_copyback((struct mbuf *)crp->crp_buf,
crd->crd_skip, crd->crd_len, sc->op_buf);
else if (crp->crp_flags & CRYPTO_F_IOV)
cuio_copyback((struct uio *)crp->crp_buf,
crd->crd_skip, crd->crd_len, sc->op_buf);
else
memcpy((char *)crp->crp_buf + crd->crd_skip, sc->op_buf,
crd->crd_len);
/* copy out last block for use as next session IV */
if (crd->crd_flags & CRD_F_ENCRYPT) {
if (crp->crp_flags & CRYPTO_F_IMBUF)
m_copydata((struct mbuf *)crp->crp_buf,
crd->crd_skip + crd->crd_len - 16, 16,
ses->ses_iv);
else if (crp->crp_flags & CRYPTO_F_IOV)
cuio_copydata((struct uio *)crp->crp_buf,
crd->crd_skip + crd->crd_len - 16, 16,
ses->ses_iv);
else
memcpy(ses->ses_iv, (char *)crp->crp_buf +
crd->crd_skip + crd->crd_len - 16, 16);
}
if (sc->op_buf != NULL) {
memset(sc->op_buf, 0, crd->crd_len);
free(sc->op_buf, M_DEVBUF);
sc->op_buf = NULL;
}
return (err);
}
int
ubsec_process(struct cryptop *crp)
{
struct ubsec_q *q = NULL;
int card, err = 0, i, j, s, nicealign;
struct ubsec_softc *sc;
struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
int encoffset = 0, macoffset = 0, cpskip, cpoffset;
int sskip, dskip, stheend, dtheend;
int16_t coffset;
struct ubsec_session *ses, key;
struct ubsec_dma *dmap = NULL;
u_int16_t flags = 0;
int ivlen = 0, keylen = 0;
if (crp == NULL || crp->crp_callback == NULL) {
ubsecstats.hst_invalid++;
return (EINVAL);
}
card = UBSEC_CARD(crp->crp_sid);
if (card >= ubsec_cd.cd_ndevs || ubsec_cd.cd_devs[card] == NULL) {
ubsecstats.hst_invalid++;
return (EINVAL);
}
sc = ubsec_cd.cd_devs[card];
s = splnet();
if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
ubsecstats.hst_queuefull++;
splx(s);
err = ENOMEM;
goto errout2;
}
q = SIMPLEQ_FIRST(&sc->sc_freequeue);
SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
splx(s);
dmap = q->q_dma; /* Save dma pointer */
bzero(q, sizeof(struct ubsec_q));
bzero(&key, sizeof(key));
q->q_sesn = UBSEC_SESSION(crp->crp_sid);
q->q_dma = dmap;
ses = &sc->sc_sessions[q->q_sesn];
if (crp->crp_flags & CRYPTO_F_IMBUF) {
q->q_src_m = (struct mbuf *)crp->crp_buf;
q->q_dst_m = (struct mbuf *)crp->crp_buf;
} else if (crp->crp_flags & CRYPTO_F_IOV) {
q->q_src_io = (struct uio *)crp->crp_buf;
q->q_dst_io = (struct uio *)crp->crp_buf;
} else {
err = EINVAL;
goto errout; /* XXX we don't handle contiguous blocks! */
}
bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
dmap->d_dma->d_mcr.mcr_flags = 0;
q->q_crp = crp;
crd1 = crp->crp_desc;
if (crd1 == NULL) {
err = EINVAL;
goto errout;
}
crd2 = crd1->crd_next;
if (crd2 == NULL) {
if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC) {
maccrd = crd1;
enccrd = NULL;
} else if (crd1->crd_alg == CRYPTO_3DES_CBC ||
crd1->crd_alg == CRYPTO_AES_CBC) {
maccrd = NULL;
enccrd = crd1;
} else {
err = EINVAL;
goto errout;
}
} else {
if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
(crd2->crd_alg == CRYPTO_3DES_CBC ||
crd2->crd_alg == CRYPTO_AES_CBC) &&
((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
maccrd = crd1;
enccrd = crd2;
} else if ((crd1->crd_alg == CRYPTO_3DES_CBC ||
crd1->crd_alg == CRYPTO_AES_CBC) &&
(crd2->crd_alg == CRYPTO_MD5_HMAC ||
crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
(crd1->crd_flags & CRD_F_ENCRYPT)) {
enccrd = crd1;
maccrd = crd2;
} else {
/*
* We cannot order the ubsec as requested
*/
err = EINVAL;
goto errout;
}
}
if (enccrd) {
if (enccrd->crd_alg == CRYPTO_AES_CBC) {
if ((sc->sc_flags & UBS_FLAGS_AES) == 0) {
err = EINVAL;
goto errout;
}
flags |= htole16(UBS_PKTCTX_ENC_AES);
switch (enccrd->crd_klen) {
case 128:
case 192:
case 256:
keylen = enccrd->crd_klen / 8;
break;
default:
err = EINVAL;
goto errout;
}
ivlen = 16;
} else {
flags |= htole16(UBS_PKTCTX_ENC_3DES);
ivlen = 8;
keylen = 24;
}
encoffset = enccrd->crd_skip;
if (enccrd->crd_flags & CRD_F_ENCRYPT) {
if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(enccrd->crd_iv, key.ses_iv, ivlen);
else
arc4random_buf(key.ses_iv, ivlen);
if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
if (crp->crp_flags & CRYPTO_F_IMBUF)
err = m_copyback(q->q_src_m,
enccrd->crd_inject,
ivlen, key.ses_iv, M_NOWAIT);
else if (crp->crp_flags & CRYPTO_F_IOV)
cuio_copyback(q->q_src_io,
enccrd->crd_inject,
ivlen, key.ses_iv);
if (err)
goto errout;
}
} else {
flags |= htole16(UBS_PKTCTX_INBOUND);
if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(enccrd->crd_iv, key.ses_iv, ivlen);
else if (crp->crp_flags & CRYPTO_F_IMBUF)
m_copydata(q->q_src_m, enccrd->crd_inject,
ivlen, (caddr_t)key.ses_iv);
else if (crp->crp_flags & CRYPTO_F_IOV)
cuio_copydata(q->q_src_io,
enccrd->crd_inject, ivlen,
(caddr_t)key.ses_iv);
}
for (i = 0; i < (keylen / 4); i++)
key.ses_key[i] = ses->ses_key[i];
for (i = 0; i < (ivlen / 4); i++)
SWAP32(key.ses_iv[i]);
}
if (maccrd) {
macoffset = maccrd->crd_skip;
if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
flags |= htole16(UBS_PKTCTX_AUTH_MD5);
else
flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
for (i = 0; i < 5; i++) {
key.ses_hminner[i] = ses->ses_hminner[i];
key.ses_hmouter[i] = ses->ses_hmouter[i];
HTOLE32(key.ses_hminner[i]);
HTOLE32(key.ses_hmouter[i]);
}
}
if (enccrd && maccrd) {
/*
* ubsec cannot handle packets where the end of encryption
* and authentication are not the same, or where the
* encrypted part begins before the authenticated part.
*/
if (((encoffset + enccrd->crd_len) !=
(macoffset + maccrd->crd_len)) ||
(enccrd->crd_skip < maccrd->crd_skip)) {
err = EINVAL;
goto errout;
}
sskip = maccrd->crd_skip;
cpskip = dskip = enccrd->crd_skip;
stheend = maccrd->crd_len;
dtheend = enccrd->crd_len;
coffset = enccrd->crd_skip - maccrd->crd_skip;
cpoffset = cpskip + dtheend;
#ifdef UBSEC_DEBUG
printf("mac: skip %d, len %d, inject %d\n",
maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
printf("enc: skip %d, len %d, inject %d\n",
enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
printf("src: skip %d, len %d\n", sskip, stheend);
printf("dst: skip %d, len %d\n", dskip, dtheend);
printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
coffset, stheend, cpskip, cpoffset);
#endif
} else {
cpskip = dskip = sskip = macoffset + encoffset;
dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
cpoffset = cpskip + dtheend;
coffset = 0;
}
if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER,
0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) {
err = ENOMEM;
goto errout;
}
if (crp->crp_flags & CRYPTO_F_IMBUF) {
if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
q->q_src_m, BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
q->q_src_map = NULL;
err = ENOMEM;
goto errout;
}
} else if (crp->crp_flags & CRYPTO_F_IOV) {
if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
q->q_src_io, BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
q->q_src_map = NULL;
err = ENOMEM;
goto errout;
}
}
nicealign = ubsec_dmamap_aligned(q->q_src_map);
dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
#ifdef UBSEC_DEBUG
printf("src skip: %d\n", sskip);
#endif
for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) {
struct ubsec_pktbuf *pb;
bus_size_t packl = q->q_src_map->dm_segs[i].ds_len;
bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr;
if (sskip >= packl) {
sskip -= packl;
continue;
}
packl -= sskip;
packp += sskip;
sskip = 0;
if (packl > 0xfffc) {
err = EIO;
goto errout;
}
if (j == 0)
pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
else
pb = &dmap->d_dma->d_sbuf[j - 1];
pb->pb_addr = htole32(packp);
if (stheend) {
if (packl > stheend) {
pb->pb_len = htole32(stheend);
stheend = 0;
} else {
pb->pb_len = htole32(packl);
stheend -= packl;
}
} else
pb->pb_len = htole32(packl);
if ((i + 1) == q->q_src_map->dm_nsegs)
pb->pb_next = 0;
else
pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_sbuf[j]));
j++;
}
if (enccrd == NULL && maccrd != NULL) {
dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
dmap->d_dma->d_mcr.mcr_opktbuf.pb_next =
htole32(dmap->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_macbuf[0]));
#ifdef UBSEC_DEBUG
printf("opkt: %x %x %x\n",
dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
#endif
} else {
