static int crypto_cbc_decrypt_inplace(struct blkcipher_desc *desc,
struct blkcipher_walk *walk,
struct crypto_cipher *tfm,
void (*xor)(u8 *, const u8 *,
unsigned int))
{
void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
crypto_cipher_alg(tfm)->cia_decrypt;
int bsize = crypto_cipher_blocksize(tfm);
unsigned long alignmask = crypto_cipher_alignmask(tfm);
unsigned int nbytes = walk->nbytes;
u8 *src = walk->src.virt.addr;
u8 stack[bsize + alignmask];
u8 *first_iv = (u8 *)ALIGN((unsigned long)stack, alignmask + 1);
memcpy(first_iv, walk->iv, bsize);
/* Start of the last block. */
src += nbytes - nbytes % bsize - bsize;
memcpy(walk->iv, src, bsize);
for (;;) {
fn(crypto_cipher_tfm(tfm), src, src);
if ((nbytes -= bsize) < bsize)
break;
xor(src, src - bsize, bsize);
src -= bsize;
}
xor(src, first_iv, bsize);
return nbytes;
}
static int crypto_cbc_encrypt_inplace(struct blkcipher_desc *desc,
struct blkcipher_walk *walk,
struct crypto_cipher *tfm,
void (*xor)(u8 *, const u8 *,
unsigned int))
{
void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
crypto_cipher_alg(tfm)->cia_encrypt;
int bsize = crypto_cipher_blocksize(tfm);
unsigned int nbytes = walk->nbytes;
u8 *src = walk->src.virt.addr;
u8 *iv = walk->iv;
do {
xor(src, iv, bsize);
fn(crypto_cipher_tfm(tfm), src, src);
iv = src;
src += bsize;
} while ((nbytes -= bsize) >= bsize);
memcpy(walk->iv, iv, bsize);
return nbytes;
}
static int crypto_ctr_crypt_segment(struct blkcipher_walk *walk,
struct crypto_cipher *tfm)
{
void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
crypto_cipher_alg(tfm)->cia_encrypt;
unsigned int bsize = crypto_cipher_blocksize(tfm);
u8 *ctrblk = walk->iv;
u8 *src = walk->src.virt.addr;
u8 *dst = walk->dst.virt.addr;
unsigned int nbytes = walk->nbytes;
do {
/* create keystream */
fn(crypto_cipher_tfm(tfm), dst, ctrblk);
crypto_xor(dst, src, bsize);
/* increment counter in counterblock */
crypto_inc(ctrblk, bsize);
src += bsize;
dst += bsize;
} while ((nbytes -= bsize) >= bsize);
return nbytes;
}
static int crypto_ctr_crypt_inplace(struct blkcipher_walk *walk,
struct crypto_cipher *tfm)
{
void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
crypto_cipher_alg(tfm)->cia_encrypt;
unsigned int bsize = crypto_cipher_blocksize(tfm);
unsigned long alignmask = crypto_cipher_alignmask(tfm);
unsigned int nbytes = walk->nbytes;
u8 *ctrblk = walk->iv;
u8 *src = walk->src.virt.addr;
u8 tmp[bsize + alignmask];
u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1);
do {
/* create keystream */
fn(crypto_cipher_tfm(tfm), keystream, ctrblk);
crypto_xor(src, keystream, bsize);
/* increment counter in counterblock */
crypto_inc(ctrblk, bsize);
src += bsize;
} while ((nbytes -= bsize) >= bsize);
return nbytes;
}
static int crypto_cbc_decrypt_inplace(struct blkcipher_desc *desc,
struct blkcipher_walk *walk,
struct crypto_cipher *tfm)
{
void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
crypto_cipher_alg(tfm)->cia_decrypt;
int bsize = crypto_cipher_blocksize(tfm);
unsigned int nbytes = walk->nbytes;
u8 *src = walk->src.virt.addr;
u8 last_iv[bsize];
/* Start of the last block. */
src += nbytes - (nbytes & (bsize - 1)) - bsize;
memcpy(last_iv, src, bsize);
for (;;) {
fn(crypto_cipher_tfm(tfm), src, src);
if ((nbytes -= bsize) < bsize)
break;
crypto_xor(src, src - bsize, bsize);
src -= bsize;
}
crypto_xor(src, walk->iv, bsize);
memcpy(walk->iv, last_iv, bsize);
return nbytes;
}
static int crypto_cbc_decrypt_segment(struct blkcipher_desc *desc,
struct blkcipher_walk *walk,
struct crypto_cipher *tfm)
{
void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
crypto_cipher_alg(tfm)->cia_decrypt;
int bsize = crypto_cipher_blocksize(tfm);
unsigned int nbytes = walk->nbytes;
u8 *src = walk->src.virt.addr;
u8 *dst = walk->dst.virt.addr;
u8 *iv = walk->iv;
do {
fn(crypto_cipher_tfm(tfm), dst, src);
crypto_xor(dst, iv, bsize);
iv = src;
src += bsize;
dst += bsize;
} while ((nbytes -= bsize) >= bsize);
memcpy(walk->iv, iv, bsize);
return nbytes;
}
static int crypto_pcbc_encrypt_inplace(struct blkcipher_desc *desc,
struct blkcipher_walk *walk,
struct crypto_cipher *tfm)
{
void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
crypto_cipher_alg(tfm)->cia_encrypt;
int bsize = crypto_cipher_blocksize(tfm);
unsigned int nbytes = walk->nbytes;
u8 *src = walk->src.virt.addr;
u8 *iv = walk->iv;
u8 tmpbuf[bsize];
do {
memcpy(tmpbuf, src, bsize);
crypto_xor(iv, src, bsize);
fn(crypto_cipher_tfm(tfm), src, iv);
memcpy(iv, tmpbuf, bsize);
crypto_xor(iv, src, bsize);
src += bsize;
} while ((nbytes -= bsize) >= bsize);
memcpy(walk->iv, iv, bsize);
return nbytes;
}
static int crypto_ecb_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct blkcipher_walk walk;
struct crypto_blkcipher *tfm = desc->tfm;
struct crypto_ecb_ctx *ctx = crypto_blkcipher_ctx(tfm);
struct crypto_cipher *child = ctx->child;
blkcipher_walk_init(&walk, dst, src, nbytes);
return crypto_ecb_crypt(desc, &walk, child,
crypto_cipher_alg(child)->cia_decrypt);
}
static int crypto_ecb_crypt(struct blkcipher_desc *desc,
struct blkcipher_walk *walk,
struct crypto_cipher *tfm,
void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
{
int bsize = crypto_cipher_blocksize(tfm);
unsigned int nbytes;
int err;
#ifdef CONFIG_CRYPTO_DEV_REALTEK
struct crypto_ecb_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
#endif
err = blkcipher_walk_virt(desc, walk);
while ((nbytes = walk->nbytes)) {
u8 *wsrc = walk->src.virt.addr;
u8 *wdst = walk->dst.virt.addr;
#ifdef CONFIG_CRYPTO_DEV_REALTEK
if (ctx->rtl_ctx.mode >= 0)
{
nbytes = rtl_cipher_crypt(tfm,
fn == crypto_cipher_alg(tfm)->cia_encrypt,
&ctx->rtl_ctx, wsrc, nbytes,
walk->iv, wdst);
err = blkcipher_walk_done(desc, walk, nbytes);
continue;
}
#endif
do {
fn(crypto_cipher_tfm(tfm), wdst, wsrc);
wsrc += bsize;
wdst += bsize;
} while ((nbytes -= bsize) >= bsize);
err = blkcipher_walk_done(desc, walk, nbytes);
}
return err;
}
static int crypt(struct blkcipher_desc *d,
struct blkcipher_walk *w, struct priv *ctx,
void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
{
int err;
unsigned int avail;
const int bs = LRW_BLOCK_SIZE;
struct sinfo s = {
.tfm = crypto_cipher_tfm(ctx->child),
.fn = fn
};
be128 *iv;
u8 *wsrc;
u8 *wdst;
err = blkcipher_walk_virt(d, w);
if (!(avail = w->nbytes))
return err;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
/* calculate first value of T */
iv = (be128 *)w->iv;
s.t = *iv;
/* T <- I*Key2 */
gf128mul_64k_bbe(&s.t, ctx->table.table);
goto first;
for (;;) {
do {
/* T <- I*Key2, using the optimization
* discussed in the specification */
be128_xor(&s.t, &s.t,
&ctx->table.mulinc[get_index128(iv)]);
inc(iv);
first:
lrw_round(&s, wdst, wsrc);
wsrc += bs;
wdst += bs;
} while ((avail -= bs) >= bs);
err = blkcipher_walk_done(d, w, avail);
if (!(avail = w->nbytes))
break;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
}
return err;
}
static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx,
crypto_cipher_alg(ctx->child)->cia_encrypt);
}
static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx,
crypto_cipher_alg(ctx->child)->cia_decrypt);
}
int lrw_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
struct scatterlist *ssrc, unsigned int nbytes,
struct lrw_crypt_req *req)
{
const unsigned int bsize = LRW_BLOCK_SIZE;
const unsigned int max_blks = req->tbuflen / bsize;
struct lrw_table_ctx *ctx = req->table_ctx;
struct blkcipher_walk walk;
unsigned int nblocks;
be128 *iv, *src, *dst, *t;
be128 *t_buf = req->tbuf;
int err, i;
BUG_ON(max_blks < 1);
blkcipher_walk_init(&walk, sdst, ssrc, nbytes);
err = blkcipher_walk_virt(desc, &walk);
nbytes = walk.nbytes;
if (!nbytes)
return err;
nblocks = min(walk.nbytes / bsize, max_blks);
src = (be128 *)walk.src.virt.addr;
dst = (be128 *)walk.dst.virt.addr;
/* calculate first value of T */
iv = (be128 *)walk.iv;
t_buf[0] = *iv;
/* T <- I*Key2 */
gf128mul_64k_bbe(&t_buf[0], ctx->table);
i = 0;
goto first;
for (;;) {
do {
for (i = 0; i < nblocks; i++) {
/* T <- I*Key2, using the optimization
* discussed in the specification */
be128_xor(&t_buf[i], t,
&ctx->mulinc[get_index128(iv)]);
inc(iv);
first:
t = &t_buf[i];
/* PP <- T xor P */
be128_xor(dst + i, t, src + i);
}
/* CC <- E(Key2,PP) */
req->crypt_fn(req->crypt_ctx, (u8 *)dst,
nblocks * bsize);
/* C <- T xor CC */
for (i = 0; i < nblocks; i++)
be128_xor(dst + i, dst + i, &t_buf[i]);
src += nblocks;
dst += nblocks;
nbytes -= nblocks * bsize;
nblocks = min(nbytes / bsize, max_blks);
} while (nblocks > 0);
err = blkcipher_walk_done(desc, &walk, nbytes);
nbytes = walk.nbytes;
if (!nbytes)
break;
nblocks = min(nbytes / bsize, max_blks);
src = (be128 *)walk.src.virt.addr;
dst = (be128 *)walk.dst.virt.addr;
}
return err;
}
EXPORT_SYMBOL_GPL(lrw_crypt);
static int init_tfm(struct crypto_tfm *tfm)
{
struct crypto_cipher *cipher;
struct crypto_instance *inst = (void *)tfm->__crt_alg;
struct crypto_spawn *spawn = crypto_instance_ctx(inst);
struct priv *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher))
return PTR_ERR(cipher);
if (crypto_cipher_blocksize(cipher) != LRW_BLOCK_SIZE) {
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
crypto_free_cipher(cipher);
return -EINVAL;
}
ctx->child = cipher;
return 0;
}
static void exit_tfm(struct crypto_tfm *tfm)
{
struct priv *ctx = crypto_tfm_ctx(tfm);
lrw_free_table(&ctx->table);
crypto_free_cipher(ctx->child);
}
static struct crypto_instance *alloc(struct rtattr **tb)
{
struct crypto_instance *inst;
struct crypto_alg *alg;
int err;
err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
if (err)
return ERR_PTR(err);
alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
CRYPTO_ALG_TYPE_MASK);
if (IS_ERR(alg))
return ERR_CAST(alg);
inst = crypto_alloc_instance("lrw", alg);
if (IS_ERR(inst))
goto out_put_alg;
inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
inst->alg.cra_priority = alg->cra_priority;
inst->alg.cra_blocksize = alg->cra_blocksize;
if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7;
else inst->alg.cra_alignmask = alg->cra_alignmask;
inst->alg.cra_type = &crypto_blkcipher_type;
if (!(alg->cra_blocksize % 4))
inst->alg.cra_alignmask |= 3;
inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
inst->alg.cra_blkcipher.min_keysize =
alg->cra_cipher.cia_min_keysize + alg->cra_blocksize;
inst->alg.cra_blkcipher.max_keysize =
alg->cra_cipher.cia_max_keysize + alg->cra_blocksize;
inst->alg.cra_ctxsize = sizeof(struct priv);
inst->alg.cra_init = init_tfm;
inst->alg.cra_exit = exit_tfm;
inst->alg.cra_blkcipher.setkey = setkey;
inst->alg.cra_blkcipher.encrypt = encrypt;
inst->alg.cra_blkcipher.decrypt = decrypt;
out_put_alg:
crypto_mod_put(alg);
return inst;
}
static void free(struct crypto_instance *inst)
{
crypto_drop_spawn(crypto_instance_ctx(inst));
kfree(inst);
}
static int crypt(struct blkcipher_desc *d,
struct blkcipher_walk *w, struct priv *ctx,
void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
{
int err;
unsigned int avail;
const int bs = crypto_cipher_blocksize(ctx->child);
struct sinfo s = {
.tfm = crypto_cipher_tfm(ctx->child),
.fn = fn
};
be128 *iv;
u8 *wsrc;
u8 *wdst;
err = blkcipher_walk_virt(d, w);
if (!(avail = w->nbytes))
return err;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
/* calculate first value of T */
iv = (be128 *)w->iv;
s.t = *iv;
/* T <- I*Key2 */
gf128mul_64k_bbe(&s.t, ctx->table);
goto first;
for (;;) {
do {
/* T <- I*Key2, using the optimization
* discussed in the specification */
be128_xor(&s.t, &s.t, &ctx->mulinc[get_index128(iv)]);
inc(iv);
first:
lrw_round(&s, wdst, wsrc);
wsrc += bs;
wdst += bs;
} while ((avail -= bs) >= bs);
err = blkcipher_walk_done(d, w, avail);
if (!(avail = w->nbytes))
break;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
}
return err;
}
static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx,
crypto_cipher_alg(ctx->child)->cia_encrypt);
}
static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx,
crypto_cipher_alg(ctx->child)->cia_decrypt);
}
static int init_tfm(struct crypto_tfm *tfm)
{
struct crypto_cipher *cipher;
struct crypto_instance *inst = (void *)tfm->__crt_alg;
struct crypto_spawn *spawn = crypto_instance_ctx(inst);
struct priv *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher))
return PTR_ERR(cipher);
if (crypto_cipher_blocksize(cipher) != 16) {
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
return -EINVAL;
}
ctx->child = cipher;
return 0;
}
static void exit_tfm(struct crypto_tfm *tfm)
{
struct priv *ctx = crypto_tfm_ctx(tfm);
if (ctx->table)
gf128mul_free_64k(ctx->table);
crypto_free_cipher(ctx->child);
}
static struct crypto_instance *alloc(struct rtattr **tb)
{
struct crypto_instance *inst;
struct crypto_alg *alg;
int err;
err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
if (err)
return ERR_PTR(err);
alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
CRYPTO_ALG_TYPE_MASK);
if (IS_ERR(alg))
return ERR_CAST(alg);
inst = crypto_alloc_instance("lrw", alg);
if (IS_ERR(inst))
goto out_put_alg;
inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
inst->alg.cra_priority = alg->cra_priority;
inst->alg.cra_blocksize = alg->cra_blocksize;
if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7;
else inst->alg.cra_alignmask = alg->cra_alignmask;
inst->alg.cra_type = &crypto_blkcipher_type;
if (!(alg->cra_blocksize % 4))
inst->alg.cra_alignmask |= 3;
inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
inst->alg.cra_blkcipher.min_keysize =
alg->cra_cipher.cia_min_keysize + alg->cra_blocksize;
inst->alg.cra_blkcipher.max_keysize =
alg->cra_cipher.cia_max_keysize + alg->cra_blocksize;
inst->alg.cra_ctxsize = sizeof(struct priv);
inst->alg.cra_init = init_tfm;
inst->alg.cra_exit = exit_tfm;
inst->alg.cra_blkcipher.setkey = setkey;
inst->alg.cra_blkcipher.encrypt = encrypt;
inst->alg.cra_blkcipher.decrypt = decrypt;
out_put_alg:
crypto_mod_put(alg);
return inst;
}
static void free(struct crypto_instance *inst)
{
crypto_drop_spawn(crypto_instance_ctx(inst));
kfree(inst);
}
static int crypt(struct blkcipher_desc *d,
struct blkcipher_walk *w, struct priv *ctx,
void (*tw)(struct crypto_tfm *, u8 *, const u8 *),
void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
{
int err;
unsigned int avail;
const int bs = XTS_BLOCK_SIZE;
struct sinfo s = {
.tfm = crypto_cipher_tfm(ctx->child),
.fn = fn
};
u8 *wsrc;
u8 *wdst;
err = blkcipher_walk_virt(d, w);
if (!w->nbytes)
return err;
s.t = (be128 *)w->iv;
avail = w->nbytes;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
tw(crypto_cipher_tfm(ctx->tweak), w->iv, w->iv);
goto first;
for (;;) {
do {
gf128mul_x_ble(s.t, s.t);
first:
xts_round(&s, wdst, wsrc);
wsrc += bs;
wdst += bs;
} while ((avail -= bs) >= bs);
err = blkcipher_walk_done(d, w, avail);
if (!w->nbytes)
break;
avail = w->nbytes;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
}
return err;
}
static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
crypto_cipher_alg(ctx->child)->cia_encrypt);
}
static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
crypto_cipher_alg(ctx->child)->cia_decrypt);
}
int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
struct scatterlist *ssrc, unsigned int nbytes,
struct xts_crypt_req *req)
{
const unsigned int bsize = XTS_BLOCK_SIZE;
const unsigned int max_blks = req->tbuflen / bsize;
struct blkcipher_walk walk={};
unsigned int nblocks;
be128 *src, *dst, *t;
be128 *t_buf = req->tbuf;
int err, i;
BUG_ON(max_blks < 1);
blkcipher_walk_init(&walk, sdst, ssrc, nbytes);
err = blkcipher_walk_virt(desc, &walk);
nbytes = walk.nbytes;
if (!nbytes)
return err;
nblocks = min(nbytes / bsize, max_blks);
src = (be128 *)walk.src.virt.addr;
dst = (be128 *)walk.dst.virt.addr;
req->tweak_fn(req->tweak_ctx, (u8 *)&t_buf[0], walk.iv);
i = 0;
goto first;
for (;;) {
do {
for (i = 0; i < nblocks; i++) {
gf128mul_x_ble(&t_buf[i], t);
first:
t = &t_buf[i];
be128_xor(dst + i, t, src + i);
}
req->crypt_fn(req->crypt_ctx, (u8 *)dst,
nblocks * bsize);
for (i = 0; i < nblocks; i++)
be128_xor(dst + i, dst + i, &t_buf[i]);
src += nblocks;
dst += nblocks;
nbytes -= nblocks * bsize;
nblocks = min(nbytes / bsize, max_blks);
} while (nblocks > 0);
*(be128 *)walk.iv = *t;
err = blkcipher_walk_done(desc, &walk, nbytes);
nbytes = walk.nbytes;
if (!nbytes)
break;
nblocks = min(nbytes / bsize, max_blks);
src = (be128 *)walk.src.virt.addr;
dst = (be128 *)walk.dst.virt.addr;
}
return err;
}
EXPORT_SYMBOL_GPL(xts_crypt);
static int init_tfm(struct crypto_tfm *tfm)
{
struct crypto_cipher *cipher;
struct crypto_instance *inst = (void *)tfm->__crt_alg;
struct crypto_spawn *spawn = crypto_instance_ctx(inst);
struct priv *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher))
return PTR_ERR(cipher);
if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) {
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
crypto_free_cipher(cipher);
return -EINVAL;
}
ctx->child = cipher;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher)) {
crypto_free_cipher(ctx->child);
return PTR_ERR(cipher);
}
if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) {
crypto_free_cipher(cipher);
crypto_free_cipher(ctx->child);
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
return -EINVAL;
}
ctx->tweak = cipher;
return 0;
}
static void exit_tfm(struct crypto_tfm *tfm)
{
struct priv *ctx = crypto_tfm_ctx(tfm);
crypto_free_cipher(ctx->child);
crypto_free_cipher(ctx->tweak);
}
static struct crypto_instance *alloc(struct rtattr **tb)
{
struct crypto_instance *inst;
struct crypto_alg *alg;
int err;
err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
if (err)
return ERR_PTR(err);
alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
CRYPTO_ALG_TYPE_MASK);
if (IS_ERR(alg))
return ERR_CAST(alg);
inst = crypto_alloc_instance("xts", alg);
if (IS_ERR(inst))
goto out_put_alg;
inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
inst->alg.cra_priority = alg->cra_priority;
inst->alg.cra_blocksize = alg->cra_blocksize;
if (alg->cra_alignmask < 7)
inst->alg.cra_alignmask = 7;
else
inst->alg.cra_alignmask = alg->cra_alignmask;
inst->alg.cra_type = &crypto_blkcipher_type;
inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
inst->alg.cra_blkcipher.min_keysize =
2 * alg->cra_cipher.cia_min_keysize;
inst->alg.cra_blkcipher.max_keysize =
2 * alg->cra_cipher.cia_max_keysize;
inst->alg.cra_ctxsize = sizeof(struct priv);
inst->alg.cra_init = init_tfm;
inst->alg.cra_exit = exit_tfm;
inst->alg.cra_blkcipher.setkey = setkey;
inst->alg.cra_blkcipher.encrypt = encrypt;
inst->alg.cra_blkcipher.decrypt = decrypt;
out_put_alg:
crypto_mod_put(alg);
return inst;
}
static void free(struct crypto_instance *inst)
{
crypto_drop_spawn(crypto_instance_ctx(inst));
kfree(inst);
}
static int crypt(struct blkcipher_desc *d,
struct blkcipher_walk *w, struct priv *ctx,
void (*tw)(struct crypto_tfm *, u8 *, const u8 *),
void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
{
int err;
unsigned int avail;
const int bs = crypto_cipher_blocksize(ctx->child);
struct sinfo s = {
.tfm = crypto_cipher_tfm(ctx->child),
.fn = fn
};
u8 *wsrc;
u8 *wdst;
err = blkcipher_walk_virt(d, w);
if (!w->nbytes)
return err;
s.t = (be128 *)w->iv;
avail = w->nbytes;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
/* calculate first value of T */
tw(crypto_cipher_tfm(ctx->tweak), w->iv, w->iv);
goto first;
for (;;) {
do {
gf128mul_x_ble(s.t, s.t);
first:
xts_round(&s, wdst, wsrc);
wsrc += bs;
wdst += bs;
} while ((avail -= bs) >= bs);
err = blkcipher_walk_done(d, w, avail);
if (!w->nbytes)
break;
avail = w->nbytes;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
}
return err;
}
static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
crypto_cipher_alg(ctx->child)->cia_encrypt);
}
static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
crypto_cipher_alg(ctx->child)->cia_decrypt);
}
static int init_tfm(struct crypto_tfm *tfm)
{
struct crypto_cipher *cipher;
struct crypto_instance *inst = (void *)tfm->__crt_alg;
struct crypto_spawn *spawn = crypto_instance_ctx(inst);
struct priv *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher))
return PTR_ERR(cipher);
if (crypto_cipher_blocksize(cipher) != 16) {
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
crypto_free_cipher(cipher);
return -EINVAL;
}
ctx->child = cipher;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher)) {
crypto_free_cipher(ctx->child);
return PTR_ERR(cipher);
}
/* this check isn't really needed, leave it here just in case */
if (crypto_cipher_blocksize(cipher) != 16) {
crypto_free_cipher(cipher);
crypto_free_cipher(ctx->child);
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
return -EINVAL;
}
ctx->tweak = cipher;
return 0;
}
static void exit_tfm(struct crypto_tfm *tfm)
{
struct priv *ctx = crypto_tfm_ctx(tfm);
crypto_free_cipher(ctx->child);
crypto_free_cipher(ctx->tweak);
}
static struct crypto_instance *alloc(struct rtattr **tb)
{
struct crypto_instance *inst;
struct crypto_alg *alg;
int err;
err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
if (err)
return ERR_PTR(err);
alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
CRYPTO_ALG_TYPE_MASK);
if (IS_ERR(alg))
return ERR_PTR(PTR_ERR(alg));
inst = crypto_alloc_instance("xts", alg);
if (IS_ERR(inst))
goto out_put_alg;
inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
inst->alg.cra_priority = alg->cra_priority;
inst->alg.cra_blocksize = alg->cra_blocksize;
if (alg->cra_alignmask < 7)
inst->alg.cra_alignmask = 7;
else
inst->alg.cra_alignmask = alg->cra_alignmask;
inst->alg.cra_type = &crypto_blkcipher_type;
inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
inst->alg.cra_blkcipher.min_keysize =
2 * alg->cra_cipher.cia_min_keysize;
inst->alg.cra_blkcipher.max_keysize =
2 * alg->cra_cipher.cia_max_keysize;
inst->alg.cra_ctxsize = sizeof(struct priv);
inst->alg.cra_init = init_tfm;
inst->alg.cra_exit = exit_tfm;
inst->alg.cra_blkcipher.setkey = setkey;
inst->alg.cra_blkcipher.encrypt = encrypt;
inst->alg.cra_blkcipher.decrypt = decrypt;
out_put_alg:
crypto_mod_put(alg);
return inst;
}
static void free(struct crypto_instance *inst)
{
crypto_drop_spawn(crypto_instance_ctx(inst));
kfree(inst);
}
