static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk walk;
unsigned int blocks;
int err;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
kernel_neon_begin();
while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
ce_aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
(u8 *)ctx->key_dec, num_rounds(ctx), blocks,
walk.iv);
err = blkcipher_walk_done(desc, &walk,
walk.nbytes % AES_BLOCK_SIZE);
}
kernel_neon_end();
return err;
}
static int ecb_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
struct blkcipher_walk walk;
int err;
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
kernel_fpu_begin();
while ((nbytes = walk.nbytes)) {
aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
nbytes & AES_BLOCK_MASK);
nbytes &= AES_BLOCK_SIZE - 1;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
kernel_fpu_end();
return err;
}
static int crypto_cbc_encrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct blkcipher_walk walk;
struct crypto_blkcipher *tfm = desc->tfm;
struct crypto_cbc_ctx *ctx = crypto_blkcipher_ctx(tfm);
struct crypto_cipher *child = ctx->child;
int err;
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
while ((nbytes = walk.nbytes)) {
if (walk.src.virt.addr == walk.dst.virt.addr)
nbytes = crypto_cbc_encrypt_inplace(desc, &walk, child);
else
nbytes = crypto_cbc_encrypt_segment(desc, &walk, child);
err = blkcipher_walk_done(desc, &walk, nbytes);
}
return err;
}
static int ecb_paes_crypt(struct blkcipher_desc *desc,
unsigned long modifier,
struct blkcipher_walk *walk)
{
struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
unsigned int nbytes, n, k;
int ret;
ret = blkcipher_walk_virt(desc, walk);
while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) {
/* only use complete blocks */
n = nbytes & ~(AES_BLOCK_SIZE - 1);
k = cpacf_km(ctx->fc | modifier, ctx->pk.protkey,
walk->dst.virt.addr, walk->src.virt.addr, n);
if (k)
ret = blkcipher_walk_done(desc, walk, nbytes - k);
if (k < n) {
if (__paes_set_key(ctx) != 0)
return blkcipher_walk_done(desc, walk, -EIO);
}
}
return ret;
}
int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
const unsigned int bsize = 128 / 8;
bool fpu_enabled = false;
struct blkcipher_walk walk;
int err;
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
while ((nbytes = walk.nbytes)) {
fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
desc, fpu_enabled, nbytes);
nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
err = blkcipher_walk_done(desc, &walk, nbytes);
}
glue_fpu_end(fpu_enabled);
return err;
}
static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
int err, first, rounds = 6 + ctx->key1.key_length / 4;
struct blkcipher_walk walk;
unsigned int blocks;
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
kernel_neon_begin();
for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
(u8 *)ctx->key1.key_dec, rounds, blocks,
(u8 *)ctx->key2.key_enc, walk.iv, first);
err = blkcipher_walk_done(desc, &walk, 0);
}
kernel_neon_end();
return err;
}
/* for implementations implementing faster XTS IV generator */
int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes,
void (*tweak_fn)(void *ctx, u8 *dst, const u8 *src),
void *tweak_ctx, void *crypt_ctx)
{
const unsigned int bsize = 128 / 8;
bool fpu_enabled = false;
struct blkcipher_walk walk;
int err;
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
nbytes = walk.nbytes;
if (!nbytes)
return err;
/* set minimum length to bsize, for tweak_fn */
fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
desc, fpu_enabled,
nbytes < bsize ? bsize : nbytes);
/* calculate first value of T */
tweak_fn(tweak_ctx, walk.iv, walk.iv);
while (nbytes) {
nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk);
err = blkcipher_walk_done(desc, &walk, nbytes);
nbytes = walk.nbytes;
}
glue_fpu_end(fpu_enabled);
return err;
}
/*! \fn int ctr_rfc3686_aes_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes)
* \ingroup IFX_AES_FUNCTIONS
* \brief Counter mode AES (rfc3686) decrypt using linux crypto blkcipher
* \param desc blkcipher descriptor
* \param dst output scatterlist
* \param src input scatterlist
* \param nbytes data size in bytes
* \return err
*/
int ctr_rfc3686_aes_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int nbytes)
{
struct aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk walk;
int err;
u8 rfc3686_iv[16];
#ifdef CONFIG_CRYPTO_DEV_PWR_SAVE_MODE
powerup_deu(AES_INIT);
#endif
blkcipher_walk_init(&walk, dst, src, nbytes);
err = blkcipher_walk_virt(desc, &walk);
/* set up counter block */
memcpy(rfc3686_iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
memcpy(rfc3686_iv + CTR_RFC3686_NONCE_SIZE, walk.iv, CTR_RFC3686_IV_SIZE);
/* initialize counter portion of counter block */
*(__be32 *)(rfc3686_iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
cpu_to_be32(1);
while ((nbytes = walk.nbytes)) {
nbytes -= (nbytes % AES_BLOCK_SIZE);
ifx_deu_aes_ctr(ctx, walk.dst.virt.addr, walk.src.virt.addr,
rfc3686_iv, nbytes, CRYPTO_DIR_DECRYPT, 0);
nbytes &= AES_BLOCK_SIZE - 1;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
#ifdef CONFIG_CRYPTO_DEV_PWR_SAVE_MODE
powerdown_deu(AES_INIT);
#endif
return err;
}
static int cbc_desall_crypt(struct blkcipher_desc *desc, unsigned long fc,
struct blkcipher_walk *walk)
{
struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
unsigned int nbytes, n;
int ret;
struct {
u8 iv[DES_BLOCK_SIZE];
u8 key[DES3_KEY_SIZE];
} param;
ret = blkcipher_walk_virt(desc, walk);
memcpy(param.iv, walk->iv, DES_BLOCK_SIZE);
memcpy(param.key, ctx->key, DES3_KEY_SIZE);
while ((nbytes = walk->nbytes) >= DES_BLOCK_SIZE) {
/* only use complete blocks */
n = nbytes & ~(DES_BLOCK_SIZE - 1);
cpacf_kmc(fc, ¶m, walk->dst.virt.addr,
walk->src.virt.addr, n);
ret = blkcipher_walk_done(desc, walk, nbytes - n);
}
memcpy(walk->iv, param.iv, DES_BLOCK_SIZE);
return ret;
}
static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
int ret;
struct blkcipher_walk walk;
struct p8_aes_cbc_ctx *ctx =
crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
struct blkcipher_desc fallback_desc = {
.tfm = ctx->fallback,
.info = desc->info,
.flags = desc->flags
};
if (in_interrupt()) {
ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src,
nbytes);
} else {
preempt_disable();
pagefault_disable();
enable_kernel_vsx();
blkcipher_walk_init(&walk, dst, src, nbytes);
ret = blkcipher_walk_virt(desc, &walk);
while ((nbytes = walk.nbytes)) {
aes_p8_cbc_encrypt(walk.src.virt.addr,
walk.dst.virt.addr,
nbytes & AES_BLOCK_MASK,
&ctx->enc_key, walk.iv, 1);
nbytes &= AES_BLOCK_SIZE - 1;
ret = blkcipher_walk_done(desc, &walk, nbytes);
}
disable_kernel_vsx();
pagefault_enable();
preempt_enable();
}
return ret;
}
static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
int ret;
struct blkcipher_walk walk;
struct p8_aes_cbc_ctx *ctx =
crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
struct blkcipher_desc fallback_desc = {
.tfm = ctx->fallback,
.info = desc->info,
.flags = desc->flags
};
if (in_interrupt()) {
ret = crypto_blkcipher_decrypt(&fallback_desc, dst, src,
nbytes);
} else {
preempt_disable();
pagefault_disable();
enable_kernel_vsx();
blkcipher_walk_init(&walk, dst, src, nbytes);
ret = blkcipher_walk_virt(desc, &walk);
while ((nbytes = walk.nbytes)) {
aes_p8_cbc_encrypt(walk.src.virt.addr,
walk.dst.virt.addr,
nbytes & AES_BLOCK_MASK,
&ctx->dec_key, walk.iv, 0);
nbytes &= AES_BLOCK_SIZE - 1;
ret = blkcipher_walk_done(desc, &walk, nbytes);
}
disable_kernel_vsx();
pagefault_enable();
preempt_enable();
}
return ret;
}
struct crypto_alg p8_aes_cbc_alg = {
.cra_name = "cbc(aes)",
.cra_driver_name = "p8_aes_cbc",
.cra_module = THIS_MODULE,
.cra_priority = 1000,
.cra_type = &crypto_blkcipher_type,
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
.cra_alignmask = 0,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct p8_aes_cbc_ctx),
.cra_init = p8_aes_cbc_init,
.cra_exit = p8_aes_cbc_exit,
.cra_blkcipher = {
.ivsize = AES_BLOCK_SIZE,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = p8_aes_cbc_setkey,
.encrypt = p8_aes_cbc_encrypt,
.decrypt = p8_aes_cbc_decrypt,
},
};
static int crypt(struct blkcipher_desc *d,
struct blkcipher_walk *w, struct priv *ctx,
void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
{
int err;
unsigned int avail;
const int bs = LRW_BLOCK_SIZE;
struct sinfo s = {
.tfm = crypto_cipher_tfm(ctx->child),
.fn = fn
};
be128 *iv;
u8 *wsrc;
u8 *wdst;
err = blkcipher_walk_virt(d, w);
if (!(avail = w->nbytes))
return err;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
/* calculate first value of T */
iv = (be128 *)w->iv;
s.t = *iv;
/* T <- I*Key2 */
gf128mul_64k_bbe(&s.t, ctx->table.table);
goto first;
for (;;) {
do {
/* T <- I*Key2, using the optimization
* discussed in the specification */
be128_xor(&s.t, &s.t,
&ctx->table.mulinc[get_index128(iv)]);
inc(iv);
first:
lrw_round(&s, wdst, wsrc);
wsrc += bs;
wdst += bs;
} while ((avail -= bs) >= bs);
err = blkcipher_walk_done(d, w, avail);
if (!(avail = w->nbytes))
break;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
}
return err;
}
static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx,
crypto_cipher_alg(ctx->child)->cia_encrypt);
}
static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx,
crypto_cipher_alg(ctx->child)->cia_decrypt);
}
int lrw_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
struct scatterlist *ssrc, unsigned int nbytes,
struct lrw_crypt_req *req)
{
const unsigned int bsize = LRW_BLOCK_SIZE;
const unsigned int max_blks = req->tbuflen / bsize;
struct lrw_table_ctx *ctx = req->table_ctx;
struct blkcipher_walk walk;
unsigned int nblocks;
be128 *iv, *src, *dst, *t;
be128 *t_buf = req->tbuf;
int err, i;
BUG_ON(max_blks < 1);
blkcipher_walk_init(&walk, sdst, ssrc, nbytes);
err = blkcipher_walk_virt(desc, &walk);
nbytes = walk.nbytes;
if (!nbytes)
return err;
nblocks = min(walk.nbytes / bsize, max_blks);
src = (be128 *)walk.src.virt.addr;
dst = (be128 *)walk.dst.virt.addr;
/* calculate first value of T */
iv = (be128 *)walk.iv;
t_buf[0] = *iv;
/* T <- I*Key2 */
gf128mul_64k_bbe(&t_buf[0], ctx->table);
i = 0;
goto first;
for (;;) {
do {
for (i = 0; i < nblocks; i++) {
/* T <- I*Key2, using the optimization
* discussed in the specification */
be128_xor(&t_buf[i], t,
&ctx->mulinc[get_index128(iv)]);
inc(iv);
first:
t = &t_buf[i];
/* PP <- T xor P */
be128_xor(dst + i, t, src + i);
}
/* CC <- E(Key2,PP) */
req->crypt_fn(req->crypt_ctx, (u8 *)dst,
nblocks * bsize);
/* C <- T xor CC */
for (i = 0; i < nblocks; i++)
be128_xor(dst + i, dst + i, &t_buf[i]);
src += nblocks;
dst += nblocks;
nbytes -= nblocks * bsize;
nblocks = min(nbytes / bsize, max_blks);
} while (nblocks > 0);
err = blkcipher_walk_done(desc, &walk, nbytes);
nbytes = walk.nbytes;
if (!nbytes)
break;
nblocks = min(nbytes / bsize, max_blks);
src = (be128 *)walk.src.virt.addr;
dst = (be128 *)walk.dst.virt.addr;
}
return err;
}
EXPORT_SYMBOL_GPL(lrw_crypt);
static int init_tfm(struct crypto_tfm *tfm)
{
struct crypto_cipher *cipher;
struct crypto_instance *inst = (void *)tfm->__crt_alg;
struct crypto_spawn *spawn = crypto_instance_ctx(inst);
struct priv *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher))
return PTR_ERR(cipher);
if (crypto_cipher_blocksize(cipher) != LRW_BLOCK_SIZE) {
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
crypto_free_cipher(cipher);
return -EINVAL;
}
ctx->child = cipher;
return 0;
}
static void exit_tfm(struct crypto_tfm *tfm)
{
struct priv *ctx = crypto_tfm_ctx(tfm);
lrw_free_table(&ctx->table);
crypto_free_cipher(ctx->child);
}
static struct crypto_instance *alloc(struct rtattr **tb)
{
struct crypto_instance *inst;
struct crypto_alg *alg;
int err;
err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
if (err)
return ERR_PTR(err);
alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
CRYPTO_ALG_TYPE_MASK);
if (IS_ERR(alg))
return ERR_CAST(alg);
inst = crypto_alloc_instance("lrw", alg);
if (IS_ERR(inst))
goto out_put_alg;
inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
inst->alg.cra_priority = alg->cra_priority;
inst->alg.cra_blocksize = alg->cra_blocksize;
if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7;
else inst->alg.cra_alignmask = alg->cra_alignmask;
inst->alg.cra_type = &crypto_blkcipher_type;
if (!(alg->cra_blocksize % 4))
inst->alg.cra_alignmask |= 3;
inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
inst->alg.cra_blkcipher.min_keysize =
alg->cra_cipher.cia_min_keysize + alg->cra_blocksize;
inst->alg.cra_blkcipher.max_keysize =
alg->cra_cipher.cia_max_keysize + alg->cra_blocksize;
inst->alg.cra_ctxsize = sizeof(struct priv);
inst->alg.cra_init = init_tfm;
inst->alg.cra_exit = exit_tfm;
inst->alg.cra_blkcipher.setkey = setkey;
inst->alg.cra_blkcipher.encrypt = encrypt;
inst->alg.cra_blkcipher.decrypt = decrypt;
out_put_alg:
crypto_mod_put(alg);
return inst;
}
static void free(struct crypto_instance *inst)
{
crypto_drop_spawn(crypto_instance_ctx(inst));
kfree(inst);
}
static int crypt(struct blkcipher_desc *d,
struct blkcipher_walk *w, struct priv *ctx,
void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
{
int err;
unsigned int avail;
const int bs = crypto_cipher_blocksize(ctx->child);
struct sinfo s = {
.tfm = crypto_cipher_tfm(ctx->child),
.fn = fn
};
be128 *iv;
u8 *wsrc;
u8 *wdst;
err = blkcipher_walk_virt(d, w);
if (!(avail = w->nbytes))
return err;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
/* calculate first value of T */
iv = (be128 *)w->iv;
s.t = *iv;
/* T <- I*Key2 */
gf128mul_64k_bbe(&s.t, ctx->table);
goto first;
for (;;) {
do {
/* T <- I*Key2, using the optimization
* discussed in the specification */
be128_xor(&s.t, &s.t, &ctx->mulinc[get_index128(iv)]);
inc(iv);
first:
lrw_round(&s, wdst, wsrc);
wsrc += bs;
wdst += bs;
} while ((avail -= bs) >= bs);
err = blkcipher_walk_done(d, w, avail);
if (!(avail = w->nbytes))
break;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
}
return err;
}
static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx,
crypto_cipher_alg(ctx->child)->cia_encrypt);
}
static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx,
crypto_cipher_alg(ctx->child)->cia_decrypt);
}
static int init_tfm(struct crypto_tfm *tfm)
{
struct crypto_cipher *cipher;
struct crypto_instance *inst = (void *)tfm->__crt_alg;
struct crypto_spawn *spawn = crypto_instance_ctx(inst);
struct priv *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher))
return PTR_ERR(cipher);
if (crypto_cipher_blocksize(cipher) != 16) {
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
return -EINVAL;
}
ctx->child = cipher;
return 0;
}
static void exit_tfm(struct crypto_tfm *tfm)
{
struct priv *ctx = crypto_tfm_ctx(tfm);
if (ctx->table)
gf128mul_free_64k(ctx->table);
crypto_free_cipher(ctx->child);
}
static struct crypto_instance *alloc(struct rtattr **tb)
{
struct crypto_instance *inst;
struct crypto_alg *alg;
int err;
err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
if (err)
return ERR_PTR(err);
alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
CRYPTO_ALG_TYPE_MASK);
if (IS_ERR(alg))
return ERR_CAST(alg);
inst = crypto_alloc_instance("lrw", alg);
if (IS_ERR(inst))
goto out_put_alg;
inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
inst->alg.cra_priority = alg->cra_priority;
inst->alg.cra_blocksize = alg->cra_blocksize;
if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7;
else inst->alg.cra_alignmask = alg->cra_alignmask;
inst->alg.cra_type = &crypto_blkcipher_type;
if (!(alg->cra_blocksize % 4))
inst->alg.cra_alignmask |= 3;
inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
inst->alg.cra_blkcipher.min_keysize =
alg->cra_cipher.cia_min_keysize + alg->cra_blocksize;
inst->alg.cra_blkcipher.max_keysize =
alg->cra_cipher.cia_max_keysize + alg->cra_blocksize;
inst->alg.cra_ctxsize = sizeof(struct priv);
inst->alg.cra_init = init_tfm;
inst->alg.cra_exit = exit_tfm;
inst->alg.cra_blkcipher.setkey = setkey;
inst->alg.cra_blkcipher.encrypt = encrypt;
inst->alg.cra_blkcipher.decrypt = decrypt;
out_put_alg:
crypto_mod_put(alg);
return inst;
}
static void free(struct crypto_instance *inst)
{
crypto_drop_spawn(crypto_instance_ctx(inst));
kfree(inst);
}
static int crypt(struct blkcipher_desc *d,
struct blkcipher_walk *w, struct priv *ctx,
void (*tw)(struct crypto_tfm *, u8 *, const u8 *),
void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
{
int err;
unsigned int avail;
const int bs = XTS_BLOCK_SIZE;
struct sinfo s = {
.tfm = crypto_cipher_tfm(ctx->child),
.fn = fn
};
u8 *wsrc;
u8 *wdst;
err = blkcipher_walk_virt(d, w);
if (!w->nbytes)
return err;
s.t = (be128 *)w->iv;
avail = w->nbytes;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
tw(crypto_cipher_tfm(ctx->tweak), w->iv, w->iv);
goto first;
for (;;) {
do {
gf128mul_x_ble(s.t, s.t);
first:
xts_round(&s, wdst, wsrc);
wsrc += bs;
wdst += bs;
} while ((avail -= bs) >= bs);
err = blkcipher_walk_done(d, w, avail);
if (!w->nbytes)
break;
avail = w->nbytes;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
}
return err;
}
static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
crypto_cipher_alg(ctx->child)->cia_encrypt);
}
static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
crypto_cipher_alg(ctx->child)->cia_decrypt);
}
int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
struct scatterlist *ssrc, unsigned int nbytes,
struct xts_crypt_req *req)
{
const unsigned int bsize = XTS_BLOCK_SIZE;
const unsigned int max_blks = req->tbuflen / bsize;
struct blkcipher_walk walk={};
unsigned int nblocks;
be128 *src, *dst, *t;
be128 *t_buf = req->tbuf;
int err, i;
BUG_ON(max_blks < 1);
blkcipher_walk_init(&walk, sdst, ssrc, nbytes);
err = blkcipher_walk_virt(desc, &walk);
nbytes = walk.nbytes;
if (!nbytes)
return err;
nblocks = min(nbytes / bsize, max_blks);
src = (be128 *)walk.src.virt.addr;
dst = (be128 *)walk.dst.virt.addr;
req->tweak_fn(req->tweak_ctx, (u8 *)&t_buf[0], walk.iv);
i = 0;
goto first;
for (;;) {
do {
for (i = 0; i < nblocks; i++) {
gf128mul_x_ble(&t_buf[i], t);
first:
t = &t_buf[i];
be128_xor(dst + i, t, src + i);
}
req->crypt_fn(req->crypt_ctx, (u8 *)dst,
nblocks * bsize);
for (i = 0; i < nblocks; i++)
be128_xor(dst + i, dst + i, &t_buf[i]);
src += nblocks;
dst += nblocks;
nbytes -= nblocks * bsize;
nblocks = min(nbytes / bsize, max_blks);
} while (nblocks > 0);
*(be128 *)walk.iv = *t;
err = blkcipher_walk_done(desc, &walk, nbytes);
nbytes = walk.nbytes;
if (!nbytes)
break;
nblocks = min(nbytes / bsize, max_blks);
src = (be128 *)walk.src.virt.addr;
dst = (be128 *)walk.dst.virt.addr;
}
return err;
}
EXPORT_SYMBOL_GPL(xts_crypt);
static int init_tfm(struct crypto_tfm *tfm)
{
struct crypto_cipher *cipher;
struct crypto_instance *inst = (void *)tfm->__crt_alg;
struct crypto_spawn *spawn = crypto_instance_ctx(inst);
struct priv *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher))
return PTR_ERR(cipher);
if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) {
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
crypto_free_cipher(cipher);
return -EINVAL;
}
ctx->child = cipher;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher)) {
crypto_free_cipher(ctx->child);
return PTR_ERR(cipher);
}
if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) {
crypto_free_cipher(cipher);
crypto_free_cipher(ctx->child);
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
return -EINVAL;
}
ctx->tweak = cipher;
return 0;
}
static void exit_tfm(struct crypto_tfm *tfm)
{
struct priv *ctx = crypto_tfm_ctx(tfm);
crypto_free_cipher(ctx->child);
crypto_free_cipher(ctx->tweak);
}
static struct crypto_instance *alloc(struct rtattr **tb)
{
struct crypto_instance *inst;
struct crypto_alg *alg;
int err;
err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
if (err)
return ERR_PTR(err);
alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
CRYPTO_ALG_TYPE_MASK);
if (IS_ERR(alg))
return ERR_CAST(alg);
inst = crypto_alloc_instance("xts", alg);
if (IS_ERR(inst))
goto out_put_alg;
inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
inst->alg.cra_priority = alg->cra_priority;
inst->alg.cra_blocksize = alg->cra_blocksize;
if (alg->cra_alignmask < 7)
inst->alg.cra_alignmask = 7;
else
inst->alg.cra_alignmask = alg->cra_alignmask;
inst->alg.cra_type = &crypto_blkcipher_type;
inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
inst->alg.cra_blkcipher.min_keysize =
2 * alg->cra_cipher.cia_min_keysize;
inst->alg.cra_blkcipher.max_keysize =
2 * alg->cra_cipher.cia_max_keysize;
inst->alg.cra_ctxsize = sizeof(struct priv);
inst->alg.cra_init = init_tfm;
inst->alg.cra_exit = exit_tfm;
inst->alg.cra_blkcipher.setkey = setkey;
inst->alg.cra_blkcipher.encrypt = encrypt;
inst->alg.cra_blkcipher.decrypt = decrypt;
out_put_alg:
crypto_mod_put(alg);
return inst;
}
static void free(struct crypto_instance *inst)
{
crypto_drop_spawn(crypto_instance_ctx(inst));
kfree(inst);
}
static int crypt(struct blkcipher_desc *d,
struct blkcipher_walk *w, struct priv *ctx,
void (*tw)(struct crypto_tfm *, u8 *, const u8 *),
void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
{
int err;
unsigned int avail;
const int bs = crypto_cipher_blocksize(ctx->child);
struct sinfo s = {
.tfm = crypto_cipher_tfm(ctx->child),
.fn = fn
};
u8 *wsrc;
u8 *wdst;
err = blkcipher_walk_virt(d, w);
if (!w->nbytes)
return err;
s.t = (be128 *)w->iv;
avail = w->nbytes;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
/* calculate first value of T */
tw(crypto_cipher_tfm(ctx->tweak), w->iv, w->iv);
goto first;
for (;;) {
do {
gf128mul_x_ble(s.t, s.t);
first:
xts_round(&s, wdst, wsrc);
wsrc += bs;
wdst += bs;
} while ((avail -= bs) >= bs);
err = blkcipher_walk_done(d, w, avail);
if (!w->nbytes)
break;
avail = w->nbytes;
wsrc = w->src.virt.addr;
wdst = w->dst.virt.addr;
}
return err;
}
static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
crypto_cipher_alg(ctx->child)->cia_encrypt);
}
static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
struct blkcipher_walk w;
blkcipher_walk_init(&w, dst, src, nbytes);
return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
crypto_cipher_alg(ctx->child)->cia_decrypt);
}
static int init_tfm(struct crypto_tfm *tfm)
{
struct crypto_cipher *cipher;
struct crypto_instance *inst = (void *)tfm->__crt_alg;
struct crypto_spawn *spawn = crypto_instance_ctx(inst);
struct priv *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher))
return PTR_ERR(cipher);
if (crypto_cipher_blocksize(cipher) != 16) {
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
crypto_free_cipher(cipher);
return -EINVAL;
}
ctx->child = cipher;
cipher = crypto_spawn_cipher(spawn);
if (IS_ERR(cipher)) {
crypto_free_cipher(ctx->child);
return PTR_ERR(cipher);
}
/* this check isn't really needed, leave it here just in case */
if (crypto_cipher_blocksize(cipher) != 16) {
crypto_free_cipher(cipher);
crypto_free_cipher(ctx->child);
*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
return -EINVAL;
}
ctx->tweak = cipher;
return 0;
}
static void exit_tfm(struct crypto_tfm *tfm)
{
struct priv *ctx = crypto_tfm_ctx(tfm);
crypto_free_cipher(ctx->child);
crypto_free_cipher(ctx->tweak);
}
static struct crypto_instance *alloc(struct rtattr **tb)
{
struct crypto_instance *inst;
struct crypto_alg *alg;
int err;
err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
if (err)
return ERR_PTR(err);
alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
CRYPTO_ALG_TYPE_MASK);
if (IS_ERR(alg))
return ERR_PTR(PTR_ERR(alg));
inst = crypto_alloc_instance("xts", alg);
if (IS_ERR(inst))
goto out_put_alg;
inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
inst->alg.cra_priority = alg->cra_priority;
inst->alg.cra_blocksize = alg->cra_blocksize;
if (alg->cra_alignmask < 7)
inst->alg.cra_alignmask = 7;
else
inst->alg.cra_alignmask = alg->cra_alignmask;
inst->alg.cra_type = &crypto_blkcipher_type;
inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
inst->alg.cra_blkcipher.min_keysize =
2 * alg->cra_cipher.cia_min_keysize;
inst->alg.cra_blkcipher.max_keysize =
2 * alg->cra_cipher.cia_max_keysize;
inst->alg.cra_ctxsize = sizeof(struct priv);
inst->alg.cra_init = init_tfm;
inst->alg.cra_exit = exit_tfm;
inst->alg.cra_blkcipher.setkey = setkey;
inst->alg.cra_blkcipher.encrypt = encrypt;
inst->alg.cra_blkcipher.decrypt = decrypt;
out_put_alg:
crypto_mod_put(alg);
return inst;
}
static void free(struct crypto_instance *inst)
{
crypto_drop_spawn(crypto_instance_ctx(inst));
kfree(inst);
}
static int crypto_gecb_crypt(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
unsigned int sz,
int enc)
{
int err=0;
unsigned int nbytes;
u8* gpos;
int i = 0;
struct kgpu_req *req;
struct kgpu_resp *resp;
struct kgpu_buffer *buf;
struct crypto_blkcipher *tfm = desc->tfm;
struct crypto_gecb_ctx *ctx = crypto_blkcipher_ctx(tfm);
struct blkcipher_walk walk;
blkcipher_walk_init(&walk, dst, src, sz);
buf = alloc_gpu_buffer();
if (!buf) {
printk("[gecb] Error: GPU buffer is null.\n");
return -EFAULT;
}
req = alloc_kgpu_request();
resp = alloc_kgpu_response();
if (!req || !resp) {
return -EFAULT;
}
err = blkcipher_walk_virt(desc, &walk);
while ((nbytes = walk.nbytes)) {
u8 *wsrc = walk.src.virt.addr;
if (nbytes > KGPU_BUF_FRAME_SIZE) {
return -EFAULT;
}
#ifndef _NDEBUG
if (nbytes != PAGE_SIZE)
printk("[gecb] WARNING: %u is not PAGE_SIZE\n", nbytes);
#endif
gpos = buf->paddrs[i++];
memcpy(__va(gpos), wsrc, nbytes);
err = blkcipher_walk_done(desc, &walk, 0);
}
gpos = buf->paddrs[i];
memcpy(__va(gpos), &(ctx->aes_ctx), sizeof(struct crypto_aes_ctx));
strcpy(req->kureq.sname, enc?"gecb-enc":"gecb-dec");
req->kureq.input = buf->gb.addr;
req->kureq.output = buf->gb.addr;
req->kureq.insize = sz+PAGE_SIZE;
req->kureq.outsize = sz;
if (call_gpu_sync(req, resp)) {
err = -EFAULT;
printk("[gecb] Error: callgpu error\n");
} else {
i=0;
blkcipher_walk_init(&walk, dst, src, sz);
err = blkcipher_walk_virt(desc, &walk);
while ((nbytes = walk.nbytes)) {
u8 *wdst = walk.dst.virt.addr;
if (nbytes > KGPU_BUF_FRAME_SIZE) {
return -EFAULT;
}
#ifndef _NDEBUG
if (nbytes != PAGE_SIZE)
printk("[gecb] WARNING: %u is not PAGE_SIZE\n", nbytes);
#endif
gpos = buf->paddrs[i++];
memcpy(wdst, __va(gpos), nbytes);
err = blkcipher_walk_done(desc, &walk, 0);
}
}
free_kgpu_request(req);
free_kgpu_response(resp);
free_gpu_buffer(buf);
return err;
}
