static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); while ((nbytes = walk.nbytes)) { nbytes = __cbc_encrypt(desc, &walk); err = blkcipher_walk_done(desc, &walk, nbytes); } return err; }
static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); int err, first, rounds = 6 + ctx->key_length / 4; struct blkcipher_walk walk; int blocks; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); first = 1; kernel_neon_begin(); while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, (u8 *)ctx->key_enc, rounds, blocks, walk.iv, first); first = 0; nbytes -= blocks * AES_BLOCK_SIZE; if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE) break; err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } if (nbytes) { u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; u8 __aligned(8) tail[AES_BLOCK_SIZE]; /* * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need * to tell aes_ctr_encrypt() to only read half a block. */ blocks = (nbytes <= 8) ? -1 : 1; aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, rounds, blocks, walk.iv, first); memcpy(tdst, tail, nbytes); err = blkcipher_walk_done(desc, &walk, 0); } kernel_neon_end(); return err; }
int glue_cbc_encrypt_128bit(const common_glue_func_t fn, struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); while ((nbytes = walk.nbytes)) { nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk); err = blkcipher_walk_done(desc, &walk, nbytes); } return err; }
static int crypto_cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct blkcipher_walk walk; struct crypto_blkcipher *tfm = desc->tfm; struct crypto_cbc_ctx *ctx = crypto_blkcipher_ctx(tfm); struct crypto_cipher *child = ctx->child; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); while ((nbytes = walk.nbytes)) { #ifdef CONFIG_CRYPTO_DEV_REALTEK_DBG printk("%s: total=%d, walk=%d, blk=%d, src=%p, dst=%p\n", __FUNCTION__, walk.total, walk.nbytes, crypto_cipher_blocksize(child), walk.src.virt.addr, walk.dst.virt.addr ); #endif #ifdef CONFIG_CRYPTO_DEV_REALTEK if (ctx->rtl_ctx.mode >= 0) { nbytes = rtl_cipher_crypt(child, 1, &ctx->rtl_ctx, walk.src.virt.addr, nbytes, walk.iv, walk.dst.virt.addr); // cbc mode update memcpy(walk.iv, walk.dst.virt.addr, crypto_cipher_blocksize(child)); err = blkcipher_walk_done(desc, &walk, nbytes); continue; } #endif if (walk.src.virt.addr == walk.dst.virt.addr) nbytes = crypto_cbc_encrypt_inplace(desc, &walk, child); else nbytes = crypto_cbc_encrypt_segment(desc, &walk, child); err = blkcipher_walk_done(desc, &walk, nbytes); } return err; }
static int aesbs_cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); while ((walk.nbytes / AES_BLOCK_SIZE) >= 8) { kernel_neon_begin(); bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes, &ctx->dec, walk.iv); kernel_neon_end(); err = blkcipher_walk_done(desc, &walk, 0); } while (walk.nbytes) { u32 blocks = walk.nbytes / AES_BLOCK_SIZE; u8 *dst = walk.dst.virt.addr; u8 *src = walk.src.virt.addr; u8 bk[2][AES_BLOCK_SIZE]; u8 *iv = walk.iv; do { if (walk.dst.virt.addr == walk.src.virt.addr) memcpy(bk[blocks & 1], src, AES_BLOCK_SIZE); AES_decrypt(src, dst, &ctx->dec.rk); crypto_xor(dst, iv, AES_BLOCK_SIZE); if (walk.dst.virt.addr == walk.src.virt.addr) iv = bk[blocks & 1]; else iv = src; dst += AES_BLOCK_SIZE; src += AES_BLOCK_SIZE; } while (--blocks); err = blkcipher_walk_done(desc, &walk, 0); } return err; }
static int skcipher_null_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); while (walk.nbytes) { if (walk.src.virt.addr != walk.dst.virt.addr) memcpy(walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes); err = blkcipher_walk_done(desc, &walk, 0); } return err; }
static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { bool fpu_enabled = false; struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; while ((nbytes = walk.nbytes)) { fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes); nbytes = __cbc_decrypt(desc, &walk); err = blkcipher_walk_done(desc, &walk, nbytes); } cast5_fpu_end(fpu_enabled); return err; }
static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE); while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) { nbytes = __ctr_crypt(desc, &walk); err = blkcipher_walk_done(desc, &walk, nbytes); } if (walk.nbytes) { ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk); err = blkcipher_walk_done(desc, &walk, 0); } return err; }
static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); int err, first, rounds = 6 + ctx->key_length / 4; struct blkcipher_walk walk; unsigned int blocks; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); kernel_neon_begin(); for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, (u8 *)ctx->key_enc, rounds, blocks, first); err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } kernel_neon_end(); return err; }
static int aesbs_cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); while (walk.nbytes) { u32 blocks = walk.nbytes / AES_BLOCK_SIZE; u8 *src = walk.src.virt.addr; if (walk.dst.virt.addr == walk.src.virt.addr) { u8 *iv = walk.iv; do { crypto_xor(src, iv, AES_BLOCK_SIZE); AES_encrypt(src, src, &ctx->enc); iv = src; src += AES_BLOCK_SIZE; } while (--blocks); memcpy(walk.iv, iv, AES_BLOCK_SIZE); } else { u8 *dst = walk.dst.virt.addr; do { crypto_xor(walk.iv, src, AES_BLOCK_SIZE); AES_encrypt(walk.iv, dst, &ctx->enc); memcpy(walk.iv, dst, AES_BLOCK_SIZE); src += AES_BLOCK_SIZE; dst += AES_BLOCK_SIZE; } while (--blocks); } err = blkcipher_walk_done(desc, &walk, 0); } return err; }
static int ecb_arc4_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct arc4_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk={}; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); while (walk.nbytes > 0) { u8 *wsrc = walk.src.virt.addr; u8 *wdst = walk.dst.virt.addr; arc4_crypt(ctx, wdst, wsrc, walk.nbytes); err = blkcipher_walk_done(desc, &walk, 0); } return err; }
static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; unsigned int blocks; int err; desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); kernel_neon_begin(); while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, (u8 *)ctx->key_dec, num_rounds(ctx), blocks); err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } kernel_neon_end(); return err; }
static int xlr_ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct xlr_crypt_state *op = crypto_blkcipher_ctx(desc->tfm); op_handle_t handle = NLMSAE_HANDLE_INITIALIZER; Crypto_Operation_pt cop=NULL; struct blkcipher_walk walk; int err, ret; if (( ret = xlr_crypt_op_init(op, &handle, &cop))) return ret; cop->m = NLM_ECB; cop->encrypt = CIPHER_ENCRYPT; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); while((nbytes = walk.nbytes)) { cop->iv_len = 0; cop->input = walk.src.virt.addr; cop->input_len = nbytes - (nbytes % op->block_size); cop->output = walk.dst.virt.addr; ret = nlmsec_cipher_digest_hmac_cksum(handle, cop); if(IS_SUCCESS_SAEOP(ret)) xlr_inc_enc_stat(cop->c, cop->m, cop->input_len); if (ret == -1) return ret; nbytes -= ret; ret = blkcipher_walk_done(desc, &walk, nbytes); } nlmsec_op_cleanup(&handle); return err; }
int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx, struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { const unsigned int bsize = 128 / 8; bool fpu_enabled = false; struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); while ((nbytes = walk.nbytes)) { fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, desc, fpu_enabled, nbytes); nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk); err = blkcipher_walk_done(desc, &walk, nbytes); } glue_fpu_end(fpu_enabled); return err; }
static int aesbs_xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); /* generate the initial tweak */ AES_encrypt(walk.iv, walk.iv, &ctx->twkey); while (walk.nbytes) { kernel_neon_begin(); bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes, &ctx->dec, walk.iv); kernel_neon_end(); err = blkcipher_walk_done(desc, &walk, 0); } return err; }
static int crypto_cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct blkcipher_walk walk; struct crypto_blkcipher *tfm = desc->tfm; struct crypto_cbc_ctx *ctx = crypto_blkcipher_ctx(tfm); struct crypto_cipher *child = ctx->child; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); while ((nbytes = walk.nbytes)) { if (walk.src.virt.addr == walk.dst.virt.addr) nbytes = crypto_cbc_encrypt_inplace(desc, &walk, child); else nbytes = crypto_cbc_encrypt_segment(desc, &walk, child); err = blkcipher_walk_done(desc, &walk, nbytes); } return err; }
static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & AES_BLOCK_MASK, walk.iv); nbytes &= AES_BLOCK_SIZE - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } kernel_fpu_end(); return err; }
/* for implementations implementing faster XTS IV generator */ int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx, struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes, void (*tweak_fn)(void *ctx, u8 *dst, const u8 *src), void *tweak_ctx, void *crypt_ctx) { const unsigned int bsize = 128 / 8; bool fpu_enabled = false; struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); nbytes = walk.nbytes; if (!nbytes) return err; /* set minimum length to bsize, for tweak_fn */ fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, desc, fpu_enabled, nbytes < bsize ? bsize : nbytes); /* calculate first value of T */ tweak_fn(tweak_ctx, walk.iv, walk.iv); while (nbytes) { nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk); err = blkcipher_walk_done(desc, &walk, nbytes); nbytes = walk.nbytes; } glue_fpu_end(fpu_enabled); return err; }
/*! \fn int ctr_rfc3686_aes_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) * \ingroup IFX_AES_FUNCTIONS * \brief Counter mode AES (rfc3686) decrypt using linux crypto blkcipher * \param desc blkcipher descriptor * \param dst output scatterlist * \param src input scatterlist * \param nbytes data size in bytes * \return err */ int ctr_rfc3686_aes_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk walk; int err; u8 rfc3686_iv[16]; #ifdef CONFIG_CRYPTO_DEV_PWR_SAVE_MODE powerup_deu(AES_INIT); #endif blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); /* set up counter block */ memcpy(rfc3686_iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE); memcpy(rfc3686_iv + CTR_RFC3686_NONCE_SIZE, walk.iv, CTR_RFC3686_IV_SIZE); /* initialize counter portion of counter block */ *(__be32 *)(rfc3686_iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); while ((nbytes = walk.nbytes)) { nbytes -= (nbytes % AES_BLOCK_SIZE); ifx_deu_aes_ctr(ctx, walk.dst.virt.addr, walk.src.virt.addr, rfc3686_iv, nbytes, CRYPTO_DIR_DECRYPT, 0); nbytes &= AES_BLOCK_SIZE - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } #ifdef CONFIG_CRYPTO_DEV_PWR_SAVE_MODE powerdown_deu(AES_INIT); #endif return err; }
static int chacha20_simd(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { u32 *state, state_buf[16 + (CHACHA20_STATE_ALIGN / sizeof(u32)) - 1]; struct blkcipher_walk walk; int err; if (!may_use_simd()) return crypto_chacha20_crypt(desc, dst, src, nbytes); state = (u32 *)roundup((uintptr_t)state_buf, CHACHA20_STATE_ALIGN); blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt_block(desc, &walk, CHACHA20_BLOCK_SIZE); crypto_chacha20_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv); kernel_fpu_begin(); while (walk.nbytes >= CHACHA20_BLOCK_SIZE) { chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr, rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE)); err = blkcipher_walk_done(desc, &walk, walk.nbytes % CHACHA20_BLOCK_SIZE); } if (walk.nbytes) { chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes); err = blkcipher_walk_done(desc, &walk, 0); } kernel_fpu_end(); return err; }
static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct blkcipher_walk walk; struct crypto_blkcipher *tfm = desc->tfm; struct salsa20_ctx *ctx = crypto_blkcipher_ctx(tfm); int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt_block(desc, &walk, 64); salsa20_ivsetup(ctx, walk.iv); if (likely(walk.nbytes == nbytes)) { salsa20_encrypt_bytes(ctx, walk.src.virt.addr, walk.dst.virt.addr, nbytes); return blkcipher_walk_done(desc, &walk, 0); } while (walk.nbytes >= 64) { salsa20_encrypt_bytes(ctx, walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes - (walk.nbytes % 64)); err = blkcipher_walk_done(desc, &walk, walk.nbytes % 64); } if (walk.nbytes) { salsa20_encrypt_bytes(ctx, walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes); err = blkcipher_walk_done(desc, &walk, 0); } return err; }
static int ccm_encrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); struct blkcipher_desc desc = { .info = req->iv }; struct blkcipher_walk walk; struct scatterlist srcbuf[2]; struct scatterlist dstbuf[2]; struct scatterlist *src; struct scatterlist *dst; u8 __aligned(8) mac[AES_BLOCK_SIZE]; u8 buf[AES_BLOCK_SIZE]; u32 len = req->cryptlen; int err; err = ccm_init_mac(req, mac, len); if (err) return err; kernel_neon_begin_partial(6); if (req->assoclen) ccm_calculate_auth_mac(req, mac); /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); src = scatterwalk_ffwd(srcbuf, req->src, req->assoclen); dst = src; if (req->src != req->dst) dst = scatterwalk_ffwd(dstbuf, req->dst, req->assoclen); blkcipher_walk_init(&walk, dst, src, len); err = blkcipher_aead_walk_virt_block(&desc, &walk, aead, AES_BLOCK_SIZE); while (walk.nbytes) { u32 tail = walk.nbytes % AES_BLOCK_SIZE; if (walk.nbytes == len) tail = 0; ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes - tail, ctx->key_enc, num_rounds(ctx), mac, walk.iv); len -= walk.nbytes - tail; err = blkcipher_walk_done(&desc, &walk, tail); } if (!err) ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); kernel_neon_end(); if (err) return err; /* copy authtag to end of dst */ scatterwalk_map_and_copy(mac, dst, req->cryptlen, crypto_aead_authsize(aead), 1); return 0; } static int ccm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); unsigned int authsize = crypto_aead_authsize(aead); struct blkcipher_desc desc = { .info = req->iv }; struct blkcipher_walk walk; struct scatterlist srcbuf[2]; struct scatterlist dstbuf[2]; struct scatterlist *src; struct scatterlist *dst; u8 __aligned(8) mac[AES_BLOCK_SIZE]; u8 buf[AES_BLOCK_SIZE]; u32 len = req->cryptlen - authsize; int err; err = ccm_init_mac(req, mac, len); if (err) return err; kernel_neon_begin_partial(6); if (req->assoclen) ccm_calculate_auth_mac(req, mac); /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); src = scatterwalk_ffwd(srcbuf, req->src, req->assoclen); dst = src; if (req->src != req->dst) dst = scatterwalk_ffwd(dstbuf, req->dst, req->assoclen); blkcipher_walk_init(&walk, dst, src, len); err = blkcipher_aead_walk_virt_block(&desc, &walk, aead, AES_BLOCK_SIZE); while (walk.nbytes) { u32 tail = walk.nbytes % AES_BLOCK_SIZE; if (walk.nbytes == len) tail = 0; ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes - tail, ctx->key_enc, num_rounds(ctx), mac, walk.iv); len -= walk.nbytes - tail; err = blkcipher_walk_done(&desc, &walk, tail); } if (!err) ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); kernel_neon_end(); if (err) return err; /* compare calculated auth tag with the stored one */ scatterwalk_map_and_copy(buf, src, req->cryptlen - authsize, authsize, 0); if (crypto_memneq(mac, buf, authsize)) return -EBADMSG; return 0; } static struct aead_alg ccm_aes_alg = { .base = { .cra_name = "ccm(aes)", .cra_driver_name = "ccm-aes-ce", .cra_flags = CRYPTO_ALG_AEAD_NEW, .cra_priority = 300, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_alignmask = 7, .cra_module = THIS_MODULE, }, .ivsize = AES_BLOCK_SIZE, .maxauthsize = AES_BLOCK_SIZE, .setkey = ccm_setkey, .setauthsize = ccm_setauthsize, .encrypt = ccm_encrypt, .decrypt = ccm_decrypt, }; static int __init aes_mod_init(void) { if (!(elf_hwcap & HWCAP_AES)) return -ENODEV; return crypto_register_aead(&ccm_aes_alg); } static void __exit aes_mod_exit(void) { crypto_unregister_aead(&ccm_aes_alg); }
static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { int ret; u64 inc; struct blkcipher_walk walk; struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); struct blkcipher_desc fallback_desc = { .tfm = ctx->fallback, .info = desc->info, .flags = desc->flags }; if (in_interrupt()) { ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes); } else { blkcipher_walk_init(&walk, dst, src, nbytes); ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { preempt_disable(); pagefault_disable(); enable_kernel_vsx(); aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, walk.dst.virt.addr, (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE, &ctx->enc_key, walk.iv); disable_kernel_vsx(); pagefault_enable(); preempt_enable(); /* We need to update IV mostly for last bytes/round */ inc = (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE; if (inc > 0) while (inc--) crypto_inc(walk.iv, AES_BLOCK_SIZE); nbytes &= AES_BLOCK_SIZE - 1; ret = blkcipher_walk_done(desc, &walk, nbytes); } if (walk.nbytes) { p8_aes_ctr_final(ctx, &walk); ret = blkcipher_walk_done(desc, &walk, 0); } } return ret; } struct crypto_alg p8_aes_ctr_alg = { .cra_name = "ctr(aes)", .cra_driver_name = "p8_aes_ctr", .cra_module = THIS_MODULE, .cra_priority = 2000, .cra_type = &crypto_blkcipher_type, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_alignmask = 0, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct p8_aes_ctr_ctx), .cra_init = p8_aes_ctr_init, .cra_exit = p8_aes_ctr_exit, .cra_blkcipher = { .ivsize = AES_BLOCK_SIZE, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = p8_aes_ctr_setkey, .encrypt = p8_aes_ctr_crypt, .decrypt = p8_aes_ctr_crypt, }, };
static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { int ret; struct blkcipher_walk walk; struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); struct blkcipher_desc fallback_desc = { .tfm = ctx->fallback, .info = desc->info, .flags = desc->flags }; if (in_interrupt()) { ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes); } else { preempt_disable(); pagefault_disable(); enable_kernel_vsx(); blkcipher_walk_init(&walk, dst, src, nbytes); ret = blkcipher_walk_virt(desc, &walk); while ((nbytes = walk.nbytes)) { aes_p8_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, nbytes & AES_BLOCK_MASK, &ctx->enc_key, walk.iv, 1); nbytes &= AES_BLOCK_SIZE - 1; ret = blkcipher_walk_done(desc, &walk, nbytes); } disable_kernel_vsx(); pagefault_enable(); preempt_enable(); } return ret; } static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { int ret; struct blkcipher_walk walk; struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); struct blkcipher_desc fallback_desc = { .tfm = ctx->fallback, .info = desc->info, .flags = desc->flags }; if (in_interrupt()) { ret = crypto_blkcipher_decrypt(&fallback_desc, dst, src, nbytes); } else { preempt_disable(); pagefault_disable(); enable_kernel_vsx(); blkcipher_walk_init(&walk, dst, src, nbytes); ret = blkcipher_walk_virt(desc, &walk); while ((nbytes = walk.nbytes)) { aes_p8_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, nbytes & AES_BLOCK_MASK, &ctx->dec_key, walk.iv, 0); nbytes &= AES_BLOCK_SIZE - 1; ret = blkcipher_walk_done(desc, &walk, nbytes); } disable_kernel_vsx(); pagefault_enable(); preempt_enable(); } return ret; } struct crypto_alg p8_aes_cbc_alg = { .cra_name = "cbc(aes)", .cra_driver_name = "p8_aes_cbc", .cra_module = THIS_MODULE, .cra_priority = 1000, .cra_type = &crypto_blkcipher_type, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_alignmask = 0, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct p8_aes_cbc_ctx), .cra_init = p8_aes_cbc_init, .cra_exit = p8_aes_cbc_exit, .cra_blkcipher = { .ivsize = AES_BLOCK_SIZE, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = p8_aes_cbc_setkey, .encrypt = p8_aes_cbc_encrypt, .decrypt = p8_aes_cbc_decrypt, }, };
static int crypto_ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct blkcipher_walk walk; struct crypto_blkcipher *tfm = desc->tfm; struct crypto_ctr_ctx *ctx = crypto_blkcipher_ctx(tfm); struct crypto_cipher *child = ctx->child; unsigned int bsize = crypto_cipher_blocksize(child); int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt_block(desc, &walk, bsize); while (walk.nbytes >= bsize) { #ifdef CONFIG_CRYPTO_DEV_REALTEK if (ctx->rtl_ctx.mode >= 0) { int i, over_flag = 0; unsigned int one = 0, len = 0; u8 over_iv[32] = {0}; u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; /* hw CTRBLK overflow handle different with linux kernel * hw engine: CTRBLK := NONCE || IV || ONE, NONCE 4 bytes, IV 8bytes, ONE 4bytes * hw engine only the ONE(4bytes) is treated as counter bytes * linux kernel uses the second method, which means the entire byte block is treated as counter bytes */ over_flag = 0; one = *((unsigned int *)(walk.iv + bsize - 4)); for (i = 0; i < (walk.nbytes / bsize); i++) { if (one == 0xffffffff) { //printk("%s %d i=%d one=%u\n", __FUNCTION__, __LINE__, i, one); over_flag = 1; break; } one++; } if (over_flag) { //before ONE overflow len = bsize*(i+1); nbytes = rtl_cipher_crypt(child, 1, &ctx->rtl_ctx, walk.src.virt.addr, len, walk.iv, walk.dst.virt.addr); //printk("%s %d len=%u nbytes=%u \n", __FUNCTION__, __LINE__, len, nbytes); src += (len - nbytes); dst += (len - nbytes); //after ONE overflow,update IV memcpy(over_iv, walk.iv, bsize - 4); crypto_inc(over_iv, bsize-4); memcpy(walk.iv, over_iv, bsize); nbytes = rtl_cipher_crypt(child, 1, &ctx->rtl_ctx, src, walk.nbytes -len, walk.iv, dst); /* increment counter in counterblock */ for (i = 0; i < ((walk.nbytes -len) / bsize); i++) crypto_inc(walk.iv, bsize); if (walk.src.virt.addr == walk.dst.virt.addr) { src += ((walk.nbytes -len) - nbytes); } else { src += ((walk.nbytes -len) - nbytes); dst += ((walk.nbytes -len) - nbytes); } } else { nbytes = rtl_cipher_crypt(child, 1, &ctx->rtl_ctx, walk.src.virt.addr, walk.nbytes, walk.iv, walk.dst.virt.addr); if (walk.src.virt.addr == walk.dst.virt.addr) { walk.src.virt.addr += (walk.nbytes - nbytes); } else { walk.dst.virt.addr += (walk.nbytes - nbytes); walk.dst.virt.addr += (walk.nbytes - nbytes); } /* increment counter in counterblock */ for (i = 0; i < (walk.nbytes / bsize); i++) crypto_inc(walk.iv, bsize); } err = blkcipher_walk_done(desc, &walk, nbytes); continue; } #endif if (walk.src.virt.addr == walk.dst.virt.addr) nbytes = crypto_ctr_crypt_inplace(&walk, child); else nbytes = crypto_ctr_crypt_segment(&walk, child); err = blkcipher_walk_done(desc, &walk, nbytes); } if (walk.nbytes) { crypto_ctr_crypt_final(&walk, child); err = blkcipher_walk_done(desc, &walk, 0); } return err; }
static int crypt(struct blkcipher_desc *d, struct blkcipher_walk *w, struct priv *ctx, void (*fn)(struct crypto_tfm *, u8 *, const u8 *)) { int err; unsigned int avail; const int bs = LRW_BLOCK_SIZE; struct sinfo s = { .tfm = crypto_cipher_tfm(ctx->child), .fn = fn }; be128 *iv; u8 *wsrc; u8 *wdst; err = blkcipher_walk_virt(d, w); if (!(avail = w->nbytes)) return err; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; /* calculate first value of T */ iv = (be128 *)w->iv; s.t = *iv; /* T <- I*Key2 */ gf128mul_64k_bbe(&s.t, ctx->table.table); goto first; for (;;) { do { /* T <- I*Key2, using the optimization * discussed in the specification */ be128_xor(&s.t, &s.t, &ctx->table.mulinc[get_index128(iv)]); inc(iv); first: lrw_round(&s, wdst, wsrc); wsrc += bs; wdst += bs; } while ((avail -= bs) >= bs); err = blkcipher_walk_done(d, w, avail); if (!(avail = w->nbytes)) break; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; } return err; } static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->child)->cia_encrypt); } static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->child)->cia_decrypt); } int lrw_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst, struct scatterlist *ssrc, unsigned int nbytes, struct lrw_crypt_req *req) { const unsigned int bsize = LRW_BLOCK_SIZE; const unsigned int max_blks = req->tbuflen / bsize; struct lrw_table_ctx *ctx = req->table_ctx; struct blkcipher_walk walk; unsigned int nblocks; be128 *iv, *src, *dst, *t; be128 *t_buf = req->tbuf; int err, i; BUG_ON(max_blks < 1); blkcipher_walk_init(&walk, sdst, ssrc, nbytes); err = blkcipher_walk_virt(desc, &walk); nbytes = walk.nbytes; if (!nbytes) return err; nblocks = min(walk.nbytes / bsize, max_blks); src = (be128 *)walk.src.virt.addr; dst = (be128 *)walk.dst.virt.addr; /* calculate first value of T */ iv = (be128 *)walk.iv; t_buf[0] = *iv; /* T <- I*Key2 */ gf128mul_64k_bbe(&t_buf[0], ctx->table); i = 0; goto first; for (;;) { do { for (i = 0; i < nblocks; i++) { /* T <- I*Key2, using the optimization * discussed in the specification */ be128_xor(&t_buf[i], t, &ctx->mulinc[get_index128(iv)]); inc(iv); first: t = &t_buf[i]; /* PP <- T xor P */ be128_xor(dst + i, t, src + i); } /* CC <- E(Key2,PP) */ req->crypt_fn(req->crypt_ctx, (u8 *)dst, nblocks * bsize); /* C <- T xor CC */ for (i = 0; i < nblocks; i++) be128_xor(dst + i, dst + i, &t_buf[i]); src += nblocks; dst += nblocks; nbytes -= nblocks * bsize; nblocks = min(nbytes / bsize, max_blks); } while (nblocks > 0); err = blkcipher_walk_done(desc, &walk, nbytes); nbytes = walk.nbytes; if (!nbytes) break; nblocks = min(nbytes / bsize, max_blks); src = (be128 *)walk.src.virt.addr; dst = (be128 *)walk.dst.virt.addr; } return err; } EXPORT_SYMBOL_GPL(lrw_crypt); static int init_tfm(struct crypto_tfm *tfm) { struct crypto_cipher *cipher; struct crypto_instance *inst = (void *)tfm->__crt_alg; struct crypto_spawn *spawn = crypto_instance_ctx(inst); struct priv *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); if (crypto_cipher_blocksize(cipher) != LRW_BLOCK_SIZE) { *flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; crypto_free_cipher(cipher); return -EINVAL; } ctx->child = cipher; return 0; } static void exit_tfm(struct crypto_tfm *tfm) { struct priv *ctx = crypto_tfm_ctx(tfm); lrw_free_table(&ctx->table); crypto_free_cipher(ctx->child); } static struct crypto_instance *alloc(struct rtattr **tb) { struct crypto_instance *inst; struct crypto_alg *alg; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER); if (err) return ERR_PTR(err); alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) return ERR_CAST(alg); inst = crypto_alloc_instance("lrw", alg); if (IS_ERR(inst)) goto out_put_alg; inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER; inst->alg.cra_priority = alg->cra_priority; inst->alg.cra_blocksize = alg->cra_blocksize; if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7; else inst->alg.cra_alignmask = alg->cra_alignmask; inst->alg.cra_type = &crypto_blkcipher_type; if (!(alg->cra_blocksize % 4)) inst->alg.cra_alignmask |= 3; inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize; inst->alg.cra_blkcipher.min_keysize = alg->cra_cipher.cia_min_keysize + alg->cra_blocksize; inst->alg.cra_blkcipher.max_keysize = alg->cra_cipher.cia_max_keysize + alg->cra_blocksize; inst->alg.cra_ctxsize = sizeof(struct priv); inst->alg.cra_init = init_tfm; inst->alg.cra_exit = exit_tfm; inst->alg.cra_blkcipher.setkey = setkey; inst->alg.cra_blkcipher.encrypt = encrypt; inst->alg.cra_blkcipher.decrypt = decrypt; out_put_alg: crypto_mod_put(alg); return inst; } static void free(struct crypto_instance *inst) { crypto_drop_spawn(crypto_instance_ctx(inst)); kfree(inst); }
static int crypt(struct blkcipher_desc *d, struct blkcipher_walk *w, struct priv *ctx, void (*tw)(struct crypto_tfm *, u8 *, const u8 *), void (*fn)(struct crypto_tfm *, u8 *, const u8 *)) { int err; unsigned int avail; const int bs = crypto_cipher_blocksize(ctx->child); struct sinfo s = { .tfm = crypto_cipher_tfm(ctx->child), .fn = fn }; u8 *wsrc; u8 *wdst; err = blkcipher_walk_virt(d, w); if (!w->nbytes) return err; s.t = (be128 *)w->iv; avail = w->nbytes; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; /* calculate first value of T */ tw(crypto_cipher_tfm(ctx->tweak), w->iv, w->iv); goto first; for (;;) { do { gf128mul_x_ble(s.t, s.t); first: xts_round(&s, wdst, wsrc); wsrc += bs; wdst += bs; } while ((avail -= bs) >= bs); err = blkcipher_walk_done(d, w, avail); if (!w->nbytes) break; avail = w->nbytes; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; } return err; } static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt, crypto_cipher_alg(ctx->child)->cia_encrypt); } static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt, crypto_cipher_alg(ctx->child)->cia_decrypt); } static int init_tfm(struct crypto_tfm *tfm) { struct crypto_cipher *cipher; struct crypto_instance *inst = (void *)tfm->__crt_alg; struct crypto_spawn *spawn = crypto_instance_ctx(inst); struct priv *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); if (crypto_cipher_blocksize(cipher) != 16) { *flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; crypto_free_cipher(cipher); return -EINVAL; } ctx->child = cipher; cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) { crypto_free_cipher(ctx->child); return PTR_ERR(cipher); } /* this check isn't really needed, leave it here just in case */ if (crypto_cipher_blocksize(cipher) != 16) { crypto_free_cipher(cipher); crypto_free_cipher(ctx->child); *flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; return -EINVAL; } ctx->tweak = cipher; return 0; } static void exit_tfm(struct crypto_tfm *tfm) { struct priv *ctx = crypto_tfm_ctx(tfm); crypto_free_cipher(ctx->child); crypto_free_cipher(ctx->tweak); } static struct crypto_instance *alloc(struct rtattr **tb) { struct crypto_instance *inst; struct crypto_alg *alg; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER); if (err) return ERR_PTR(err); alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) return ERR_PTR(PTR_ERR(alg)); inst = crypto_alloc_instance("xts", alg); if (IS_ERR(inst)) goto out_put_alg; inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER; inst->alg.cra_priority = alg->cra_priority; inst->alg.cra_blocksize = alg->cra_blocksize; if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7; else inst->alg.cra_alignmask = alg->cra_alignmask; inst->alg.cra_type = &crypto_blkcipher_type; inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize; inst->alg.cra_blkcipher.min_keysize = 2 * alg->cra_cipher.cia_min_keysize; inst->alg.cra_blkcipher.max_keysize = 2 * alg->cra_cipher.cia_max_keysize; inst->alg.cra_ctxsize = sizeof(struct priv); inst->alg.cra_init = init_tfm; inst->alg.cra_exit = exit_tfm; inst->alg.cra_blkcipher.setkey = setkey; inst->alg.cra_blkcipher.encrypt = encrypt; inst->alg.cra_blkcipher.decrypt = decrypt; out_put_alg: crypto_mod_put(alg); return inst; } static void free(struct crypto_instance *inst) { crypto_drop_spawn(crypto_instance_ctx(inst)); kfree(inst); }
static int crypt(struct blkcipher_desc *d, struct blkcipher_walk *w, struct priv *ctx, void (*tw)(struct crypto_tfm *, u8 *, const u8 *), void (*fn)(struct crypto_tfm *, u8 *, const u8 *)) { int err; unsigned int avail; const int bs = XTS_BLOCK_SIZE; struct sinfo s = { .tfm = crypto_cipher_tfm(ctx->child), .fn = fn }; u8 *wsrc; u8 *wdst; err = blkcipher_walk_virt(d, w); if (!w->nbytes) return err; s.t = (be128 *)w->iv; avail = w->nbytes; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; tw(crypto_cipher_tfm(ctx->tweak), w->iv, w->iv); goto first; for (;;) { do { gf128mul_x_ble(s.t, s.t); first: xts_round(&s, wdst, wsrc); wsrc += bs; wdst += bs; } while ((avail -= bs) >= bs); err = blkcipher_walk_done(d, w, avail); if (!w->nbytes) break; avail = w->nbytes; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; } return err; } static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt, crypto_cipher_alg(ctx->child)->cia_encrypt); } static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt, crypto_cipher_alg(ctx->child)->cia_decrypt); } int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst, struct scatterlist *ssrc, unsigned int nbytes, struct xts_crypt_req *req) { const unsigned int bsize = XTS_BLOCK_SIZE; const unsigned int max_blks = req->tbuflen / bsize; struct blkcipher_walk walk={}; unsigned int nblocks; be128 *src, *dst, *t; be128 *t_buf = req->tbuf; int err, i; BUG_ON(max_blks < 1); blkcipher_walk_init(&walk, sdst, ssrc, nbytes); err = blkcipher_walk_virt(desc, &walk); nbytes = walk.nbytes; if (!nbytes) return err; nblocks = min(nbytes / bsize, max_blks); src = (be128 *)walk.src.virt.addr; dst = (be128 *)walk.dst.virt.addr; req->tweak_fn(req->tweak_ctx, (u8 *)&t_buf[0], walk.iv); i = 0; goto first; for (;;) { do { for (i = 0; i < nblocks; i++) { gf128mul_x_ble(&t_buf[i], t); first: t = &t_buf[i]; be128_xor(dst + i, t, src + i); } req->crypt_fn(req->crypt_ctx, (u8 *)dst, nblocks * bsize); for (i = 0; i < nblocks; i++) be128_xor(dst + i, dst + i, &t_buf[i]); src += nblocks; dst += nblocks; nbytes -= nblocks * bsize; nblocks = min(nbytes / bsize, max_blks); } while (nblocks > 0); *(be128 *)walk.iv = *t; err = blkcipher_walk_done(desc, &walk, nbytes); nbytes = walk.nbytes; if (!nbytes) break; nblocks = min(nbytes / bsize, max_blks); src = (be128 *)walk.src.virt.addr; dst = (be128 *)walk.dst.virt.addr; } return err; } EXPORT_SYMBOL_GPL(xts_crypt); static int init_tfm(struct crypto_tfm *tfm) { struct crypto_cipher *cipher; struct crypto_instance *inst = (void *)tfm->__crt_alg; struct crypto_spawn *spawn = crypto_instance_ctx(inst); struct priv *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) { *flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; crypto_free_cipher(cipher); return -EINVAL; } ctx->child = cipher; cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) { crypto_free_cipher(ctx->child); return PTR_ERR(cipher); } if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) { crypto_free_cipher(cipher); crypto_free_cipher(ctx->child); *flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; return -EINVAL; } ctx->tweak = cipher; return 0; } static void exit_tfm(struct crypto_tfm *tfm) { struct priv *ctx = crypto_tfm_ctx(tfm); crypto_free_cipher(ctx->child); crypto_free_cipher(ctx->tweak); } static struct crypto_instance *alloc(struct rtattr **tb) { struct crypto_instance *inst; struct crypto_alg *alg; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER); if (err) return ERR_PTR(err); alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) return ERR_CAST(alg); inst = crypto_alloc_instance("xts", alg); if (IS_ERR(inst)) goto out_put_alg; inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER; inst->alg.cra_priority = alg->cra_priority; inst->alg.cra_blocksize = alg->cra_blocksize; if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7; else inst->alg.cra_alignmask = alg->cra_alignmask; inst->alg.cra_type = &crypto_blkcipher_type; inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize; inst->alg.cra_blkcipher.min_keysize = 2 * alg->cra_cipher.cia_min_keysize; inst->alg.cra_blkcipher.max_keysize = 2 * alg->cra_cipher.cia_max_keysize; inst->alg.cra_ctxsize = sizeof(struct priv); inst->alg.cra_init = init_tfm; inst->alg.cra_exit = exit_tfm; inst->alg.cra_blkcipher.setkey = setkey; inst->alg.cra_blkcipher.encrypt = encrypt; inst->alg.cra_blkcipher.decrypt = decrypt; out_put_alg: crypto_mod_put(alg); return inst; } static void free(struct crypto_instance *inst) { crypto_drop_spawn(crypto_instance_ctx(inst)); kfree(inst); }
static int crypto_gecb_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int sz, int enc) { int err=0; unsigned int nbytes; u8* gpos; int i = 0; struct kgpu_req *req; struct kgpu_resp *resp; struct kgpu_buffer *buf; struct crypto_blkcipher *tfm = desc->tfm; struct crypto_gecb_ctx *ctx = crypto_blkcipher_ctx(tfm); struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, sz); buf = alloc_gpu_buffer(); if (!buf) { printk("[gecb] Error: GPU buffer is null.\n"); return -EFAULT; } req = alloc_kgpu_request(); resp = alloc_kgpu_response(); if (!req || !resp) { return -EFAULT; } err = blkcipher_walk_virt(desc, &walk); while ((nbytes = walk.nbytes)) { u8 *wsrc = walk.src.virt.addr; if (nbytes > KGPU_BUF_FRAME_SIZE) { return -EFAULT; } #ifndef _NDEBUG if (nbytes != PAGE_SIZE) printk("[gecb] WARNING: %u is not PAGE_SIZE\n", nbytes); #endif gpos = buf->paddrs[i++]; memcpy(__va(gpos), wsrc, nbytes); err = blkcipher_walk_done(desc, &walk, 0); } gpos = buf->paddrs[i]; memcpy(__va(gpos), &(ctx->aes_ctx), sizeof(struct crypto_aes_ctx)); strcpy(req->kureq.sname, enc?"gecb-enc":"gecb-dec"); req->kureq.input = buf->gb.addr; req->kureq.output = buf->gb.addr; req->kureq.insize = sz+PAGE_SIZE; req->kureq.outsize = sz; if (call_gpu_sync(req, resp)) { err = -EFAULT; printk("[gecb] Error: callgpu error\n"); } else { i=0; blkcipher_walk_init(&walk, dst, src, sz); err = blkcipher_walk_virt(desc, &walk); while ((nbytes = walk.nbytes)) { u8 *wdst = walk.dst.virt.addr; if (nbytes > KGPU_BUF_FRAME_SIZE) { return -EFAULT; } #ifndef _NDEBUG if (nbytes != PAGE_SIZE) printk("[gecb] WARNING: %u is not PAGE_SIZE\n", nbytes); #endif gpos = buf->paddrs[i++]; memcpy(wdst, __va(gpos), nbytes); err = blkcipher_walk_done(desc, &walk, 0); } } free_kgpu_request(req); free_kgpu_response(resp); free_gpu_buffer(buf); return err; }
static int crypt(struct blkcipher_desc *d, struct blkcipher_walk *w, struct priv *ctx, void (*fn)(struct crypto_tfm *, u8 *, const u8 *)) { int err; unsigned int avail; const int bs = crypto_cipher_blocksize(ctx->child); struct sinfo s = { .tfm = crypto_cipher_tfm(ctx->child), .fn = fn }; be128 *iv; u8 *wsrc; u8 *wdst; err = blkcipher_walk_virt(d, w); if (!(avail = w->nbytes)) return err; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; /* calculate first value of T */ iv = (be128 *)w->iv; s.t = *iv; /* T <- I*Key2 */ gf128mul_64k_bbe(&s.t, ctx->table); goto first; for (;;) { do { /* T <- I*Key2, using the optimization * discussed in the specification */ be128_xor(&s.t, &s.t, &ctx->mulinc[get_index128(iv)]); inc(iv); first: lrw_round(&s, wdst, wsrc); wsrc += bs; wdst += bs; } while ((avail -= bs) >= bs); err = blkcipher_walk_done(d, w, avail); if (!(avail = w->nbytes)) break; wsrc = w->src.virt.addr; wdst = w->dst.virt.addr; } return err; } static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->child)->cia_encrypt); } static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct priv *ctx = crypto_blkcipher_ctx(desc->tfm); struct blkcipher_walk w; blkcipher_walk_init(&w, dst, src, nbytes); return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->child)->cia_decrypt); } static int init_tfm(struct crypto_tfm *tfm) { struct crypto_cipher *cipher; struct crypto_instance *inst = (void *)tfm->__crt_alg; struct crypto_spawn *spawn = crypto_instance_ctx(inst); struct priv *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); if (crypto_cipher_blocksize(cipher) != 16) { *flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; return -EINVAL; } ctx->child = cipher; return 0; } static void exit_tfm(struct crypto_tfm *tfm) { struct priv *ctx = crypto_tfm_ctx(tfm); if (ctx->table) gf128mul_free_64k(ctx->table); crypto_free_cipher(ctx->child); } static struct crypto_instance *alloc(struct rtattr **tb) { struct crypto_instance *inst; struct crypto_alg *alg; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER); if (err) return ERR_PTR(err); alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) return ERR_CAST(alg); inst = crypto_alloc_instance("lrw", alg); if (IS_ERR(inst)) goto out_put_alg; inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER; inst->alg.cra_priority = alg->cra_priority; inst->alg.cra_blocksize = alg->cra_blocksize; if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7; else inst->alg.cra_alignmask = alg->cra_alignmask; inst->alg.cra_type = &crypto_blkcipher_type; if (!(alg->cra_blocksize % 4)) inst->alg.cra_alignmask |= 3; inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize; inst->alg.cra_blkcipher.min_keysize = alg->cra_cipher.cia_min_keysize + alg->cra_blocksize; inst->alg.cra_blkcipher.max_keysize = alg->cra_cipher.cia_max_keysize + alg->cra_blocksize; inst->alg.cra_ctxsize = sizeof(struct priv); inst->alg.cra_init = init_tfm; inst->alg.cra_exit = exit_tfm; inst->alg.cra_blkcipher.setkey = setkey; inst->alg.cra_blkcipher.encrypt = encrypt; inst->alg.cra_blkcipher.decrypt = decrypt; out_put_alg: crypto_mod_put(alg); return inst; } static void free(struct crypto_instance *inst) { crypto_drop_spawn(crypto_instance_ctx(inst)); kfree(inst); }