Beispiel #1
0
static int chacha20_neon(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	u32 state[16];
	int err;

	if (!may_use_simd() || req->cryptlen <= CHACHA20_BLOCK_SIZE)
		return crypto_chacha20_crypt(req);

	err = skcipher_walk_virt(&walk, req, true);

	crypto_chacha20_init(state, ctx, walk.iv);

	kernel_neon_begin();
	while (walk.nbytes > 0) {
		unsigned int nbytes = walk.nbytes;

		if (nbytes < walk.total)
			nbytes = round_down(nbytes, walk.stride);

		chacha20_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
				nbytes);
		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
	}
	kernel_neon_end();

	return err;
}
static int chacha20_simd(struct blkcipher_desc *desc, struct scatterlist *dst,
			 struct scatterlist *src, unsigned int nbytes)
{
	struct blkcipher_walk walk;
	u32 state[16];
	int err;

	if (nbytes <= CHACHA20_BLOCK_SIZE || !may_use_simd())
		return crypto_chacha20_crypt(desc, dst, src, nbytes);

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt_block(desc, &walk, CHACHA20_BLOCK_SIZE);

	crypto_chacha20_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv);

	kernel_neon_begin();

	while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
		chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
				rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE));
		err = blkcipher_walk_done(desc, &walk,
					  walk.nbytes % CHACHA20_BLOCK_SIZE);
	}

	if (walk.nbytes) {
		chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
				walk.nbytes);
		err = blkcipher_walk_done(desc, &walk, 0);
	}

	kernel_neon_end();

	return err;
}
static int chacha20_simd(struct skcipher_request *req)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
	u32 *state, state_buf[16 + 2] __aligned(8);
	struct skcipher_walk walk;
	int err;

	BUILD_BUG_ON(CHACHA20_STATE_ALIGN != 16);
	state = PTR_ALIGN(state_buf + 0, CHACHA20_STATE_ALIGN);

	if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd())
		return crypto_chacha20_crypt(req);

	err = skcipher_walk_virt(&walk, req, true);

	crypto_chacha20_init(state, ctx, walk.iv);

	kernel_fpu_begin();

	while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
		chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
				rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE));
		err = skcipher_walk_done(&walk,
					 walk.nbytes % CHACHA20_BLOCK_SIZE);
	}

	if (walk.nbytes) {
		chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
				walk.nbytes);
		err = skcipher_walk_done(&walk, 0);
	}

	kernel_fpu_end();

	return err;
}
Beispiel #4
0
static int chacha20_simd(struct blkcipher_desc *desc, struct scatterlist *dst,
                         struct scatterlist *src, unsigned int nbytes)
{
    u32 *state, state_buf[16 + (CHACHA20_STATE_ALIGN / sizeof(u32)) - 1];
    struct blkcipher_walk walk;
    int err;

    if (!may_use_simd())
        return crypto_chacha20_crypt(desc, dst, src, nbytes);

    state = (u32 *)roundup((uintptr_t)state_buf, CHACHA20_STATE_ALIGN);

    blkcipher_walk_init(&walk, dst, src, nbytes);
    err = blkcipher_walk_virt_block(desc, &walk, CHACHA20_BLOCK_SIZE);

    crypto_chacha20_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv);

    kernel_fpu_begin();

    while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
        chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
                        rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE));
        err = blkcipher_walk_done(desc, &walk,
                                  walk.nbytes % CHACHA20_BLOCK_SIZE);
    }

    if (walk.nbytes) {
        chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
                        walk.nbytes);
        err = blkcipher_walk_done(desc, &walk, 0);
    }

    kernel_fpu_end();

    return err;
}