static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx, struct scatterlist *sg, int len) { struct scatter_walk walk; u8 *src; int n; if (!len) return; scatterwalk_start(&walk, sg); while (len) { n = scatterwalk_clamp(&walk, len); if (!n) { scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg)); n = scatterwalk_clamp(&walk, len); } src = scatterwalk_map(&walk, 0); crypto_gcm_ghash_update(ctx, src, n); len -= n; scatterwalk_unmap(src, 0); scatterwalk_advance(&walk, n); scatterwalk_done(&walk, 0, len); if (len) crypto_yield(ctx->flags); } }
static int blkcipher_walk_next(struct blkcipher_desc *desc, struct blkcipher_walk *walk) { struct crypto_blkcipher *tfm = desc->tfm; unsigned int alignmask = crypto_blkcipher_alignmask(tfm); unsigned int bsize; unsigned int n; int err; n = walk->total; if (unlikely(n < crypto_blkcipher_blocksize(tfm))) { desc->flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; return blkcipher_walk_done(desc, walk, -EINVAL); } walk->flags &= ~(BLKCIPHER_WALK_SLOW | BLKCIPHER_WALK_COPY | BLKCIPHER_WALK_DIFF); if (!scatterwalk_aligned(&walk->in, alignmask) || !scatterwalk_aligned(&walk->out, alignmask) || (walk->flags & AMBA_CIPHER_COPY)) { walk->flags |= BLKCIPHER_WALK_COPY; if (!walk->page) { walk->page = (void *)__get_free_page(GFP_ATOMIC); if (!walk->page) n = 0; } } bsize = min(walk->blocksize, n); n = scatterwalk_clamp(&walk->in, n); n = scatterwalk_clamp(&walk->out, n); if (unlikely(n < bsize)) { err = blkcipher_next_slow(desc, walk, bsize, alignmask); goto set_phys_lowmem; } walk->nbytes = n; if (walk->flags & BLKCIPHER_WALK_COPY) { err = blkcipher_next_copy(walk); goto set_phys_lowmem; } return blkcipher_next_fast(desc, walk); set_phys_lowmem: if (walk->flags & BLKCIPHER_WALK_PHYS) { walk->src.phys.page = virt_to_page(walk->src.virt.addr); walk->dst.phys.page = virt_to_page(walk->dst.virt.addr); walk->src.phys.offset &= PAGE_SIZE - 1; walk->dst.phys.offset &= PAGE_SIZE - 1; } return err; }
/** * nx_walk_and_build - walk a linux scatterlist and build an nx scatterlist * * @nx_dst: pointer to the first nx_sg element to write * @sglen: max number of nx_sg entries we're allowed to write * @sg_src: pointer to the source linux scatterlist to walk * @start: number of bytes to fast-forward past at the beginning of @sg_src * @src_len: number of bytes to walk in @sg_src */ struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst, unsigned int sglen, struct scatterlist *sg_src, unsigned int start, unsigned int src_len) { struct scatter_walk walk; struct nx_sg *nx_sg = nx_dst; unsigned int n, offset = 0, len = src_len; char *dst; /* we need to fast forward through @start bytes first */ for (;;) { scatterwalk_start(&walk, sg_src); if (start < offset + sg_src->length) break; offset += sg_src->length; sg_src = scatterwalk_sg_next(sg_src); } /* start - offset is the number of bytes to advance in the scatterlist * element we're currently looking at */ scatterwalk_advance(&walk, start - offset); while (len && nx_sg) { n = scatterwalk_clamp(&walk, len); if (!n) { scatterwalk_start(&walk, sg_next(walk.sg)); n = scatterwalk_clamp(&walk, len); } dst = scatterwalk_map(&walk); nx_sg = nx_build_sg_list(nx_sg, dst, n, sglen); len -= n; scatterwalk_unmap(dst); scatterwalk_advance(&walk, n); scatterwalk_done(&walk, SCATTERWALK_FROM_SG, len); } /* return the moved destination pointer */ return nx_sg; }
static void get_data_to_compute(struct crypto_cipher *tfm, struct crypto_ccm_req_priv_ctx *pctx, struct scatterlist *sg, unsigned int len) { struct scatter_walk walk; u8 *data_src; int n; scatterwalk_start(&walk, sg); while (len) { n = scatterwalk_clamp(&walk, len); if (!n) { scatterwalk_start(&walk, sg_next(walk.sg)); n = scatterwalk_clamp(&walk, len); } data_src = scatterwalk_map(&walk); compute_mac(tfm, data_src, n, pctx); len -= n; scatterwalk_unmap(data_src); scatterwalk_advance(&walk, n); scatterwalk_done(&walk, 0, len); if (len) crypto_yield(pctx->flags); } /* any leftover needs padding and then encrypted */ if (pctx->ilen) { int padlen; u8 *odata = pctx->odata; u8 *idata = pctx->idata; padlen = 16 - pctx->ilen; memset(idata + pctx->ilen, 0, padlen); crypto_xor(odata, idata, 16); crypto_cipher_encrypt_one(tfm, odata, odata); pctx->ilen = 0; } }
static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead); struct __packed { __be16 l; __be32 h; u16 len; } ltag; struct scatter_walk walk; u32 len = req->assoclen; u32 macp = 0; /* prepend the AAD with a length tag */ if (len < 0xff00) { ltag.l = cpu_to_be16(len); ltag.len = 2; } else { ltag.l = cpu_to_be16(0xfffe); put_unaligned_be32(len, <ag.h); ltag.len = 6; } ce_aes_ccm_auth_data(mac, (u8 *)<ag, ltag.len, &macp, ctx->key_enc, num_rounds(ctx)); scatterwalk_start(&walk, req->src); do { u32 n = scatterwalk_clamp(&walk, len); u8 *p; if (!n) { scatterwalk_start(&walk, sg_next(walk.sg)); n = scatterwalk_clamp(&walk, len); } p = scatterwalk_map(&walk); ce_aes_ccm_auth_data(mac, p, n, &macp, ctx->key_enc, num_rounds(ctx)); len -= n; scatterwalk_unmap(p); scatterwalk_advance(&walk, n); scatterwalk_done(&walk, 0, len); } while (len); }
static void crypto_aegis256_aesni_process_ad( struct aegis_state *state, struct scatterlist *sg_src, unsigned int assoclen) { struct scatter_walk walk; struct aegis_block buf; unsigned int pos = 0; scatterwalk_start(&walk, sg_src); while (assoclen != 0) { unsigned int size = scatterwalk_clamp(&walk, assoclen); unsigned int left = size; void *mapped = scatterwalk_map(&walk); const u8 *src = (const u8 *)mapped; if (pos + size >= AEGIS256_BLOCK_SIZE) { if (pos > 0) { unsigned int fill = AEGIS256_BLOCK_SIZE - pos; memcpy(buf.bytes + pos, src, fill); crypto_aegis256_aesni_ad(state, AEGIS256_BLOCK_SIZE, buf.bytes); pos = 0; left -= fill; src += fill; } crypto_aegis256_aesni_ad(state, left, src); src += left & ~(AEGIS256_BLOCK_SIZE - 1); left &= AEGIS256_BLOCK_SIZE - 1; } memcpy(buf.bytes + pos, src, left); pos += left; assoclen -= size; scatterwalk_unmap(mapped); scatterwalk_advance(&walk, size); scatterwalk_done(&walk, 0, assoclen); } if (pos > 0) { memset(buf.bytes + pos, 0, AEGIS256_BLOCK_SIZE - pos); crypto_aegis256_aesni_ad(state, AEGIS256_BLOCK_SIZE, buf.bytes); } }
/* * Generic encrypt/decrypt wrapper for ciphers, handles operations across * multiple page boundaries by using temporary blocks. In user context, * the kernel is given a chance to schedule us once per page. */ static int crypt(const struct cipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct scatter_walk walk_in, walk_out; struct crypto_tfm *tfm = desc->tfm; const unsigned int bsize = crypto_tfm_alg_blocksize(tfm); unsigned int alignmask = crypto_tfm_alg_alignmask(tfm); unsigned long buffer = 0; if (!nbytes) return 0; if (nbytes % bsize) { tfm->crt_flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; return -EINVAL; } scatterwalk_start(&walk_in, src); scatterwalk_start(&walk_out, dst); for(;;) { unsigned int n = nbytes; u8 *tmp = NULL; if (!scatterwalk_aligned(&walk_in, alignmask) || !scatterwalk_aligned(&walk_out, alignmask)) { if (!buffer) { buffer = __get_free_page(GFP_ATOMIC); if (!buffer) n = 0; } tmp = (u8 *)buffer; } scatterwalk_map(&walk_in, 0); scatterwalk_map(&walk_out, 1); n = scatterwalk_clamp(&walk_in, n); n = scatterwalk_clamp(&walk_out, n); if (likely(n >= bsize)) n = crypt_fast(desc, &walk_in, &walk_out, n, tmp); else n = crypt_slow(desc, &walk_in, &walk_out, bsize); nbytes -= n; scatterwalk_done(&walk_in, 0, nbytes); scatterwalk_done(&walk_out, 1, nbytes); if (!nbytes) break; crypto_yield(tfm); } if (buffer) free_page(buffer); return 0; }