/** * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor * (non-protocol). * @desc: pointer to buffer used for descriptor construction * @cdata: pointer to block cipher transform definitions * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. * @icvsize: integrity check value (ICV) size (truncated or full) */ void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, unsigned int icvsize) { u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; init_sh_desc(desc, HDR_SHARE_SERIAL); /* Skip key loading if it is loaded due to sharing */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); if (cdata->key_inline) append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); else append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); set_jump_tgt_here(desc, key_jump_cmd); /* Class 1 operation */ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); /* assoclen + cryptlen = seqinlen */ append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); /* * MOVE_LEN opcode is not available in all SEC HW revisions, * thus need to do some magic, i.e. self-patch the descriptor * buffer. */ read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | (0x6 << MOVE_LEN_SHIFT)); write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | (0x8 << MOVE_LEN_SHIFT)); /* Will read assoclen + cryptlen bytes */ append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); /* Will write assoclen + cryptlen bytes */ append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); /* Read and write assoclen + cryptlen bytes */ aead_append_src_dst(desc, FIFOLD_TYPE_AAD); set_move_tgt_here(desc, read_move_cmd); set_move_tgt_here(desc, write_move_cmd); append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); /* Move payload data to OFIFO */ append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); /* Write ICV */ append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); #ifdef DEBUG print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif }
/** * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor * @desc: pointer to buffer used for descriptor construction * @cdata: pointer to block cipher transform definitions * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. * @ivsize: initialization vector size * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @ctx1_iv_off: IV offset in CONTEXT1 register */ void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata, unsigned int ivsize, const bool is_rfc3686, const u32 ctx1_iv_off) { u32 *key_jump_cmd; init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); /* Load class1 key only */ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); /* Load nonce into CONTEXT1 reg */ if (is_rfc3686) { u8 *nonce = cdata->key_virt + cdata->keylen; append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, LDST_CLASS_IND_CCB | LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); } set_jump_tgt_here(desc, key_jump_cmd); /* load IV */ append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); /* Load counter into CONTEXT1 reg */ if (is_rfc3686) append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << LDST_OFFSET_SHIFT)); /* Choose operation */ if (ctx1_iv_off) append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); else append_dec_op1(desc, cdata->algtype); /* Perform operation */ ablkcipher_append_src_dst(desc); #ifdef DEBUG print_hex_dump(KERN_ERR, "ablkcipher dec shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif }
/** * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared * descriptor * @desc: pointer to buffer used for descriptor construction * @cdata: pointer to block cipher transform definitions * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. */ void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata) { __be64 sector_size = cpu_to_be64(512); u32 *key_jump_cmd; init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); /* Load class1 keys only */ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); /* Load sector size with index 40 bytes (0x28) */ append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | (0x28 << LDST_OFFSET_SHIFT)); set_jump_tgt_here(desc, key_jump_cmd); /* * create sequence for loading the sector index * Upper 8B of IV - will be used as sector index * Lower 8B of IV - will be discarded */ append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT)); append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); /* Load operation */ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); /* Perform operation */ ablkcipher_append_src_dst(desc); #ifdef DEBUG print_hex_dump(KERN_ERR, "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif }
/* get a split ipad/opad key Split key generation----------------------------------------------- [00] 0xb0810008 jobdesc: stidx=1 share=never len=8 [01] 0x04000014 key: class2->keyreg len=20 @0xffe01000 [03] 0x84410014 operation: cls2-op sha1 hmac init dec [04] 0x24940000 fifold: class2 msgdata-last2 len=0 imm [05] 0xa4000001 jump: class2 local all ->1 [06] [06] 0x64260028 fifostr: class2 mdsplit-jdk len=40 @0xffe04000 */ int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len, int split_key_pad_len, const u8 *key_in, u32 keylen, u32 alg_op) { u32 *desc; struct split_key_result result; dma_addr_t dma_addr_in, dma_addr_out; int ret = -ENOMEM; desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); if (!desc) { dev_err(jrdev, "unable to allocate key input memory\n"); return ret; } dma_addr_in = dma_map_single(jrdev, (void *)key_in, keylen, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, dma_addr_in)) { dev_err(jrdev, "unable to map key input memory\n"); goto out_free; } dma_addr_out = dma_map_single(jrdev, key_out, split_key_pad_len, DMA_FROM_DEVICE); if (dma_mapping_error(jrdev, dma_addr_out)) { dev_err(jrdev, "unable to map key output memory\n"); goto out_unmap_in; } init_job_desc(desc, 0); append_key(desc, dma_addr_in, keylen, CLASS_2 | KEY_DEST_CLASS_REG); /* Sets MDHA up into an HMAC-INIT */ append_operation(desc, alg_op | OP_ALG_DECRYPT | OP_ALG_AS_INIT); /* * do a FIFO_LOAD of zero, this will trigger the internal key expansion * into both pads inside MDHA */ append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2); /* * FIFO_STORE with the explicit split-key content store * (0x26 output type) */ append_fifo_store(desc, dma_addr_out, split_key_len, LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif result.err = 0; init_completion(&result.completion); ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); if (!ret) { /* in progress */ wait_for_completion_interruptible(&result.completion); ret = result.err; #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key_out, split_key_pad_len, 1); #endif } dma_unmap_single(jrdev, dma_addr_out, split_key_pad_len, DMA_FROM_DEVICE); out_unmap_in: dma_unmap_single(jrdev, dma_addr_in, keylen, DMA_TO_DEVICE); out_free: kfree(desc); return ret; }
struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, int *cpu, u32 *sh_desc) { size_t size; u32 num_words; dma_addr_t hwdesc; struct caam_drv_ctx *drv_ctx; const cpumask_t *cpus = qman_affine_cpus(); static DEFINE_PER_CPU(int, last_cpu); num_words = desc_len(sh_desc); if (num_words > MAX_SDLEN) { dev_err(qidev, "Invalid descriptor len: %d words\n", num_words); return ERR_PTR(-EINVAL); } drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC); if (!drv_ctx) return ERR_PTR(-ENOMEM); /* * Initialise pre-header - set RSLS and SDLEN - and shared descriptor * and dma-map them. */ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | num_words); memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, DMA_BIDIRECTIONAL); if (dma_mapping_error(qidev, hwdesc)) { dev_err(qidev, "DMA map error for preheader + shdesc\n"); kfree(drv_ctx); return ERR_PTR(-ENOMEM); } drv_ctx->context_a = hwdesc; /* If given CPU does not own the portal, choose another one that does */ if (!cpumask_test_cpu(*cpu, cpus)) { int *pcpu = &get_cpu_var(last_cpu); *pcpu = cpumask_next(*pcpu, cpus); if (*pcpu >= nr_cpu_ids) *pcpu = cpumask_first(cpus); *cpu = *pcpu; put_cpu_var(last_cpu); } drv_ctx->cpu = *cpu; /* Find response FQ hooked with this CPU */ drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu); /* Attach request FQ */ drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc, QMAN_INITFQ_FLAG_SCHED); if (unlikely(IS_ERR_OR_NULL(drv_ctx->req_fq))) { dev_err(qidev, "create_caam_req_fq failed\n"); dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL); kfree(drv_ctx); return ERR_PTR(-ENOMEM); } drv_ctx->qidev = qidev; return drv_ctx; }
int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) { int ret; u32 num_words; struct qman_fq *new_fq, *old_fq; struct device *qidev = drv_ctx->qidev; num_words = desc_len(sh_desc); if (num_words > MAX_SDLEN) { dev_err(qidev, "Invalid descriptor len: %d words\n", num_words); return -EINVAL; } /* Note down older req FQ */ old_fq = drv_ctx->req_fq; /* Create a new req FQ in parked state */ new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq, drv_ctx->context_a, 0); if (unlikely(IS_ERR_OR_NULL(new_fq))) { dev_err(qidev, "FQ allocation for shdesc update failed\n"); return PTR_ERR(new_fq); } /* Hook up new FQ to context so that new requests keep queuing */ drv_ctx->req_fq = new_fq; /* Empty and remove the older FQ */ ret = empty_caam_fq(old_fq); if (ret) { dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret); /* We can revert to older FQ */ drv_ctx->req_fq = old_fq; if (kill_fq(qidev, new_fq)) dev_warn(qidev, "New CAAM FQ: %u kill failed\n", new_fq->fqid); return ret; } /* * Re-initialise pre-header. Set RSLS and SDLEN. * Update the shared descriptor for driver context. */ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | num_words); memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); dma_sync_single_for_device(qidev, drv_ctx->context_a, sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr), DMA_BIDIRECTIONAL); /* Put the new FQ in scheduled state */ ret = qman_schedule_fq(new_fq); if (ret) { dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret); /* * We can kill new FQ and revert to old FQ. * Since the desc is already modified, it is success case */ drv_ctx->req_fq = old_fq; if (kill_fq(qidev, new_fq)) dev_warn(qidev, "New CAAM FQ: %u kill failed\n", new_fq->fqid); } else if (kill_fq(qidev, old_fq)) { dev_warn(qidev, "Old CAAM FQ: %u kill failed\n", old_fq->fqid); } return 0; }
/** * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor * (non-protocol). * @desc: pointer to buffer used for descriptor construction * @cdata: pointer to block cipher transform definitions * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. * @icvsize: integrity check value (ICV) size (truncated or full) */ void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, unsigned int icvsize) { u32 *key_jump_cmd; init_sh_desc(desc, HDR_SHARE_SERIAL); /* Skip key loading if it is loaded due to sharing */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); if (cdata->key_inline) append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); else append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); set_jump_tgt_here(desc, key_jump_cmd); /* Class 1 operation */ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); /* Read assoc data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); /* Skip IV */ append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); /* Will read cryptlen bytes */ append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); /* Skip assoc data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* Will write cryptlen bytes */ append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); /* Store payload data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); /* Read encrypted data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); /* Read ICV */ append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); #ifdef DEBUG print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif }
/** * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor * @desc: pointer to buffer used for descriptor construction * @cdata: pointer to block cipher transform definitions * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. * @icvsize: integrity check value (ICV) size (truncated or full) */ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, unsigned int icvsize) { u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1; init_sh_desc(desc, HDR_SHARE_SERIAL); /* skip key loading if they are loaded due to sharing */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD | JUMP_COND_SELF); if (cdata->key_inline) append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); else append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); set_jump_tgt_here(desc, key_jump_cmd); /* class 1 operation */ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); /* if assoclen is ZERO, skip reading the assoc data */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_MATH_Z); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); /* skip assoc data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* read assoc data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); set_jump_tgt_here(desc, zero_assoc_jump_cmd1); /* cryptlen = seqoutlen - assoclen */ append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); /* jump to zero-payload command if cryptlen is zero */ zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_MATH_Z); append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); /* store encrypted data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); /* read payload data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); /* zero-payload command */ set_jump_tgt_here(desc, zero_payload_jump_cmd); /* read ICV */ append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); #ifdef DEBUG print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif }
/** * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor * (non-protocol) with no (null) encryption. * @desc: pointer to buffer used for descriptor construction * @adata: pointer to authentication transform definitions. Note that since a * split key is to be used, the size of the split key itself is * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. * @icvsize: integrity check value (ICV) size (truncated or full) * * Note: Requires an MDHA split key. */ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, unsigned int icvsize) { u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; init_sh_desc(desc, HDR_SHARE_SERIAL); /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); if (adata->key_inline) append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); else append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); set_jump_tgt_here(desc, key_jump_cmd); /* assoclen + cryptlen = seqinlen */ append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); /* Prepare to read and write cryptlen + assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); /* * MOVE_LEN opcode is not available in all SEC HW revisions, * thus need to do some magic, i.e. self-patch the descriptor * buffer. */ read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | (0x6 << MOVE_LEN_SHIFT)); write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | MOVE_WAITCOMP | (0x8 << MOVE_LEN_SHIFT)); /* Class 2 operation */ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); /* Read and write cryptlen bytes */ aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); set_move_tgt_here(desc, read_move_cmd); set_move_tgt_here(desc, write_move_cmd); append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | MOVE_AUX_LS); /* Write ICV */ append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); #ifdef DEBUG print_hex_dump(KERN_ERR, "aead null enc shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif }
/** * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor * (non-protocol) with HW-generated initialization * vector. * @desc: pointer to buffer used for descriptor construction * @cdata: pointer to block cipher transform definitions * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. * @adata: pointer to authentication transform definitions. Note that since a * split key is to be used, the size of the split key itself is * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. * @ivsize: initialization vector size * @icvsize: integrity check value (ICV) size (truncated or full) * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register * @is_qi: true when called from caam/qi * * Note: Requires an MDHA split key. */ void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool is_rfc3686, u32 *nonce, const u32 ctx1_iv_off, const bool is_qi) { u32 geniv, moveiv; /* Note: Context registers are saved. */ init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); if (is_qi) { u32 *wait_load_cmd; /* REG3 = assoclen */ append_seq_load(desc, 4, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH3 | (4 << LDST_OFFSET_SHIFT)); wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_CALM | JUMP_COND_NCP | JUMP_COND_NOP | JUMP_COND_NIP | JUMP_COND_NIFP); set_jump_tgt_here(desc, wait_load_cmd); } if (is_rfc3686) { if (is_qi) append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | (ctx1_iv_off << LDST_OFFSET_SHIFT)); goto copy_iv; } /* Generate IV */ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | (ctx1_iv_off << MOVE_OFFSET_SHIFT) | (ivsize << MOVE_LEN_SHIFT)); append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); copy_iv: /* Copy IV to class 1 context */ append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | (ctx1_iv_off << MOVE_OFFSET_SHIFT) | (ivsize << MOVE_LEN_SHIFT)); /* Return to encryption */ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); /* Read and write assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); /* Skip assoc data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); /* Copy iv from outfifo to class 2 fifo */ moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); /* Load Counter into CONTEXT1 reg */ if (is_rfc3686) append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << LDST_OFFSET_SHIFT)); /* Class 1 operation */ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); /* Will write ivsize + cryptlen */ append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); /* Not need to reload iv */ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); /* Will read cryptlen */ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); /* Write ICV */ append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); #ifdef DEBUG print_hex_dump(KERN_ERR, "aead givenc shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif }
/** * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor * (non-protocol). * @desc: pointer to buffer used for descriptor construction * @cdata: pointer to block cipher transform definitions * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. * @adata: pointer to authentication transform definitions. Note that since a * split key is to be used, the size of the split key itself is * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. * @ivsize: initialization vector size * @icvsize: integrity check value (ICV) size (truncated or full) * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @nonce: pointer to rfc3686 nonce * @ctx1_iv_off: IV offset in CONTEXT1 register * @is_qi: true when called from caam/qi * * Note: Requires an MDHA split key. */ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, struct alginfo *adata, unsigned int ivsize, unsigned int icvsize, const bool geniv, const bool is_rfc3686, u32 *nonce, const u32 ctx1_iv_off, const bool is_qi) { /* Note: Context registers are saved. */ init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); /* Class 2 operation */ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); if (is_qi) { u32 *wait_load_cmd; /* REG3 = assoclen */ append_seq_load(desc, 4, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH3 | (4 << LDST_OFFSET_SHIFT)); wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_CALM | JUMP_COND_NCP | JUMP_COND_NOP | JUMP_COND_NIP | JUMP_COND_NIFP); set_jump_tgt_here(desc, wait_load_cmd); if (!geniv) append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | (ctx1_iv_off << LDST_OFFSET_SHIFT)); } /* Read and write assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); if (geniv) append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize); else append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); /* Skip assoc data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); if (geniv) { append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | (ctx1_iv_off << LDST_OFFSET_SHIFT)); append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); } /* Load Counter into CONTEXT1 reg */ if (is_rfc3686) append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << LDST_OFFSET_SHIFT)); /* Choose operation */ if (ctx1_iv_off) append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); else append_dec_op1(desc, cdata->algtype); /* Read and write cryptlen bytes */ append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); aead_append_src_dst(desc, FIFOLD_TYPE_MSG); /* Load ICV */ append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); #ifdef DEBUG print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif }
/** * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor * with HW-generated initialization vector. * @desc: pointer to buffer used for descriptor construction * @cdata: pointer to block cipher transform definitions * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed * with OP_ALG_AAI_CBC. * @ivsize: initialization vector size * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template * @ctx1_iv_off: IV offset in CONTEXT1 register */ void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata, unsigned int ivsize, const bool is_rfc3686, const u32 ctx1_iv_off) { u32 *key_jump_cmd, geniv; init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); /* Load class1 key only */ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); /* Load Nonce into CONTEXT1 reg */ if (is_rfc3686) { u8 *nonce = cdata->key_virt + cdata->keylen; append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, LDST_CLASS_IND_CCB | LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); } set_jump_tgt_here(desc, key_jump_cmd); /* Generate IV */ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) | (ctx1_iv_off << MOVE_OFFSET_SHIFT)); append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); /* Copy generated IV to memory */ append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); /* Load Counter into CONTEXT1 reg */ if (is_rfc3686) append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << LDST_OFFSET_SHIFT)); if (ctx1_iv_off) append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | (1 << JUMP_OFFSET_SHIFT)); /* Load operation */ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); /* Perform operation */ ablkcipher_append_src_dst(desc); #ifdef DEBUG print_hex_dump(KERN_ERR, "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif }