static int fips_rsa_verify(int dtype, const unsigned char *x, unsigned int y, unsigned char *sigbuf, unsigned int siglen, EVP_MD_SVCTX * sv) { int i, ret = 0; unsigned int dlen, diglen; int pad_mode = sv->mctx->flags & EVP_MD_CTX_FLAG_PAD_MASK; int rsa_pad_mode = 0; unsigned char *s; const unsigned char *der; unsigned char dig[EVP_MAX_MD_SIZE]; RSA *rsa = sv->key; if (siglen != (unsigned int)RSA_size(sv->key)) { RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH); return (0); } EVP_DigestFinal_ex(sv->mctx, dig, &diglen); if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_verify) { return rsa->meth->rsa_verify(dtype, dig, diglen, sigbuf, siglen, rsa); } s = OPENSSL_malloc((unsigned int)siglen); if (s == NULL) { RSAerr(RSA_F_FIPS_RSA_VERIFY, ERR_R_MALLOC_FAILURE); goto err; } if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) rsa_pad_mode = RSA_X931_PADDING; else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) rsa_pad_mode = RSA_PKCS1_PADDING; else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) rsa_pad_mode = RSA_NO_PADDING; /* NB: call underlying method directly to avoid FIPS blocking */ i = rsa->meth->rsa_pub_dec((int)siglen, sigbuf, s, rsa, rsa_pad_mode); if (i <= 0) goto err; if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) { int hash_id; if (i != (int)(diglen + 1)) { RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE); goto err; } hash_id = RSA_X931_hash_id(M_EVP_MD_CTX_type(sv->mctx)); if (hash_id == -1) { RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE); goto err; } if (s[diglen] != (unsigned char)hash_id) { RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE); goto err; } if (memcmp(s, dig, diglen)) { RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE); goto err; } ret = 1; } else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) { der = fips_digestinfo_encoding(dtype, &dlen); if (!der) { RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE); return (0); } /* * Compare, DigestInfo length, DigestInfo header and finally digest * value itself */ /* If length mismatch try alternate encoding */ if (i != (int)(dlen + diglen)) der = fips_digestinfo_nn_encoding(dtype, &dlen); if ((i != (int)(dlen + diglen)) || memcmp(der, s, dlen) || memcmp(s + dlen, dig, diglen)) { RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE); goto err; } ret = 1; } else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) { int saltlen; saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(sv->mctx); if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN) saltlen = -1; else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC) saltlen = -2; ret = RSA_verify_PKCS1_PSS(rsa, dig, M_EVP_MD_CTX_md(sv->mctx), s, saltlen); if (ret < 0) ret = 0; } err: if (s != NULL) { OPENSSL_cleanse(s, siglen); OPENSSL_free(s); } return (ret); }
static int fips_rsa_sign(int type, const unsigned char *x, unsigned int y, unsigned char *sigret, unsigned int *siglen, EVP_MD_SVCTX * sv) { int i = 0, j, ret = 0; unsigned int dlen; const unsigned char *der; unsigned int m_len; int pad_mode = sv->mctx->flags & EVP_MD_CTX_FLAG_PAD_MASK; int rsa_pad_mode = 0; RSA *rsa = sv->key; /* Largest DigestInfo: 19 (max encoding) + max MD */ unsigned char tmpdinfo[19 + EVP_MAX_MD_SIZE]; unsigned char md[EVP_MAX_MD_SIZE + 1]; EVP_DigestFinal_ex(sv->mctx, md, &m_len); if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_sign) { ret = rsa->meth->rsa_sign(type, md, m_len, sigret, siglen, rsa); goto done; } if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) { int hash_id; memcpy(tmpdinfo, md, m_len); hash_id = RSA_X931_hash_id(M_EVP_MD_CTX_type(sv->mctx)); if (hash_id == -1) { RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_UNKNOWN_ALGORITHM_TYPE); return 0; } tmpdinfo[m_len] = (unsigned char)hash_id; i = m_len + 1; rsa_pad_mode = RSA_X931_PADDING; } else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) { der = fips_digestinfo_encoding(type, &dlen); if (!der) { RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_UNKNOWN_ALGORITHM_TYPE); return 0; } memcpy(tmpdinfo, der, dlen); memcpy(tmpdinfo + dlen, md, m_len); i = dlen + m_len; rsa_pad_mode = RSA_PKCS1_PADDING; } else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) { unsigned char *sbuf; int saltlen; i = RSA_size(rsa); sbuf = OPENSSL_malloc(RSA_size(rsa)); saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(sv->mctx); if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN) saltlen = -1; else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC) saltlen = -2; if (!sbuf) { RSAerr(RSA_F_FIPS_RSA_SIGN, ERR_R_MALLOC_FAILURE); goto psserr; } if (!RSA_padding_add_PKCS1_PSS(rsa, sbuf, md, M_EVP_MD_CTX_md(sv->mctx), saltlen)) goto psserr; j = rsa->meth->rsa_priv_enc(i, sbuf, sigret, rsa, RSA_NO_PADDING); if (j > 0) { ret = 1; *siglen = j; } psserr: OPENSSL_cleanse(md, m_len); OPENSSL_cleanse(sbuf, i); OPENSSL_free(sbuf); return ret; } j = RSA_size(rsa); if (i > (j - RSA_PKCS1_PADDING_SIZE)) { RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY); goto done; } /* NB: call underlying method directly to avoid FIPS blocking */ j = rsa->meth->rsa_priv_enc(i, tmpdinfo, sigret, rsa, rsa_pad_mode); if (j > 0) { ret = 1; *siglen = j; } done: OPENSSL_cleanse(tmpdinfo, i); OPENSSL_cleanse(md, m_len); return ret; }
int EVP_SignFinal(EVP_MD_CTX *ctx, unsigned char *sigret, unsigned int *siglen, EVP_PKEY *pkey) { unsigned char m[EVP_MAX_MD_SIZE]; unsigned int m_len; int i = 0, ok = 0, v; EVP_MD_CTX tmp_ctx; EVP_PKEY_CTX *pkctx = NULL; *siglen = 0; EVP_MD_CTX_init(&tmp_ctx); if (!EVP_MD_CTX_copy_ex(&tmp_ctx, ctx)) goto err; if (!EVP_DigestFinal_ex(&tmp_ctx, &(m[0]), &m_len)) goto err; EVP_MD_CTX_cleanup(&tmp_ctx); if (ctx->digest->flags & EVP_MD_FLAG_PKEY_METHOD_SIGNATURE) { size_t sltmp = (size_t)EVP_PKEY_size(pkey); i = 0; pkctx = EVP_PKEY_CTX_new(pkey, NULL); if (!pkctx) goto err; if (EVP_PKEY_sign_init(pkctx) <= 0) goto err; if (EVP_PKEY_CTX_set_signature_md(pkctx, ctx->digest) <= 0) goto err; if (ctx->flags & EVP_MD_CTX_FLAG_PAD_X931) if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_X931_PADDING) <= 0) goto err; if (ctx->flags & EVP_MD_CTX_FLAG_PAD_PSS) { int saltlen; if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING) <= 0) goto err; saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(ctx); if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN) saltlen = -1; else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC) saltlen = -2; if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pkctx, saltlen) <= 0) goto err; } if (EVP_PKEY_sign(pkctx, sigret, &sltmp, m, m_len) <= 0) goto err; *siglen = sltmp; i = 1; err: EVP_PKEY_CTX_free(pkctx); return i; } for (i = 0; i < 4; i++) { v = ctx->digest->required_pkey_type[i]; if (v == 0) break; if (pkey->type == v) { ok = 1; break; } } if (!ok) { EVPerr(EVP_F_EVP_SIGNFINAL, EVP_R_WRONG_PUBLIC_KEY_TYPE); return (0); } if (ctx->digest->sign == NULL) { EVPerr(EVP_F_EVP_SIGNFINAL, EVP_R_NO_SIGN_FUNCTION_CONFIGURED); return (0); } return (ctx->digest->sign(ctx->digest->type, m, m_len, sigret, siglen, pkey->pkey.ptr)); }