static int fips_rsa_verify(int dtype,
const unsigned char *x, unsigned int y,
unsigned char *sigbuf, unsigned int siglen,
EVP_MD_SVCTX * sv)
{
int i, ret = 0;
unsigned int dlen, diglen;
int pad_mode = sv->mctx->flags & EVP_MD_CTX_FLAG_PAD_MASK;
int rsa_pad_mode = 0;
unsigned char *s;
const unsigned char *der;
unsigned char dig[EVP_MAX_MD_SIZE];
RSA *rsa = sv->key;
if (siglen != (unsigned int)RSA_size(sv->key)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH);
return (0);
}
EVP_DigestFinal_ex(sv->mctx, dig, &diglen);
if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_verify) {
return rsa->meth->rsa_verify(dtype, dig, diglen, sigbuf, siglen, rsa);
}
s = OPENSSL_malloc((unsigned int)siglen);
if (s == NULL) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, ERR_R_MALLOC_FAILURE);
goto err;
}
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931)
rsa_pad_mode = RSA_X931_PADDING;
else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1)
rsa_pad_mode = RSA_PKCS1_PADDING;
else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS)
rsa_pad_mode = RSA_NO_PADDING;
/* NB: call underlying method directly to avoid FIPS blocking */
i = rsa->meth->rsa_pub_dec((int)siglen, sigbuf, s, rsa, rsa_pad_mode);
if (i <= 0)
goto err;
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) {
int hash_id;
if (i != (int)(diglen + 1)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
hash_id = RSA_X931_hash_id(M_EVP_MD_CTX_type(sv->mctx));
if (hash_id == -1) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
goto err;
}
if (s[diglen] != (unsigned char)hash_id) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
if (memcmp(s, dig, diglen)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) {
der = fips_digestinfo_encoding(dtype, &dlen);
if (!der) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return (0);
}
/*
* Compare, DigestInfo length, DigestInfo header and finally digest
* value itself
*/
/* If length mismatch try alternate encoding */
if (i != (int)(dlen + diglen))
der = fips_digestinfo_nn_encoding(dtype, &dlen);
if ((i != (int)(dlen + diglen)) || memcmp(der, s, dlen)
|| memcmp(s + dlen, dig, diglen)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) {
int saltlen;
saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(sv->mctx);
if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN)
saltlen = -1;
else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC)
saltlen = -2;
ret = RSA_verify_PKCS1_PSS(rsa, dig, M_EVP_MD_CTX_md(sv->mctx),
s, saltlen);
if (ret < 0)
ret = 0;
}
err:
if (s != NULL) {
OPENSSL_cleanse(s, siglen);
OPENSSL_free(s);
}
return (ret);
}
int FIPS_rsa_verify_digest(RSA *rsa, const unsigned char *dig, int diglen,
const EVP_MD *mhash, int rsa_pad_mode, int saltlen,
const EVP_MD *mgf1Hash,
const unsigned char *sigbuf, unsigned int siglen)
{
int i,ret=0;
unsigned int dlen;
unsigned char *s;
const unsigned char *der;
int md_type;
int rsa_dec_pad_mode;
if (FIPS_selftest_failed())
{
FIPSerr(FIPS_F_FIPS_RSA_VERIFY_DIGEST, FIPS_R_SELFTEST_FAILED);
return 0;
}
if (siglen != (unsigned int)RSA_size(rsa))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_WRONG_SIGNATURE_LENGTH);
return(0);
}
if (!mhash && rsa_pad_mode == RSA_PKCS1_PADDING)
md_type = saltlen;
else
md_type = M_EVP_MD_type(mhash);
s= OPENSSL_malloc((unsigned int)siglen);
if (s == NULL)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,ERR_R_MALLOC_FAILURE);
goto err;
}
if (rsa_pad_mode == RSA_PKCS1_PSS_PADDING)
rsa_dec_pad_mode = RSA_NO_PADDING;
else
rsa_dec_pad_mode = rsa_pad_mode;
/* NB: call underlying method directly to avoid FIPS blocking */
i=rsa->meth->rsa_pub_dec((int)siglen,sigbuf,s, rsa, rsa_dec_pad_mode);
if (i <= 0) goto err;
if (rsa_pad_mode == RSA_X931_PADDING)
{
int hash_id;
if (i != (int)(diglen + 1))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
hash_id = RSA_X931_hash_id(md_type);
if (hash_id == -1)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
goto err;
}
if (s[diglen] != (unsigned char)hash_id)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
if (memcmp(s, dig, diglen))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
}
else if (rsa_pad_mode == RSA_PKCS1_PADDING)
{
der = fips_digestinfo_encoding(md_type, &dlen);
if (!der)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
return(0);
}
/* Compare, DigestInfo length, DigestInfo header and finally
* digest value itself
*/
/* If length mismatch try alternate encoding */
if (i != (int)(dlen + diglen))
der = fips_digestinfo_nn_encoding(md_type, &dlen);
if ((i != (int)(dlen + diglen)) || memcmp(der, s, dlen)
|| memcmp(s + dlen, dig, diglen))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
}
else if (rsa_pad_mode == RSA_PKCS1_PSS_PADDING)
{
ret = RSA_verify_PKCS1_PSS_mgf1(rsa, dig, mhash, mgf1Hash,
s, saltlen);
if (ret < 0)
ret = 0;
}
err:
if (s != NULL)
{
OPENSSL_cleanse(s, siglen);
OPENSSL_free(s);
}
return(ret);
}
