static int fips_rsa_verify(int dtype,
const unsigned char *x, unsigned int y,
unsigned char *sigbuf, unsigned int siglen,
EVP_MD_SVCTX * sv)
{
int i, ret = 0;
unsigned int dlen, diglen;
int pad_mode = sv->mctx->flags & EVP_MD_CTX_FLAG_PAD_MASK;
int rsa_pad_mode = 0;
unsigned char *s;
const unsigned char *der;
unsigned char dig[EVP_MAX_MD_SIZE];
RSA *rsa = sv->key;
if (siglen != (unsigned int)RSA_size(sv->key)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH);
return (0);
}
EVP_DigestFinal_ex(sv->mctx, dig, &diglen);
if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_verify) {
return rsa->meth->rsa_verify(dtype, dig, diglen, sigbuf, siglen, rsa);
}
s = OPENSSL_malloc((unsigned int)siglen);
if (s == NULL) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, ERR_R_MALLOC_FAILURE);
goto err;
}
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931)
rsa_pad_mode = RSA_X931_PADDING;
else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1)
rsa_pad_mode = RSA_PKCS1_PADDING;
else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS)
rsa_pad_mode = RSA_NO_PADDING;
/* NB: call underlying method directly to avoid FIPS blocking */
i = rsa->meth->rsa_pub_dec((int)siglen, sigbuf, s, rsa, rsa_pad_mode);
if (i <= 0)
goto err;
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) {
int hash_id;
if (i != (int)(diglen + 1)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
hash_id = RSA_X931_hash_id(M_EVP_MD_CTX_type(sv->mctx));
if (hash_id == -1) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
goto err;
}
if (s[diglen] != (unsigned char)hash_id) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
if (memcmp(s, dig, diglen)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) {
der = fips_digestinfo_encoding(dtype, &dlen);
if (!der) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return (0);
}
/*
* Compare, DigestInfo length, DigestInfo header and finally digest
* value itself
*/
/* If length mismatch try alternate encoding */
if (i != (int)(dlen + diglen))
der = fips_digestinfo_nn_encoding(dtype, &dlen);
if ((i != (int)(dlen + diglen)) || memcmp(der, s, dlen)
|| memcmp(s + dlen, dig, diglen)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) {
int saltlen;
saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(sv->mctx);
if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN)
saltlen = -1;
else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC)
saltlen = -2;
ret = RSA_verify_PKCS1_PSS(rsa, dig, M_EVP_MD_CTX_md(sv->mctx),
s, saltlen);
if (ret < 0)
ret = 0;
}
err:
if (s != NULL) {
OPENSSL_cleanse(s, siglen);
OPENSSL_free(s);
}
return (ret);
}
int FIPS_rsa_verify_ctx(RSA *rsa, EVP_MD_CTX *ctx,
int rsa_pad_mode, int saltlen, const EVP_MD *mgf1Hash,
const unsigned char *sigbuf, unsigned int siglen)
{
unsigned int md_len, rv;
unsigned char md[EVP_MAX_MD_SIZE];
FIPS_digestfinal(ctx, md, &md_len);
rv = FIPS_rsa_verify_digest(rsa, md, md_len, M_EVP_MD_CTX_md(ctx),
rsa_pad_mode, saltlen, mgf1Hash,
sigbuf, siglen);
OPENSSL_cleanse(md, md_len);
return rv;
}
static int fips_rsa_sign(int type, const unsigned char *x, unsigned int y,
unsigned char *sigret, unsigned int *siglen,
EVP_MD_SVCTX * sv)
{
int i = 0, j, ret = 0;
unsigned int dlen;
const unsigned char *der;
unsigned int m_len;
int pad_mode = sv->mctx->flags & EVP_MD_CTX_FLAG_PAD_MASK;
int rsa_pad_mode = 0;
RSA *rsa = sv->key;
/* Largest DigestInfo: 19 (max encoding) + max MD */
unsigned char tmpdinfo[19 + EVP_MAX_MD_SIZE];
unsigned char md[EVP_MAX_MD_SIZE + 1];
EVP_DigestFinal_ex(sv->mctx, md, &m_len);
if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_sign) {
ret = rsa->meth->rsa_sign(type, md, m_len, sigret, siglen, rsa);
goto done;
}
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) {
int hash_id;
memcpy(tmpdinfo, md, m_len);
hash_id = RSA_X931_hash_id(M_EVP_MD_CTX_type(sv->mctx));
if (hash_id == -1) {
RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
tmpdinfo[m_len] = (unsigned char)hash_id;
i = m_len + 1;
rsa_pad_mode = RSA_X931_PADDING;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) {
der = fips_digestinfo_encoding(type, &dlen);
if (!der) {
RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
memcpy(tmpdinfo, der, dlen);
memcpy(tmpdinfo + dlen, md, m_len);
i = dlen + m_len;
rsa_pad_mode = RSA_PKCS1_PADDING;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) {
unsigned char *sbuf;
int saltlen;
i = RSA_size(rsa);
sbuf = OPENSSL_malloc(RSA_size(rsa));
saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(sv->mctx);
if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN)
saltlen = -1;
else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC)
saltlen = -2;
if (!sbuf) {
RSAerr(RSA_F_FIPS_RSA_SIGN, ERR_R_MALLOC_FAILURE);
goto psserr;
}
if (!RSA_padding_add_PKCS1_PSS(rsa, sbuf, md,
M_EVP_MD_CTX_md(sv->mctx), saltlen))
goto psserr;
j = rsa->meth->rsa_priv_enc(i, sbuf, sigret, rsa, RSA_NO_PADDING);
if (j > 0) {
ret = 1;
*siglen = j;
}
psserr:
OPENSSL_cleanse(md, m_len);
OPENSSL_cleanse(sbuf, i);
OPENSSL_free(sbuf);
return ret;
}
j = RSA_size(rsa);
if (i > (j - RSA_PKCS1_PADDING_SIZE)) {
RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
goto done;
}
/* NB: call underlying method directly to avoid FIPS blocking */
j = rsa->meth->rsa_priv_enc(i, tmpdinfo, sigret, rsa, rsa_pad_mode);
if (j > 0) {
ret = 1;
*siglen = j;
}
done:
OPENSSL_cleanse(tmpdinfo, i);
OPENSSL_cleanse(md, m_len);
return ret;
}
