static int fips_rsa_verify(int dtype,
const unsigned char *x, unsigned int y,
unsigned char *sigbuf, unsigned int siglen,
EVP_MD_SVCTX * sv)
{
int i, ret = 0;
unsigned int dlen, diglen;
int pad_mode = sv->mctx->flags & EVP_MD_CTX_FLAG_PAD_MASK;
int rsa_pad_mode = 0;
unsigned char *s;
const unsigned char *der;
unsigned char dig[EVP_MAX_MD_SIZE];
RSA *rsa = sv->key;
if (siglen != (unsigned int)RSA_size(sv->key)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH);
return (0);
}
EVP_DigestFinal_ex(sv->mctx, dig, &diglen);
if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_verify) {
return rsa->meth->rsa_verify(dtype, dig, diglen, sigbuf, siglen, rsa);
}
s = OPENSSL_malloc((unsigned int)siglen);
if (s == NULL) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, ERR_R_MALLOC_FAILURE);
goto err;
}
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931)
rsa_pad_mode = RSA_X931_PADDING;
else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1)
rsa_pad_mode = RSA_PKCS1_PADDING;
else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS)
rsa_pad_mode = RSA_NO_PADDING;
/* NB: call underlying method directly to avoid FIPS blocking */
i = rsa->meth->rsa_pub_dec((int)siglen, sigbuf, s, rsa, rsa_pad_mode);
if (i <= 0)
goto err;
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) {
int hash_id;
if (i != (int)(diglen + 1)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
hash_id = RSA_X931_hash_id(M_EVP_MD_CTX_type(sv->mctx));
if (hash_id == -1) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
goto err;
}
if (s[diglen] != (unsigned char)hash_id) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
if (memcmp(s, dig, diglen)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) {
der = fips_digestinfo_encoding(dtype, &dlen);
if (!der) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return (0);
}
/*
* Compare, DigestInfo length, DigestInfo header and finally digest
* value itself
*/
/* If length mismatch try alternate encoding */
if (i != (int)(dlen + diglen))
der = fips_digestinfo_nn_encoding(dtype, &dlen);
if ((i != (int)(dlen + diglen)) || memcmp(der, s, dlen)
|| memcmp(s + dlen, dig, diglen)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) {
int saltlen;
saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(sv->mctx);
if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN)
saltlen = -1;
else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC)
saltlen = -2;
ret = RSA_verify_PKCS1_PSS(rsa, dig, M_EVP_MD_CTX_md(sv->mctx),
s, saltlen);
if (ret < 0)
ret = 0;
}
err:
if (s != NULL) {
OPENSSL_cleanse(s, siglen);
OPENSSL_free(s);
}
return (ret);
}
int FIPS_rsa_verify_digest(RSA *rsa, const unsigned char *dig, int diglen,
const EVP_MD *mhash, int rsa_pad_mode, int saltlen,
const EVP_MD *mgf1Hash,
const unsigned char *sigbuf, unsigned int siglen)
{
int i,ret=0;
unsigned int dlen;
unsigned char *s;
const unsigned char *der;
int md_type;
int rsa_dec_pad_mode;
if (FIPS_selftest_failed())
{
FIPSerr(FIPS_F_FIPS_RSA_VERIFY_DIGEST, FIPS_R_SELFTEST_FAILED);
return 0;
}
if (siglen != (unsigned int)RSA_size(rsa))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_WRONG_SIGNATURE_LENGTH);
return(0);
}
if (!mhash && rsa_pad_mode == RSA_PKCS1_PADDING)
md_type = saltlen;
else
md_type = M_EVP_MD_type(mhash);
s= OPENSSL_malloc((unsigned int)siglen);
if (s == NULL)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,ERR_R_MALLOC_FAILURE);
goto err;
}
if (rsa_pad_mode == RSA_PKCS1_PSS_PADDING)
rsa_dec_pad_mode = RSA_NO_PADDING;
else
rsa_dec_pad_mode = rsa_pad_mode;
/* NB: call underlying method directly to avoid FIPS blocking */
i=rsa->meth->rsa_pub_dec((int)siglen,sigbuf,s, rsa, rsa_dec_pad_mode);
if (i <= 0) goto err;
if (rsa_pad_mode == RSA_X931_PADDING)
{
int hash_id;
if (i != (int)(diglen + 1))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
hash_id = RSA_X931_hash_id(md_type);
if (hash_id == -1)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
goto err;
}
if (s[diglen] != (unsigned char)hash_id)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
if (memcmp(s, dig, diglen))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
}
else if (rsa_pad_mode == RSA_PKCS1_PADDING)
{
der = fips_digestinfo_encoding(md_type, &dlen);
if (!der)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
return(0);
}
/* Compare, DigestInfo length, DigestInfo header and finally
* digest value itself
*/
/* If length mismatch try alternate encoding */
if (i != (int)(dlen + diglen))
der = fips_digestinfo_nn_encoding(md_type, &dlen);
if ((i != (int)(dlen + diglen)) || memcmp(der, s, dlen)
|| memcmp(s + dlen, dig, diglen))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
}
else if (rsa_pad_mode == RSA_PKCS1_PSS_PADDING)
{
ret = RSA_verify_PKCS1_PSS_mgf1(rsa, dig, mhash, mgf1Hash,
s, saltlen);
if (ret < 0)
ret = 0;
}
err:
if (s != NULL)
{
OPENSSL_cleanse(s, siglen);
OPENSSL_free(s);
}
return(ret);
}
int FIPS_rsa_sign_digest(RSA *rsa, const unsigned char *md, int md_len,
const EVP_MD *mhash, int rsa_pad_mode, int saltlen,
const EVP_MD *mgf1Hash,
unsigned char *sigret, unsigned int *siglen)
{
int i=0,j,ret=0;
unsigned int dlen;
const unsigned char *der;
int md_type;
/* Largest DigestInfo: 19 (max encoding) + max MD */
unsigned char tmpdinfo[19 + EVP_MAX_MD_SIZE];
if (FIPS_selftest_failed())
{
FIPSerr(FIPS_F_FIPS_RSA_SIGN_DIGEST, FIPS_R_SELFTEST_FAILED);
return 0;
}
if (!mhash && rsa_pad_mode == RSA_PKCS1_PADDING)
md_type = saltlen;
else
md_type = M_EVP_MD_type(mhash);
if (rsa_pad_mode == RSA_X931_PADDING)
{
int hash_id;
memcpy(tmpdinfo, md, md_len);
hash_id = RSA_X931_hash_id(md_type);
if (hash_id == -1)
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
tmpdinfo[md_len] = (unsigned char)hash_id;
i = md_len + 1;
}
else if (rsa_pad_mode == RSA_PKCS1_PADDING)
{
der = fips_digestinfo_encoding(md_type, &dlen);
if (!der)
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
memcpy(tmpdinfo, der, dlen);
memcpy(tmpdinfo + dlen, md, md_len);
i = dlen + md_len;
}
else if (rsa_pad_mode == RSA_PKCS1_PSS_PADDING)
{
unsigned char *sbuf;
i = RSA_size(rsa);
sbuf = OPENSSL_malloc(RSA_size(rsa));
if (!sbuf)
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,ERR_R_MALLOC_FAILURE);
goto psserr;
}
if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, sbuf, md, mhash,
mgf1Hash, saltlen))
goto psserr;
j=rsa->meth->rsa_priv_enc(i,sbuf,sigret,rsa,RSA_NO_PADDING);
if (j > 0)
{
ret=1;
*siglen=j;
}
psserr:
OPENSSL_cleanse(sbuf, i);
OPENSSL_free(sbuf);
return ret;
}
j=RSA_size(rsa);
if (i > (j-RSA_PKCS1_PADDING_SIZE))
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
goto done;
}
/* NB: call underlying method directly to avoid FIPS blocking */
j=rsa->meth->rsa_priv_enc(i,tmpdinfo,sigret,rsa,rsa_pad_mode);
if (j > 0)
{
ret=1;
*siglen=j;
}
done:
OPENSSL_cleanse(tmpdinfo,i);
return ret;
}
static int fips_rsa_sign(int type, const unsigned char *x, unsigned int y,
unsigned char *sigret, unsigned int *siglen,
EVP_MD_SVCTX * sv)
{
int i = 0, j, ret = 0;
unsigned int dlen;
const unsigned char *der;
unsigned int m_len;
int pad_mode = sv->mctx->flags & EVP_MD_CTX_FLAG_PAD_MASK;
int rsa_pad_mode = 0;
RSA *rsa = sv->key;
/* Largest DigestInfo: 19 (max encoding) + max MD */
unsigned char tmpdinfo[19 + EVP_MAX_MD_SIZE];
unsigned char md[EVP_MAX_MD_SIZE + 1];
EVP_DigestFinal_ex(sv->mctx, md, &m_len);
if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_sign) {
ret = rsa->meth->rsa_sign(type, md, m_len, sigret, siglen, rsa);
goto done;
}
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) {
int hash_id;
memcpy(tmpdinfo, md, m_len);
hash_id = RSA_X931_hash_id(M_EVP_MD_CTX_type(sv->mctx));
if (hash_id == -1) {
RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
tmpdinfo[m_len] = (unsigned char)hash_id;
i = m_len + 1;
rsa_pad_mode = RSA_X931_PADDING;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) {
der = fips_digestinfo_encoding(type, &dlen);
if (!der) {
RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
memcpy(tmpdinfo, der, dlen);
memcpy(tmpdinfo + dlen, md, m_len);
i = dlen + m_len;
rsa_pad_mode = RSA_PKCS1_PADDING;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) {
unsigned char *sbuf;
int saltlen;
i = RSA_size(rsa);
sbuf = OPENSSL_malloc(RSA_size(rsa));
saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(sv->mctx);
if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN)
saltlen = -1;
else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC)
saltlen = -2;
if (!sbuf) {
RSAerr(RSA_F_FIPS_RSA_SIGN, ERR_R_MALLOC_FAILURE);
goto psserr;
}
if (!RSA_padding_add_PKCS1_PSS(rsa, sbuf, md,
M_EVP_MD_CTX_md(sv->mctx), saltlen))
goto psserr;
j = rsa->meth->rsa_priv_enc(i, sbuf, sigret, rsa, RSA_NO_PADDING);
if (j > 0) {
ret = 1;
*siglen = j;
}
psserr:
OPENSSL_cleanse(md, m_len);
OPENSSL_cleanse(sbuf, i);
OPENSSL_free(sbuf);
return ret;
}
j = RSA_size(rsa);
if (i > (j - RSA_PKCS1_PADDING_SIZE)) {
RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
goto done;
}
/* NB: call underlying method directly to avoid FIPS blocking */
j = rsa->meth->rsa_priv_enc(i, tmpdinfo, sigret, rsa, rsa_pad_mode);
if (j > 0) {
ret = 1;
*siglen = j;
}
done:
OPENSSL_cleanse(tmpdinfo, i);
OPENSSL_cleanse(md, m_len);
return ret;
}
