static int fips_rsa_verify(int dtype,
const unsigned char *x, unsigned int y,
unsigned char *sigbuf, unsigned int siglen,
EVP_MD_SVCTX * sv)
{
int i, ret = 0;
unsigned int dlen, diglen;
int pad_mode = sv->mctx->flags & EVP_MD_CTX_FLAG_PAD_MASK;
int rsa_pad_mode = 0;
unsigned char *s;
const unsigned char *der;
unsigned char dig[EVP_MAX_MD_SIZE];
RSA *rsa = sv->key;
if (siglen != (unsigned int)RSA_size(sv->key)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH);
return (0);
}
EVP_DigestFinal_ex(sv->mctx, dig, &diglen);
if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_verify) {
return rsa->meth->rsa_verify(dtype, dig, diglen, sigbuf, siglen, rsa);
}
s = OPENSSL_malloc((unsigned int)siglen);
if (s == NULL) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, ERR_R_MALLOC_FAILURE);
goto err;
}
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931)
rsa_pad_mode = RSA_X931_PADDING;
else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1)
rsa_pad_mode = RSA_PKCS1_PADDING;
else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS)
rsa_pad_mode = RSA_NO_PADDING;
/* NB: call underlying method directly to avoid FIPS blocking */
i = rsa->meth->rsa_pub_dec((int)siglen, sigbuf, s, rsa, rsa_pad_mode);
if (i <= 0)
goto err;
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) {
int hash_id;
if (i != (int)(diglen + 1)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
hash_id = RSA_X931_hash_id(M_EVP_MD_CTX_type(sv->mctx));
if (hash_id == -1) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
goto err;
}
if (s[diglen] != (unsigned char)hash_id) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
if (memcmp(s, dig, diglen)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) {
der = fips_digestinfo_encoding(dtype, &dlen);
if (!der) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return (0);
}
/*
* Compare, DigestInfo length, DigestInfo header and finally digest
* value itself
*/
/* If length mismatch try alternate encoding */
if (i != (int)(dlen + diglen))
der = fips_digestinfo_nn_encoding(dtype, &dlen);
if ((i != (int)(dlen + diglen)) || memcmp(der, s, dlen)
|| memcmp(s + dlen, dig, diglen)) {
RSAerr(RSA_F_FIPS_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) {
int saltlen;
saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(sv->mctx);
if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN)
saltlen = -1;
else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC)
saltlen = -2;
ret = RSA_verify_PKCS1_PSS(rsa, dig, M_EVP_MD_CTX_md(sv->mctx),
s, saltlen);
if (ret < 0)
ret = 0;
}
err:
if (s != NULL) {
OPENSSL_cleanse(s, siglen);
OPENSSL_free(s);
}
return (ret);
}
static int
pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
const unsigned char *tbs, size_t tbslen)
{
int ret;
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
if (rctx->md) {
if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
RSAerr(RSA_F_PKEY_RSA_SIGN,
RSA_R_INVALID_DIGEST_LENGTH);
return -1;
}
if (EVP_MD_type(rctx->md) == NID_mdc2) {
unsigned int sltmp;
if (rctx->pad_mode != RSA_PKCS1_PADDING)
return -1;
ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2, tbs, tbslen,
sig, &sltmp, rsa);
if (ret <= 0)
return ret;
ret = sltmp;
} else if (rctx->pad_mode == RSA_X931_PADDING) {
if (!setup_tbuf(rctx, ctx))
return -1;
memcpy(rctx->tbuf, tbs, tbslen);
rctx->tbuf[tbslen] =
RSA_X931_hash_id(EVP_MD_type(rctx->md));
ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, sig,
rsa, RSA_X931_PADDING);
} else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
unsigned int sltmp;
ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig,
&sltmp, rsa);
if (ret <= 0)
return ret;
ret = sltmp;
} else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
if (!setup_tbuf(rctx, ctx))
return -1;
if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf,
tbs, rctx->md, rctx->mgf1md, rctx->saltlen))
return -1;
ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
sig, rsa, RSA_NO_PADDING);
} else
return -1;
} else
ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
rctx->pad_mode);
if (ret < 0)
return ret;
*siglen = ret;
return 1;
}
static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
unsigned char *rout, size_t *routlen,
const unsigned char *sig, size_t siglen)
{
int ret;
RSA_PKEY_CTX *rctx = ctx->data;
if (rctx->md)
{
if (rctx->pad_mode == RSA_X931_PADDING)
{
if (!setup_tbuf(rctx, ctx))
return -1;
ret = RSA_public_decrypt(siglen, sig,
rctx->tbuf, ctx->pkey->pkey.rsa,
RSA_X931_PADDING);
if (ret < 1)
return 0;
ret--;
if (rctx->tbuf[ret] !=
RSA_X931_hash_id(EVP_MD_type(rctx->md)))
{
RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
RSA_R_ALGORITHM_MISMATCH);
return 0;
}
if (ret != EVP_MD_size(rctx->md))
{
RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
RSA_R_INVALID_DIGEST_LENGTH);
return 0;
}
if (rout)
memcpy(rout, rctx->tbuf, ret);
}
else if (rctx->pad_mode == RSA_PKCS1_PADDING)
{
size_t sltmp;
ret = int_rsa_verify(EVP_MD_type(rctx->md),
NULL, 0, rout, &sltmp,
sig, siglen, ctx->pkey->pkey.rsa);
if (ret <= 0)
return 0;
ret = sltmp;
}
else
return -1;
}
else
ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
rctx->pad_mode);
if (ret < 0)
return ret;
*routlen = ret;
return 1;
}
static int check_padding_md(const EVP_MD *md, int padding)
{
int mdnid;
if (!md)
return 1;
mdnid = EVP_MD_type(md);
if (padding == RSA_NO_PADDING) {
RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
return 0;
}
if (padding == RSA_X931_PADDING) {
if (RSA_X931_hash_id(mdnid) == -1) {
RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_X931_DIGEST);
return 0;
}
} else {
switch(mdnid) {
/* List of all supported RSA digests */
case NID_sha1:
case NID_sha224:
case NID_sha256:
case NID_sha384:
case NID_sha512:
case NID_md5:
case NID_md5_sha1:
case NID_md2:
case NID_md4:
case NID_mdc2:
case NID_ripemd160:
case NID_sha3_224:
case NID_sha3_256:
case NID_sha3_384:
case NID_sha3_512:
return 1;
default:
RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_DIGEST);
return 0;
}
}
return 1;
}
static int check_padding_md(const EVP_MD *md, int padding)
{
if (!md)
return 1;
if (padding == RSA_NO_PADDING) {
RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
return 0;
}
if (padding == RSA_X931_PADDING) {
if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) {
RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_X931_DIGEST);
return 0;
}
return 1;
}
return 1;
}
int FIPS_rsa_verify_digest(RSA *rsa, const unsigned char *dig, int diglen,
const EVP_MD *mhash, int rsa_pad_mode, int saltlen,
const EVP_MD *mgf1Hash,
const unsigned char *sigbuf, unsigned int siglen)
{
int i,ret=0;
unsigned int dlen;
unsigned char *s;
const unsigned char *der;
int md_type;
int rsa_dec_pad_mode;
if (FIPS_selftest_failed())
{
FIPSerr(FIPS_F_FIPS_RSA_VERIFY_DIGEST, FIPS_R_SELFTEST_FAILED);
return 0;
}
if (siglen != (unsigned int)RSA_size(rsa))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_WRONG_SIGNATURE_LENGTH);
return(0);
}
if (!mhash && rsa_pad_mode == RSA_PKCS1_PADDING)
md_type = saltlen;
else
md_type = M_EVP_MD_type(mhash);
s= OPENSSL_malloc((unsigned int)siglen);
if (s == NULL)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,ERR_R_MALLOC_FAILURE);
goto err;
}
if (rsa_pad_mode == RSA_PKCS1_PSS_PADDING)
rsa_dec_pad_mode = RSA_NO_PADDING;
else
rsa_dec_pad_mode = rsa_pad_mode;
/* NB: call underlying method directly to avoid FIPS blocking */
i=rsa->meth->rsa_pub_dec((int)siglen,sigbuf,s, rsa, rsa_dec_pad_mode);
if (i <= 0) goto err;
if (rsa_pad_mode == RSA_X931_PADDING)
{
int hash_id;
if (i != (int)(diglen + 1))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
hash_id = RSA_X931_hash_id(md_type);
if (hash_id == -1)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
goto err;
}
if (s[diglen] != (unsigned char)hash_id)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
if (memcmp(s, dig, diglen))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
}
else if (rsa_pad_mode == RSA_PKCS1_PADDING)
{
der = fips_digestinfo_encoding(md_type, &dlen);
if (!der)
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
return(0);
}
/* Compare, DigestInfo length, DigestInfo header and finally
* digest value itself
*/
/* If length mismatch try alternate encoding */
if (i != (int)(dlen + diglen))
der = fips_digestinfo_nn_encoding(md_type, &dlen);
if ((i != (int)(dlen + diglen)) || memcmp(der, s, dlen)
|| memcmp(s + dlen, dig, diglen))
{
RSAerr(RSA_F_FIPS_RSA_VERIFY_DIGEST,RSA_R_BAD_SIGNATURE);
goto err;
}
ret = 1;
}
else if (rsa_pad_mode == RSA_PKCS1_PSS_PADDING)
{
ret = RSA_verify_PKCS1_PSS_mgf1(rsa, dig, mhash, mgf1Hash,
s, saltlen);
if (ret < 0)
ret = 0;
}
err:
if (s != NULL)
{
OPENSSL_cleanse(s, siglen);
OPENSSL_free(s);
}
return(ret);
}
int FIPS_rsa_sign_digest(RSA *rsa, const unsigned char *md, int md_len,
const EVP_MD *mhash, int rsa_pad_mode, int saltlen,
const EVP_MD *mgf1Hash,
unsigned char *sigret, unsigned int *siglen)
{
int i=0,j,ret=0;
unsigned int dlen;
const unsigned char *der;
int md_type;
/* Largest DigestInfo: 19 (max encoding) + max MD */
unsigned char tmpdinfo[19 + EVP_MAX_MD_SIZE];
if (FIPS_selftest_failed())
{
FIPSerr(FIPS_F_FIPS_RSA_SIGN_DIGEST, FIPS_R_SELFTEST_FAILED);
return 0;
}
if (!mhash && rsa_pad_mode == RSA_PKCS1_PADDING)
md_type = saltlen;
else
md_type = M_EVP_MD_type(mhash);
if (rsa_pad_mode == RSA_X931_PADDING)
{
int hash_id;
memcpy(tmpdinfo, md, md_len);
hash_id = RSA_X931_hash_id(md_type);
if (hash_id == -1)
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
tmpdinfo[md_len] = (unsigned char)hash_id;
i = md_len + 1;
}
else if (rsa_pad_mode == RSA_PKCS1_PADDING)
{
der = fips_digestinfo_encoding(md_type, &dlen);
if (!der)
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
memcpy(tmpdinfo, der, dlen);
memcpy(tmpdinfo + dlen, md, md_len);
i = dlen + md_len;
}
else if (rsa_pad_mode == RSA_PKCS1_PSS_PADDING)
{
unsigned char *sbuf;
i = RSA_size(rsa);
sbuf = OPENSSL_malloc(RSA_size(rsa));
if (!sbuf)
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,ERR_R_MALLOC_FAILURE);
goto psserr;
}
if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, sbuf, md, mhash,
mgf1Hash, saltlen))
goto psserr;
j=rsa->meth->rsa_priv_enc(i,sbuf,sigret,rsa,RSA_NO_PADDING);
if (j > 0)
{
ret=1;
*siglen=j;
}
psserr:
OPENSSL_cleanse(sbuf, i);
OPENSSL_free(sbuf);
return ret;
}
j=RSA_size(rsa);
if (i > (j-RSA_PKCS1_PADDING_SIZE))
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
goto done;
}
/* NB: call underlying method directly to avoid FIPS blocking */
j=rsa->meth->rsa_priv_enc(i,tmpdinfo,sigret,rsa,rsa_pad_mode);
if (j > 0)
{
ret=1;
*siglen=j;
}
done:
OPENSSL_cleanse(tmpdinfo,i);
return ret;
}
static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
const unsigned char *tbs, size_t tbslen)
{
int ret;
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
#ifdef OPENSSL_FIPS
ret = pkey_fips_check_ctx(ctx);
if (ret < 0)
{
RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE);
return -1;
}
#endif
if (rctx->md)
{
if (tbslen != (size_t)EVP_MD_size(rctx->md))
{
RSAerr(RSA_F_PKEY_RSA_SIGN,
RSA_R_INVALID_DIGEST_LENGTH);
return -1;
}
#ifdef OPENSSL_FIPS
if (ret > 0)
{
unsigned int slen;
ret = FIPS_rsa_sign_digest(rsa, tbs, tbslen, rctx->md,
rctx->pad_mode,
rctx->saltlen,
rctx->mgf1md,
sig, &slen);
if (ret > 0)
*siglen = slen;
else
*siglen = 0;
return ret;
}
#endif
if (EVP_MD_type(rctx->md) == NID_mdc2)
{
unsigned int sltmp;
if (rctx->pad_mode != RSA_PKCS1_PADDING)
return -1;
ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2,
tbs, tbslen, sig, &sltmp, rsa);
if (ret <= 0)
return ret;
ret = sltmp;
}
else if (rctx->pad_mode == RSA_X931_PADDING)
{
if (!setup_tbuf(rctx, ctx))
return -1;
memcpy(rctx->tbuf, tbs, tbslen);
rctx->tbuf[tbslen] =
RSA_X931_hash_id(EVP_MD_type(rctx->md));
ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
sig, rsa, RSA_X931_PADDING);
}
else if (rctx->pad_mode == RSA_PKCS1_PADDING)
{
unsigned int sltmp;
ret = RSA_sign(EVP_MD_type(rctx->md),
tbs, tbslen, sig, &sltmp, rsa);
if (ret <= 0)
return ret;
ret = sltmp;
}
else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
{
if (!setup_tbuf(rctx, ctx))
return -1;
if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa,
rctx->tbuf, tbs,
rctx->md, rctx->mgf1md,
rctx->saltlen))
return -1;
ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
sig, rsa, RSA_NO_PADDING);
}
else
return -1;
}
else
ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
rctx->pad_mode);
if (ret < 0)
return ret;
*siglen = ret;
return 1;
}
static int fips_rsa_sign(int type, const unsigned char *x, unsigned int y,
unsigned char *sigret, unsigned int *siglen,
EVP_MD_SVCTX * sv)
{
int i = 0, j, ret = 0;
unsigned int dlen;
const unsigned char *der;
unsigned int m_len;
int pad_mode = sv->mctx->flags & EVP_MD_CTX_FLAG_PAD_MASK;
int rsa_pad_mode = 0;
RSA *rsa = sv->key;
/* Largest DigestInfo: 19 (max encoding) + max MD */
unsigned char tmpdinfo[19 + EVP_MAX_MD_SIZE];
unsigned char md[EVP_MAX_MD_SIZE + 1];
EVP_DigestFinal_ex(sv->mctx, md, &m_len);
if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_sign) {
ret = rsa->meth->rsa_sign(type, md, m_len, sigret, siglen, rsa);
goto done;
}
if (pad_mode == EVP_MD_CTX_FLAG_PAD_X931) {
int hash_id;
memcpy(tmpdinfo, md, m_len);
hash_id = RSA_X931_hash_id(M_EVP_MD_CTX_type(sv->mctx));
if (hash_id == -1) {
RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
tmpdinfo[m_len] = (unsigned char)hash_id;
i = m_len + 1;
rsa_pad_mode = RSA_X931_PADDING;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PKCS1) {
der = fips_digestinfo_encoding(type, &dlen);
if (!der) {
RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
memcpy(tmpdinfo, der, dlen);
memcpy(tmpdinfo + dlen, md, m_len);
i = dlen + m_len;
rsa_pad_mode = RSA_PKCS1_PADDING;
} else if (pad_mode == EVP_MD_CTX_FLAG_PAD_PSS) {
unsigned char *sbuf;
int saltlen;
i = RSA_size(rsa);
sbuf = OPENSSL_malloc(RSA_size(rsa));
saltlen = M_EVP_MD_CTX_FLAG_PSS_SALT(sv->mctx);
if (saltlen == EVP_MD_CTX_FLAG_PSS_MDLEN)
saltlen = -1;
else if (saltlen == EVP_MD_CTX_FLAG_PSS_MREC)
saltlen = -2;
if (!sbuf) {
RSAerr(RSA_F_FIPS_RSA_SIGN, ERR_R_MALLOC_FAILURE);
goto psserr;
}
if (!RSA_padding_add_PKCS1_PSS(rsa, sbuf, md,
M_EVP_MD_CTX_md(sv->mctx), saltlen))
goto psserr;
j = rsa->meth->rsa_priv_enc(i, sbuf, sigret, rsa, RSA_NO_PADDING);
if (j > 0) {
ret = 1;
*siglen = j;
}
psserr:
OPENSSL_cleanse(md, m_len);
OPENSSL_cleanse(sbuf, i);
OPENSSL_free(sbuf);
return ret;
}
j = RSA_size(rsa);
if (i > (j - RSA_PKCS1_PADDING_SIZE)) {
RSAerr(RSA_F_FIPS_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
goto done;
}
/* NB: call underlying method directly to avoid FIPS blocking */
j = rsa->meth->rsa_priv_enc(i, tmpdinfo, sigret, rsa, rsa_pad_mode);
if (j > 0) {
ret = 1;
*siglen = j;
}
done:
OPENSSL_cleanse(tmpdinfo, i);
OPENSSL_cleanse(md, m_len);
return ret;
}
