static int
pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
const unsigned char *tbs, size_t tbslen)
{
int ret;
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
if (rctx->md) {
if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
RSAerr(RSA_F_PKEY_RSA_SIGN,
RSA_R_INVALID_DIGEST_LENGTH);
return -1;
}
if (EVP_MD_type(rctx->md) == NID_mdc2) {
unsigned int sltmp;
if (rctx->pad_mode != RSA_PKCS1_PADDING)
return -1;
ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2, tbs, tbslen,
sig, &sltmp, rsa);
if (ret <= 0)
return ret;
ret = sltmp;
} else if (rctx->pad_mode == RSA_X931_PADDING) {
if (!setup_tbuf(rctx, ctx))
return -1;
memcpy(rctx->tbuf, tbs, tbslen);
rctx->tbuf[tbslen] =
RSA_X931_hash_id(EVP_MD_type(rctx->md));
ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, sig,
rsa, RSA_X931_PADDING);
} else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
unsigned int sltmp;
ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig,
&sltmp, rsa);
if (ret <= 0)
return ret;
ret = sltmp;
} else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
if (!setup_tbuf(rctx, ctx))
return -1;
if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf,
tbs, rctx->md, rctx->mgf1md, rctx->saltlen))
return -1;
ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
sig, rsa, RSA_NO_PADDING);
} else
return -1;
} else
ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
rctx->pad_mode);
if (ret < 0)
return ret;
*siglen = ret;
return 1;
}
static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen,
const uint8_t *tbs, size_t tbslen) {
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
const size_t key_len = EVP_PKEY_size(ctx->pkey);
if (!sig) {
*siglen = key_len;
return 1;
}
if (*siglen < key_len) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_BUFFER_TOO_SMALL);
return 0;
}
if (rctx->md) {
unsigned int out_len;
if (tbslen != EVP_MD_size(rctx->md)) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_INVALID_DIGEST_LENGTH);
return 0;
}
if (EVP_MD_type(rctx->md) == NID_mdc2) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_NO_MDC2_SUPPORT);
return 0;
}
switch (rctx->pad_mode) {
case RSA_PKCS1_PADDING:
if (!RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig, &out_len, rsa)) {
return 0;
}
*siglen = out_len;
return 1;
case RSA_PKCS1_PSS_PADDING:
if (!setup_tbuf(rctx, ctx) ||
!RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf, tbs, rctx->md,
rctx->mgf1md, rctx->saltlen) ||
!RSA_sign_raw(rsa, siglen, sig, *siglen, rctx->tbuf, key_len,
RSA_NO_PADDING)) {
return 0;
}
return 1;
default:
return 0;
}
}
return RSA_sign_raw(rsa, siglen, sig, *siglen, tbs, tbslen, rctx->pad_mode);
}
int RSA_sign_pss_mgf1(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, const EVP_MD *md,
const EVP_MD *mgf1_md, int salt_len) {
if (in_len != EVP_MD_size(md)) {
OPENSSL_PUT_ERROR(RSA, RSA_R_INVALID_MESSAGE_LENGTH);
return 0;
}
size_t padded_len = RSA_size(rsa);
uint8_t *padded = OPENSSL_malloc(padded_len);
if (padded == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
return 0;
}
int ret =
RSA_padding_add_PKCS1_PSS_mgf1(rsa, padded, in, md, mgf1_md, salt_len) &&
RSA_sign_raw(rsa, out_len, out, max_out, padded, padded_len,
RSA_NO_PADDING);
OPENSSL_free(padded);
return ret;
}
int RSA_padding_add_PKCS1_PSS(RSA *rsa, unsigned char *EM,
const unsigned char *mHash,
const EVP_MD *Hash, int sLen)
{
return RSA_padding_add_PKCS1_PSS_mgf1(rsa, EM, mHash, Hash, NULL, sLen);
}
int FIPS_rsa_sign_digest(RSA *rsa, const unsigned char *md, int md_len,
const EVP_MD *mhash, int rsa_pad_mode, int saltlen,
const EVP_MD *mgf1Hash,
unsigned char *sigret, unsigned int *siglen)
{
int i=0,j,ret=0;
unsigned int dlen;
const unsigned char *der;
int md_type;
/* Largest DigestInfo: 19 (max encoding) + max MD */
unsigned char tmpdinfo[19 + EVP_MAX_MD_SIZE];
if (FIPS_selftest_failed())
{
FIPSerr(FIPS_F_FIPS_RSA_SIGN_DIGEST, FIPS_R_SELFTEST_FAILED);
return 0;
}
if (!mhash && rsa_pad_mode == RSA_PKCS1_PADDING)
md_type = saltlen;
else
md_type = M_EVP_MD_type(mhash);
if (rsa_pad_mode == RSA_X931_PADDING)
{
int hash_id;
memcpy(tmpdinfo, md, md_len);
hash_id = RSA_X931_hash_id(md_type);
if (hash_id == -1)
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
tmpdinfo[md_len] = (unsigned char)hash_id;
i = md_len + 1;
}
else if (rsa_pad_mode == RSA_PKCS1_PADDING)
{
der = fips_digestinfo_encoding(md_type, &dlen);
if (!der)
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,RSA_R_UNKNOWN_ALGORITHM_TYPE);
return 0;
}
memcpy(tmpdinfo, der, dlen);
memcpy(tmpdinfo + dlen, md, md_len);
i = dlen + md_len;
}
else if (rsa_pad_mode == RSA_PKCS1_PSS_PADDING)
{
unsigned char *sbuf;
i = RSA_size(rsa);
sbuf = OPENSSL_malloc(RSA_size(rsa));
if (!sbuf)
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,ERR_R_MALLOC_FAILURE);
goto psserr;
}
if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, sbuf, md, mhash,
mgf1Hash, saltlen))
goto psserr;
j=rsa->meth->rsa_priv_enc(i,sbuf,sigret,rsa,RSA_NO_PADDING);
if (j > 0)
{
ret=1;
*siglen=j;
}
psserr:
OPENSSL_cleanse(sbuf, i);
OPENSSL_free(sbuf);
return ret;
}
j=RSA_size(rsa);
if (i > (j-RSA_PKCS1_PADDING_SIZE))
{
RSAerr(RSA_F_FIPS_RSA_SIGN_DIGEST,RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
goto done;
}
/* NB: call underlying method directly to avoid FIPS blocking */
j=rsa->meth->rsa_priv_enc(i,tmpdinfo,sigret,rsa,rsa_pad_mode);
if (j > 0)
{
ret=1;
*siglen=j;
}
done:
OPENSSL_cleanse(tmpdinfo,i);
return ret;
}
static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
const unsigned char *tbs, size_t tbslen)
{
int ret;
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
#ifdef OPENSSL_FIPS
ret = pkey_fips_check_ctx(ctx);
if (ret < 0)
{
RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE);
return -1;
}
#endif
if (rctx->md)
{
if (tbslen != (size_t)EVP_MD_size(rctx->md))
{
RSAerr(RSA_F_PKEY_RSA_SIGN,
RSA_R_INVALID_DIGEST_LENGTH);
return -1;
}
#ifdef OPENSSL_FIPS
if (ret > 0)
{
unsigned int slen;
ret = FIPS_rsa_sign_digest(rsa, tbs, tbslen, rctx->md,
rctx->pad_mode,
rctx->saltlen,
rctx->mgf1md,
sig, &slen);
if (ret > 0)
*siglen = slen;
else
*siglen = 0;
return ret;
}
#endif
if (EVP_MD_type(rctx->md) == NID_mdc2)
{
unsigned int sltmp;
if (rctx->pad_mode != RSA_PKCS1_PADDING)
return -1;
ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2,
tbs, tbslen, sig, &sltmp, rsa);
if (ret <= 0)
return ret;
ret = sltmp;
}
else if (rctx->pad_mode == RSA_X931_PADDING)
{
if (!setup_tbuf(rctx, ctx))
return -1;
memcpy(rctx->tbuf, tbs, tbslen);
rctx->tbuf[tbslen] =
RSA_X931_hash_id(EVP_MD_type(rctx->md));
ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
sig, rsa, RSA_X931_PADDING);
}
else if (rctx->pad_mode == RSA_PKCS1_PADDING)
{
unsigned int sltmp;
ret = RSA_sign(EVP_MD_type(rctx->md),
tbs, tbslen, sig, &sltmp, rsa);
if (ret <= 0)
return ret;
ret = sltmp;
}
else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
{
if (!setup_tbuf(rctx, ctx))
return -1;
if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa,
rctx->tbuf, tbs,
rctx->md, rctx->mgf1md,
rctx->saltlen))
return -1;
ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
sig, rsa, RSA_NO_PADDING);
}
else
return -1;
}
else
ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
rctx->pad_mode);
if (ret < 0)
return ret;
*siglen = ret;
return 1;
}
