static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen,
const uint8_t *tbs, size_t tbslen) {
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
const size_t key_len = EVP_PKEY_size(ctx->pkey);
if (!sig) {
*siglen = key_len;
return 1;
}
if (*siglen < key_len) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_BUFFER_TOO_SMALL);
return 0;
}
if (rctx->md) {
unsigned int out_len;
if (tbslen != EVP_MD_size(rctx->md)) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_INVALID_DIGEST_LENGTH);
return 0;
}
if (EVP_MD_type(rctx->md) == NID_mdc2) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_NO_MDC2_SUPPORT);
return 0;
}
switch (rctx->pad_mode) {
case RSA_PKCS1_PADDING:
if (!RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig, &out_len, rsa)) {
return 0;
}
*siglen = out_len;
return 1;
case RSA_PKCS1_PSS_PADDING:
if (!setup_tbuf(rctx, ctx) ||
!RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf, tbs, rctx->md,
rctx->mgf1md, rctx->saltlen) ||
!RSA_sign_raw(rsa, siglen, sig, *siglen, rctx->tbuf, key_len,
RSA_NO_PADDING)) {
return 0;
}
return 1;
default:
return 0;
}
}
return RSA_sign_raw(rsa, siglen, sig, *siglen, tbs, tbslen, rctx->pad_mode);
}
int RSA_sign(int hash_nid, const uint8_t *in, unsigned in_len, uint8_t *out,
unsigned *out_len, RSA *rsa) {
const unsigned rsa_size = RSA_size(rsa);
int ret = 0;
uint8_t *signed_msg = NULL;
size_t signed_msg_len = 0;
int signed_msg_is_alloced = 0;
size_t size_t_out_len;
if (rsa->meth->sign) {
return rsa->meth->sign(hash_nid, in, in_len, out, out_len, rsa);
}
if (!RSA_add_pkcs1_prefix(&signed_msg, &signed_msg_len,
&signed_msg_is_alloced, hash_nid, in, in_len) ||
!RSA_sign_raw(rsa, &size_t_out_len, out, rsa_size, signed_msg,
signed_msg_len, RSA_PKCS1_PADDING)) {
goto err;
}
*out_len = size_t_out_len;
ret = 1;
err:
if (signed_msg_is_alloced) {
OPENSSL_free(signed_msg);
}
return ret;
}
int RSA_private_encrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
int padding) {
size_t out_len;
if (!RSA_sign_raw(rsa, &out_len, to, RSA_size(rsa), from, flen, padding)) {
return -1;
}
return out_len;
}
int RSA_private_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
int padding) {
size_t out_len;
if (!RSA_sign_raw(rsa, &out_len, to, RSA_size(rsa), from, flen, padding)) {
return -1;
}
if (out_len > INT_MAX) {
OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
return -1;
}
return out_len;
}
int RSA_sign_pss_mgf1(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, const EVP_MD *md,
const EVP_MD *mgf1_md, int salt_len) {
if (in_len != EVP_MD_size(md)) {
OPENSSL_PUT_ERROR(RSA, RSA_R_INVALID_MESSAGE_LENGTH);
return 0;
}
size_t padded_len = RSA_size(rsa);
uint8_t *padded = OPENSSL_malloc(padded_len);
if (padded == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
return 0;
}
int ret =
RSA_padding_add_PKCS1_PSS_mgf1(rsa, padded, in, md, mgf1_md, salt_len) &&
RSA_sign_raw(rsa, out_len, out, max_out, padded, padded_len,
RSA_NO_PADDING);
OPENSSL_free(padded);
return ret;
}
int RSA_sign(int hash_nid, const uint8_t *in, unsigned in_len, uint8_t *out,
unsigned *out_len, RSA *rsa) {
const unsigned rsa_size = RSA_size(rsa);
int ret = 0;
uint8_t *signed_msg;
size_t signed_msg_len;
int signed_msg_is_alloced = 0;
size_t size_t_out_len;
if (rsa->meth->sign) {
return rsa->meth->sign(hash_nid, in, in_len, out, out_len, rsa);
}
if (!pkcs1_prefixed_msg(&signed_msg, &signed_msg_len, &signed_msg_is_alloced,
hash_nid, in, in_len)) {
return 0;
}
if (rsa_size < RSA_PKCS1_PADDING_SIZE ||
signed_msg_len > rsa_size - RSA_PKCS1_PADDING_SIZE) {
OPENSSL_PUT_ERROR(RSA, RSA_sign, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
goto finish;
}
if (RSA_sign_raw(rsa, &size_t_out_len, out, rsa_size, signed_msg,
signed_msg_len, RSA_PKCS1_PADDING)) {
*out_len = size_t_out_len;
ret = 1;
}
finish:
if (signed_msg_is_alloced) {
OPENSSL_free(signed_msg);
}
return ret;
}