/* signature verification */
static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM *f,*ret;
int i,num=0,r= -1;
unsigned char *p;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
if((ctx = BN_CTX_new()) == NULL) goto err;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
ret = BN_CTX_get(ctx);
num=BN_num_bytes(rsa->n);
buf = OPENSSL_malloc(num);
if(!f || !ret || !buf)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
/* This check was for equality but PGP does evil things
* and chops off the top '0' bytes */
if (flen > num)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
goto err;
}
if (BN_bin2bn(from,flen,f) == NULL) goto err;
if (BN_ucmp(f, rsa->n) >= 0)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
MONT_HELPER(rsa, ctx, n, rsa->flags & RSA_FLAG_CACHE_PUBLIC, goto err);
if (!rsa->meth->bn_mod_exp(ret,f,rsa->e,rsa->n,ctx,
rsa->_method_mod_n)) goto err;
if ((padding == RSA_X931_PADDING) && ((ret->d[0] & 0xf) != 12))
BN_sub(ret, rsa->n, ret);
p=buf;
i=BN_bn2bin(ret,p);
switch (padding)
{
case RSA_PKCS1_PADDING:
r=RSA_padding_check_PKCS1_type_1(to,num,buf,i,num);
/* Generally signatures should be at least 2/3 padding, though
this isn't possible for really short keys and some standard
signature schemes, so don't check if the unpadded data is
small. */
if(r > 42 && 3*8*r >= BN_num_bits(rsa->n))
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_PKCS1_PADDING_TOO_SHORT);
goto err;
}
break;
case RSA_X931_PADDING:
r=RSA_padding_check_X931(to,num,buf,i,num);
break;
case RSA_NO_PADDING:
r=RSA_padding_check_none(to,num,buf,i,num);
break;
default:
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0)
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
err:
if (ctx != NULL)
{
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL)
{
OPENSSL_cleanse(buf,num);
OPENSSL_free(buf);
}
return(r);
}
int RSA_verify_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
if (rsa->n == NULL || rsa->e == NULL) {
OPENSSL_PUT_ERROR(RSA, RSA_R_VALUE_MISSING);
return 0;
}
const unsigned rsa_size = RSA_size(rsa);
BIGNUM *f, *result;
int r = -1;
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
if (in_len != rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN);
return 0;
}
if (!check_modulus_and_exponent_sizes(rsa)) {
return 0;
}
BN_CTX *ctx = BN_CTX_new();
if (ctx == NULL) {
return 0;
}
int ret = 0;
uint8_t *buf = NULL;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
result = BN_CTX_get(ctx);
if (f == NULL || result == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
if (padding == RSA_NO_PADDING) {
buf = out;
} else {
/* Allocate a temporary buffer to hold the padded plaintext. */
buf = OPENSSL_malloc(rsa_size);
if (buf == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
}
if (BN_bin2bn(in, in_len, f) == NULL) {
goto err;
}
if (BN_ucmp(f, rsa->n) >= 0) {
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (!BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx) ||
!BN_mod_exp_mont(result, f, rsa->e, rsa->n, ctx, rsa->mont_n)) {
goto err;
}
if (!BN_bn2bin_padded(buf, rsa_size, result)) {
OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
r = RSA_padding_check_PKCS1_type_1(out, rsa_size, buf, rsa_size);
break;
case RSA_NO_PADDING:
r = rsa_size;
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0) {
OPENSSL_PUT_ERROR(RSA, RSA_R_PADDING_CHECK_FAILED);
} else {
*out_len = r;
ret = 1;
}
err:
BN_CTX_end(ctx);
BN_CTX_free(ctx);
if (buf != out) {
OPENSSL_free(buf);
}
return ret;
}
int rsa_default_verify_raw(RSA *rsa, size_t *out_len, uint8_t *out,
size_t max_out, const uint8_t *in, size_t in_len,
int padding) {
const unsigned rsa_size = RSA_size(rsa);
BIGNUM *f, *result;
int ret = 0;
int r = -1;
uint8_t *buf = NULL;
BN_CTX *ctx = NULL;
if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS) {
OPENSSL_PUT_ERROR(RSA, RSA_R_MODULUS_TOO_LARGE);
return 0;
}
if (BN_ucmp(rsa->n, rsa->e) <= 0) {
OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_E_VALUE);
return 0;
}
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
/* for large moduli, enforce exponent limit */
if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS &&
BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) {
OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_E_VALUE);
return 0;
}
ctx = BN_CTX_new();
if (ctx == NULL) {
goto err;
}
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
result = BN_CTX_get(ctx);
if (padding == RSA_NO_PADDING) {
buf = out;
} else {
/* Allocate a temporary buffer to hold the padded plaintext. */
buf = OPENSSL_malloc(rsa_size);
if (buf == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
}
if (!f || !result) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
if (in_len != rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN);
goto err;
}
if (BN_bin2bn(in, in_len, f) == NULL) {
goto err;
}
if (BN_ucmp(f, rsa->n) >= 0) {
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) {
if (BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx) == NULL) {
goto err;
}
}
if (!rsa->meth->bn_mod_exp(result, f, rsa->e, rsa->n, ctx, rsa->mont_n)) {
goto err;
}
if (!BN_bn2bin_padded(buf, rsa_size, result)) {
OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
r = RSA_padding_check_PKCS1_type_1(out, rsa_size, buf, rsa_size);
break;
case RSA_NO_PADDING:
r = rsa_size;
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0) {
OPENSSL_PUT_ERROR(RSA, RSA_R_PADDING_CHECK_FAILED);
} else {
*out_len = r;
ret = 1;
}
err:
if (ctx != NULL) {
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (padding != RSA_NO_PADDING && buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
return ret;
}
/* signature verification */
static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM *f,*ret;
int i,num=0,r= -1;
unsigned char *p;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_MODULUS_TOO_LARGE);
return -1;
}
if (BN_ucmp(rsa->n, rsa->e) <= 0)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE);
return -1;
}
/* for large moduli, enforce exponent limit */
if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS)
{
if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE);
return -1;
}
}
if((ctx = BN_CTX_new()) == NULL) goto err;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
ret = BN_CTX_get(ctx);
num=BN_num_bytes(rsa->n);
buf = OPENSSL_malloc(num);
if(!f || !ret || !buf)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
/* This check was for equality but PGP does evil things
* and chops off the top '0' bytes */
if (flen > num)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
goto err;
}
if (BN_bin2bn(from,flen,f) == NULL) goto err;
if (BN_ucmp(f, rsa->n) >= 0)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (rsa->flags & RSA_FLAG_CACHE_PUBLIC)
if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n, ctx))
goto err;
if (!rsa->meth->bn_mod_exp(ret,f,rsa->e,rsa->n,ctx,
rsa->_method_mod_n)) goto err;
if ((padding == RSA_X931_PADDING) && ((ret->d[0] & 0xf) != 12))
BN_sub(ret, rsa->n, ret);
p=buf;
i=BN_bn2bin(ret,p);
switch (padding)
{
case RSA_PKCS1_PADDING:
r=RSA_padding_check_PKCS1_type_1(to,num,buf,i,num);
break;
case RSA_X931_PADDING:
r=RSA_padding_check_X931(to,num,buf,i,num);
break;
case RSA_NO_PADDING:
r=RSA_padding_check_none(to,num,buf,i,num);
break;
default:
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0)
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
err:
if (ctx != NULL)
{
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL)
{
OPENSSL_cleanse(buf,num);
OPENSSL_free(buf);
}
return(r);
}
static int RSA_eay_public_decrypt(int flen, unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM f,ret;
int i,num=0,r= -1;
unsigned char *p;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
BN_init(&f);
BN_init(&ret);
ctx=BN_CTX_new();
if (ctx == NULL) goto err;
num=BN_num_bytes(rsa->n);
buf=(unsigned char *)Malloc(num);
if (buf == NULL)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
/* This check was for equality but PGP does evil things
* and chops off the top '0' bytes */
if (flen > num)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
goto err;
}
if (BN_bin2bn(from,flen,&f) == NULL) goto err;
/* do the decrypt */
if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
{
if ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)
if (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))
goto err;
}
if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
rsa->_method_mod_n)) goto err;
p=buf;
i=BN_bn2bin(&ret,p);
switch (padding)
{
case RSA_PKCS1_PADDING:
r=RSA_padding_check_PKCS1_type_1(to,num,buf,i,num);
break;
case RSA_NO_PADDING:
r=RSA_padding_check_none(to,num,buf,i,num);
break;
default:
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0)
RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
err:
if (ctx != NULL) BN_CTX_free(ctx);
BN_clear_free(&f);
BN_clear_free(&ret);
if (buf != NULL)
{
memset(buf,0,num);
Free(buf);
}
return(r);
}
