int keystore_rsa_priv_enc(int flen, const unsigned char* from, unsigned char* to, RSA* rsa,
int padding) {
ALOGV("keystore_rsa_sign(%d, %p, %p, %p, %d)", flen, from, to, rsa, padding);
int num = RSA_size(rsa);
UniquePtr<uint8_t> padded(new uint8_t[num]);
if (padded.get() == NULL) {
ALOGE("could not allocate padded signature");
return 0;
}
switch (padding) {
case RSA_PKCS1_PADDING:
if (!RSA_padding_add_PKCS1_type_1(padded.get(), num, from, flen)) {
return 0;
}
break;
case RSA_X931_PADDING:
if (!RSA_padding_add_X931(padded.get(), num, from, flen)) {
return 0;
}
break;
case RSA_NO_PADDING:
if (!RSA_padding_add_none(padded.get(), num, from, flen)) {
return 0;
}
break;
default:
ALOGE("Unknown padding type: %d", padding);
return 0;
}
uint8_t* key_id = reinterpret_cast<uint8_t*>(RSA_get_ex_data(rsa, rsa_key_handle));
if (key_id == NULL) {
ALOGE("key had no key_id!");
return 0;
}
Keystore_Reply reply;
if (keystore_cmd(CommandCodes[SIGN], &reply, 2, strlen(reinterpret_cast<const char*>(key_id)),
key_id, static_cast<size_t>(num), reinterpret_cast<const uint8_t*>(padded.get()))
!= NO_ERROR) {
ALOGE("There was an error during rsa_mod_exp");
return 0;
}
const size_t replyLen = reply.length();
if (replyLen <= 0) {
ALOGW("No valid signature returned");
return 0;
}
memcpy(to, reply.get(), replyLen);
ALOGV("rsa=%p keystore_rsa_sign => returning %p len %llu", rsa, to,
(unsigned long long) replyLen);
return static_cast<int>(replyLen);
}
/* signing */
int rsa_default_sign_raw(RSA *rsa, size_t *out_len, uint8_t *out,
size_t max_out, const uint8_t *in, size_t in_len,
int padding) {
const unsigned rsa_size = RSA_size(rsa);
uint8_t *buf = NULL;
int i, ret = 0;
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
buf = OPENSSL_malloc(rsa_size);
if (buf == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
i = RSA_padding_add_PKCS1_type_1(buf, rsa_size, in, in_len);
break;
case RSA_NO_PADDING:
i = RSA_padding_add_none(buf, rsa_size, in, in_len);
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) {
goto err;
}
if (!RSA_private_transform(rsa, out, buf, rsa_size)) {
goto err;
}
*out_len = rsa_size;
ret = 1;
err:
if (buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
return ret;
}
/* signing */
static int RSA_eay_private_encrypt(FIPS_RSA_SIZE_T flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM f,ret;
int i,j,k,num=0,r= -1;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
int local_blinding = 0;
BN_BLINDING *blinding = NULL;
BN_init(&f);
BN_init(&ret);
if ((ctx=BN_CTX_new()) == NULL) goto err;
num=BN_num_bytes(rsa->n);
if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding)
{
case RSA_PKCS1_PADDING:
i=RSA_padding_add_PKCS1_type_1(buf,num,from,flen);
break;
case RSA_NO_PADDING:
i=RSA_padding_add_none(buf,num,from,flen);
break;
case RSA_SSLV23_PADDING:
default:
RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) goto err;
if (BN_bin2bn(buf,num,&f) == NULL) goto err;
if (BN_ucmp(&f, rsa->n) >= 0)
{
/* usually the padding functions would catch this */
RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
BLINDING_HELPER(rsa, ctx, goto err;);
/* signing */
static int RSA_eay_private_encrypt(int flen, unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
const RSA_METHOD *meth;
BIGNUM f,ret;
int i,j,k,num=0,r= -1;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
int local_blinding = 0;
BN_BLINDING *blinding = NULL;
meth = rsa->meth;
BN_init(&f);
BN_init(&ret);
if ((ctx=BN_CTX_new()) == NULL) goto err;
num=BN_num_bytes(rsa->n);
if ((buf=(unsigned char *)rtlglue_malloc(num)) == NULL)
goto err;
switch (padding)
{
case RSA_PKCS1_PADDING:
i=RSA_padding_add_PKCS1_type_1(buf,num,from,flen);
break;
case RSA_NO_PADDING:
i=RSA_padding_add_none(buf,num,from,flen);
break;
case RSA_SSLV23_PADDING:
default:
goto err;
}
if (i <= 0) goto err;
if (BN_bin2bn(buf,num,&f) == NULL) goto err;
if (BN_ucmp(&f, rsa->n) >= 0)
goto err;
BLINDING_HELPER(rsa, ctx, goto err;);
/* signing */
static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM *f, *ret, *br, *res;
int i,j,k,num=0,r= -1;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
int local_blinding = 0;
BN_BLINDING *blinding = NULL;
if ((ctx=BN_CTX_new()) == NULL) goto err;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
br = BN_CTX_get(ctx);
ret = BN_CTX_get(ctx);
num = BN_num_bytes(rsa->n);
buf = OPENSSL_malloc(num);
if(!f || !ret || !buf)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding)
{
case RSA_PKCS1_PADDING:
i=RSA_padding_add_PKCS1_type_1(buf,num,from,flen);
break;
case RSA_X931_PADDING:
i=RSA_padding_add_X931(buf,num,from,flen);
break;
case RSA_NO_PADDING:
i=RSA_padding_add_none(buf,num,from,flen);
break;
case RSA_SSLV23_PADDING:
default:
RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) goto err;
if (BN_bin2bn(buf,num,f) == NULL) goto err;
if (BN_ucmp(f, rsa->n) >= 0)
{
/* usually the padding functions would catch this */
RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (!(rsa->flags & RSA_FLAG_NO_BLINDING))
{
blinding = rsa_get_blinding(rsa, &br, &local_blinding, ctx);
if (blinding == NULL)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT, ERR_R_INTERNAL_ERROR);
goto err;
}
}
if (blinding != NULL)
if (!rsa_blinding_convert(blinding, local_blinding, f, br, ctx))
goto err;
if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
((rsa->p != NULL) &&
(rsa->q != NULL) &&
(rsa->dmp1 != NULL) &&
(rsa->dmq1 != NULL) &&
(rsa->iqmp != NULL)) )
{
if (!rsa->meth->rsa_mod_exp(ret, f, rsa, ctx)) goto err;
}
else
{
BIGNUM local_d;
BIGNUM *d = NULL;
if (!(rsa->flags & RSA_FLAG_NO_EXP_CONSTTIME))
{
BN_init(&local_d);
d = &local_d;
BN_with_flags(d, rsa->d, BN_FLG_EXP_CONSTTIME);
}
else
d = rsa->d;
MONT_HELPER(rsa, ctx, n, rsa->flags & RSA_FLAG_CACHE_PUBLIC, goto err);
if (!rsa->meth->bn_mod_exp(ret,f,d,rsa->n,ctx,
rsa->_method_mod_n)) goto err;
}
if (blinding)
if (!rsa_blinding_invert(blinding, local_blinding, ret, br, ctx))
goto err;
if (padding == RSA_X931_PADDING)
{
BN_sub(f, rsa->n, ret);
if (BN_cmp(ret, f))
res = f;
else
res = ret;
}
else
res = ret;
/* put in leading 0 bytes if the number is less than the
* length of the modulus */
j=BN_num_bytes(res);
i=BN_bn2bin(res,&(to[num-j]));
for (k=0; k<(num-i); k++)
to[k]=0;
r=num;
err:
if (ctx != NULL)
{
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL)
{
OPENSSL_cleanse(buf,num);
OPENSSL_free(buf);
}
return(r);
}
static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM *f,*ret;
int i,j,k,num=0,r= -1;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
if ((ctx=BN_CTX_new()) == NULL) goto err;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
ret = BN_CTX_get(ctx);
num=BN_num_bytes(rsa->n);
buf = OPENSSL_malloc(num);
if (!f || !ret || !buf)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding)
{
case RSA_PKCS1_PADDING:
i=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);
break;
#ifndef OPENSSL_NO_SHA
case RSA_PKCS1_OAEP_PADDING:
i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);
break;
#endif
case RSA_SSLV23_PADDING:
i=RSA_padding_add_SSLv23(buf,num,from,flen);
break;
case RSA_NO_PADDING:
i=RSA_padding_add_none(buf,num,from,flen);
break;
default:
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) goto err;
if (BN_bin2bn(buf,num,f) == NULL) goto err;
if (BN_ucmp(f, rsa->n) >= 0)
{
/* usually the padding functions would catch this */
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
MONT_HELPER(rsa, ctx, n, rsa->flags & RSA_FLAG_CACHE_PUBLIC, goto err);
if (!rsa->meth->bn_mod_exp(ret,f,rsa->e,rsa->n,ctx,
rsa->_method_mod_n)) goto err;
/* put in leading 0 bytes if the number is less than the
* length of the modulus */
j=BN_num_bytes(ret);
i=BN_bn2bin(ret,&(to[num-j]));
for (k=0; k<(num-i); k++)
to[k]=0;
r=num;
err:
if (ctx != NULL)
{
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL)
{
OPENSSL_cleanse(buf,num);
OPENSSL_free(buf);
}
return(r);
}
static int RSA_eay_public_encrypt(int flen, unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
const RSA_METHOD *meth;
BIGNUM f,ret;
int i,j,k,num=0,r= -1;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
meth = rsa->meth;
BN_init(&f);
BN_init(&ret);
if ((ctx=BN_CTX_new()) == NULL) goto err;
num=BN_num_bytes(rsa->n);
if ((buf=(unsigned char *)rtlglue_malloc(num)) == NULL)
goto err;
switch (padding)
{
case RSA_PKCS1_PADDING:
i=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);
break;
case RSA_SSLV23_PADDING:
i=RSA_padding_add_SSLv23(buf,num,from,flen);
break;
case RSA_NO_PADDING:
i=RSA_padding_add_none(buf,num,from,flen);
break;
default:
goto err;
}
if (i <= 0) goto err;
if (BN_bin2bn(buf,num,&f) == NULL) goto err;
if (BN_ucmp(&f, rsa->n) >= 0)
goto err;
if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
{
BN_MONT_CTX* bn_mont_ctx;
if ((bn_mont_ctx=BN_MONT_CTX_new()) == NULL)
goto err;
if (!BN_MONT_CTX_set(bn_mont_ctx,rsa->n,ctx))
{
BN_MONT_CTX_free(bn_mont_ctx);
goto err;
}
if (rsa->_method_mod_n == NULL) /* other thread may have finished first */
{
if (rsa->_method_mod_n == NULL)
{
rsa->_method_mod_n = bn_mont_ctx;
bn_mont_ctx = NULL;
}
}
if (bn_mont_ctx)
BN_MONT_CTX_free(bn_mont_ctx);
}
if (!meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
rsa->_method_mod_n)) goto err;
/* put in leading 0 bytes if the number is less than the
* length of the modulus */
j=BN_num_bytes(&ret);
i=BN_bn2bin(&ret,&(to[num-j]));
for (k=0; k<(num-i); k++)
to[k]=0;
r=num;
err:
if (ctx != NULL) BN_CTX_free(ctx);
BN_clear_free(&f);
BN_clear_free(&ret);
if (buf != NULL)
{
memset(buf, 0, num);
rtlglue_free(buf);
}
return(r);
}
int rsa_default_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
const unsigned rsa_size = RSA_size(rsa);
BIGNUM *f, *result;
uint8_t *buf = NULL;
BN_CTX *ctx = NULL;
int i, ret = 0;
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
if (!check_modulus_and_exponent_sizes(rsa)) {
return 0;
}
ctx = BN_CTX_new();
if (ctx == NULL) {
goto err;
}
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
result = BN_CTX_get(ctx);
buf = OPENSSL_malloc(rsa_size);
if (!f || !result || !buf) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
i = RSA_padding_add_PKCS1_type_2(buf, rsa_size, in, in_len);
break;
case RSA_PKCS1_OAEP_PADDING:
/* Use the default parameters: SHA-1 for both hashes and no label. */
i = RSA_padding_add_PKCS1_OAEP_mgf1(buf, rsa_size, in, in_len,
NULL, 0, NULL, NULL);
break;
case RSA_NO_PADDING:
i = RSA_padding_add_none(buf, rsa_size, in, in_len);
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) {
goto err;
}
if (BN_bin2bn(buf, rsa_size, f) == NULL) {
goto err;
}
if (BN_ucmp(f, rsa->n) >= 0) {
/* usually the padding functions would catch this */
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (!BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx) ||
!BN_mod_exp_mont(result, f, rsa->e, rsa->n, ctx, rsa->mont_n)) {
goto err;
}
/* put in leading 0 bytes if the number is less than the length of the
* modulus */
if (!BN_bn2bin_padded(out, rsa_size, result)) {
OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
goto err;
}
*out_len = rsa_size;
ret = 1;
err:
if (ctx != NULL) {
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
return ret;
}
static int my_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
const unsigned rsa_size = RSA_size(rsa);
BIGNUM *f, *result;
uint8_t *buf = NULL;
BN_CTX *ctx = NULL;
int i, ret = 0;
printf("my encrypt\n");
if (rsa_size > OPENSSL_RSA_MAX_MODULUS_BITS) {
OPENSSL_PUT_ERROR(RSA, RSA_R_MODULUS_TOO_LARGE);
return 0;
}
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
if (BN_ucmp(rsa->n, rsa->e) <= 0) {
OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_E_VALUE);
return 0;
}
/* for large moduli, enforce exponent limit */
if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS &&
BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) {
OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_E_VALUE);
return 0;
}
ctx = BN_CTX_new();
if (ctx == NULL) {
goto err;
}
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
result = BN_CTX_get(ctx);
buf = OPENSSL_malloc(rsa_size);
if (!f || !result || !buf) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
i = RSA_padding_add_PKCS1_type_2(buf, rsa_size, in, in_len);
break;
case RSA_PKCS1_OAEP_PADDING:
/* Use the default parameters: SHA-1 for both hashes and no label. */
i = RSA_padding_add_PKCS1_OAEP_mgf1(buf, rsa_size, in, in_len,
NULL, 0, NULL, NULL);
break;
case RSA_NO_PADDING:
i = RSA_padding_add_none(buf, rsa_size, in, in_len);
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) {
goto err;
}
if (BN_bin2bn(buf, rsa_size, f) == NULL) {
goto err;
}
if (BN_ucmp(f, rsa->n) >= 0) {
/* usually the padding functions would catch this */
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) {
if (BN_MONT_CTX_set_locked(&rsa->_method_mod_n, &rsa->lock, rsa->n, ctx) ==
NULL) {
goto err;
}
}
printf("before my bn_mod_exp\n");
if (!rsa->meth->bn_mod_exp(result, f, rsa->e, rsa->n, ctx,
rsa->_method_mod_n)) {
goto err;
}
printf("after my bn_mod_exp\n");
/* put in leading 0 bytes if the number is less than the length of the
* modulus */
if (!BN_bn2bin_padded(out, rsa_size, result)) {
OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
goto err;
}
*out_len = rsa_size;
ret = 1;
err:
if (ctx != NULL) {
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
return ret;
}
static int RSA_eay_public_encrypt(FIPS_RSA_SIZE_T flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM f,ret;
int i,j,k,num=0,r= -1;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
BN_init(&f);
BN_init(&ret);
if(FIPS_selftest_failed())
{
FIPSerr(FIPS_F_RSA_EAY_PUBLIC_ENCRYPT,FIPS_R_FIPS_SELFTEST_FAILED);
goto err;
}
if ((ctx=BN_CTX_new()) == NULL) goto err;
num=BN_num_bytes(rsa->n);
if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding)
{
case RSA_PKCS1_PADDING:
i=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);
break;
#ifndef OPENSSL_NO_SHA
case RSA_PKCS1_OAEP_PADDING:
i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);
break;
#endif
case RSA_SSLV23_PADDING:
i=RSA_padding_add_SSLv23(buf,num,from,flen);
break;
case RSA_NO_PADDING:
i=RSA_padding_add_none(buf,num,from,flen);
break;
default:
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) goto err;
if (BN_bin2bn(buf,num,&f) == NULL) goto err;
if (BN_ucmp(&f, rsa->n) >= 0)
{
/* usually the padding functions would catch this */
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
{
BN_MONT_CTX* bn_mont_ctx;
if ((bn_mont_ctx=BN_MONT_CTX_new()) == NULL)
goto err;
if (!BN_MONT_CTX_set(bn_mont_ctx,rsa->n,ctx))
{
BN_MONT_CTX_free(bn_mont_ctx);
goto err;
}
if (rsa->_method_mod_n == NULL) /* other thread may have finished first */
{
CRYPTO_w_lock(CRYPTO_LOCK_RSA);
if (rsa->_method_mod_n == NULL)
{
rsa->_method_mod_n = bn_mont_ctx;
bn_mont_ctx = NULL;
}
CRYPTO_w_unlock(CRYPTO_LOCK_RSA);
}
if (bn_mont_ctx)
BN_MONT_CTX_free(bn_mont_ctx);
}
if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
rsa->_method_mod_n)) goto err;
/* put in leading 0 bytes if the number is less than the
* length of the modulus */
j=BN_num_bytes(&ret);
i=BN_bn2bin(&ret,&(to[num-j]));
for (k=0; k<(num-i); k++)
to[k]=0;
r=num;
err:
if (ctx != NULL) BN_CTX_free(ctx);
BN_clear_free(&f);
BN_clear_free(&ret);
if (buf != NULL)
{
OPENSSL_cleanse(buf,num);
OPENSSL_free(buf);
}
return(r);
}
static int rsa_ossl_public_encrypt(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM *f, *ret;
int i, j, k, num = 0, r = -1;
unsigned char *buf = NULL;
BN_CTX *ctx = NULL;
if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS) {
RSAerr(RSA_F_RSA_OSSL_PUBLIC_ENCRYPT, RSA_R_MODULUS_TOO_LARGE);
return -1;
}
if (BN_ucmp(rsa->n, rsa->e) <= 0) {
RSAerr(RSA_F_RSA_OSSL_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
return -1;
}
/* for large moduli, enforce exponent limit */
if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS) {
if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) {
RSAerr(RSA_F_RSA_OSSL_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
return -1;
}
}
if ((ctx = BN_CTX_new()) == NULL)
goto err;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
ret = BN_CTX_get(ctx);
num = BN_num_bytes(rsa->n);
buf = OPENSSL_malloc(num);
if (ret == NULL || buf == NULL) {
RSAerr(RSA_F_RSA_OSSL_PUBLIC_ENCRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
i = RSA_padding_add_PKCS1_type_2(buf, num, from, flen);
break;
case RSA_PKCS1_OAEP_PADDING:
i = RSA_padding_add_PKCS1_OAEP(buf, num, from, flen, NULL, 0);
break;
case RSA_SSLV23_PADDING:
i = RSA_padding_add_SSLv23(buf, num, from, flen);
break;
case RSA_NO_PADDING:
i = RSA_padding_add_none(buf, num, from, flen);
break;
default:
RSAerr(RSA_F_RSA_OSSL_PUBLIC_ENCRYPT, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0)
goto err;
if (BN_bin2bn(buf, num, f) == NULL)
goto err;
if (BN_ucmp(f, rsa->n) >= 0) {
/* usually the padding functions would catch this */
RSAerr(RSA_F_RSA_OSSL_PUBLIC_ENCRYPT,
RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (rsa->flags & RSA_FLAG_CACHE_PUBLIC)
if (!BN_MONT_CTX_set_locked
(&rsa->_method_mod_n, rsa->lock, rsa->n, ctx))
goto err;
if (!rsa->meth->bn_mod_exp(ret, f, rsa->e, rsa->n, ctx,
rsa->_method_mod_n))
goto err;
/*
* put in leading 0 bytes if the number is less than the length of the
* modulus
*/
j = BN_num_bytes(ret);
i = BN_bn2bin(ret, &(to[num - j]));
for (k = 0; k < (num - i); k++)
to[k] = 0;
r = num;
err:
if (ctx != NULL)
BN_CTX_end(ctx);
BN_CTX_free(ctx);
OPENSSL_clear_free(buf, num);
return r;
}
/* signing */
static int rsa_ossl_private_encrypt(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM *f, *ret, *res;
int i, j, k, num = 0, r = -1;
unsigned char *buf = NULL;
BN_CTX *ctx = NULL;
int local_blinding = 0;
/*
* Used only if the blinding structure is shared. A non-NULL unblind
* instructs rsa_blinding_convert() and rsa_blinding_invert() to store
* the unblinding factor outside the blinding structure.
*/
BIGNUM *unblind = NULL;
BN_BLINDING *blinding = NULL;
if ((ctx = BN_CTX_new()) == NULL)
goto err;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
ret = BN_CTX_get(ctx);
num = BN_num_bytes(rsa->n);
buf = OPENSSL_malloc(num);
if (ret == NULL || buf == NULL) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_ENCRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
i = RSA_padding_add_PKCS1_type_1(buf, num, from, flen);
break;
case RSA_X931_PADDING:
i = RSA_padding_add_X931(buf, num, from, flen);
break;
case RSA_NO_PADDING:
i = RSA_padding_add_none(buf, num, from, flen);
break;
case RSA_SSLV23_PADDING:
default:
RSAerr(RSA_F_RSA_OSSL_PRIVATE_ENCRYPT, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0)
goto err;
if (BN_bin2bn(buf, num, f) == NULL)
goto err;
if (BN_ucmp(f, rsa->n) >= 0) {
/* usually the padding functions would catch this */
RSAerr(RSA_F_RSA_OSSL_PRIVATE_ENCRYPT,
RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) {
blinding = rsa_get_blinding(rsa, &local_blinding, ctx);
if (blinding == NULL) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_ENCRYPT, ERR_R_INTERNAL_ERROR);
goto err;
}
}
if (blinding != NULL) {
if (!local_blinding && ((unblind = BN_CTX_get(ctx)) == NULL)) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_ENCRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
if (!rsa_blinding_convert(blinding, f, unblind, ctx))
goto err;
}
if ((rsa->flags & RSA_FLAG_EXT_PKEY) ||
(rsa->version == RSA_ASN1_VERSION_MULTI) ||
((rsa->p != NULL) &&
(rsa->q != NULL) &&
(rsa->dmp1 != NULL) && (rsa->dmq1 != NULL) && (rsa->iqmp != NULL))) {
if (!rsa->meth->rsa_mod_exp(ret, f, rsa, ctx))
goto err;
} else {
BIGNUM *d = BN_new();
if (d == NULL) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_ENCRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
if (rsa->flags & RSA_FLAG_CACHE_PUBLIC)
if (!BN_MONT_CTX_set_locked
(&rsa->_method_mod_n, rsa->lock, rsa->n, ctx)) {
BN_free(d);
goto err;
}
if (!rsa->meth->bn_mod_exp(ret, f, d, rsa->n, ctx,
rsa->_method_mod_n)) {
BN_free(d);
goto err;
}
/* We MUST free d before any further use of rsa->d */
BN_free(d);
}
if (blinding)
if (!rsa_blinding_invert(blinding, ret, unblind, ctx))
goto err;
if (padding == RSA_X931_PADDING) {
BN_sub(f, rsa->n, ret);
if (BN_cmp(ret, f) > 0)
res = f;
else
res = ret;
} else {
res = ret;
}
/*
* put in leading 0 bytes if the number is less than the length of the
* modulus
*/
j = BN_num_bytes(res);
i = BN_bn2bin(res, &(to[num - j]));
for (k = 0; k < (num - i); k++)
to[k] = 0;
r = num;
err:
if (ctx != NULL)
BN_CTX_end(ctx);
BN_CTX_free(ctx);
OPENSSL_clear_free(buf, num);
return r;
}
static int RSA_eay_private_encrypt(int flen, unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM f,ret;
int i,j,k,num=0,r= -1;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
BN_init(&f);
BN_init(&ret);
if ((ctx=BN_CTX_new()) == NULL) goto err;
num=BN_num_bytes(rsa->n);
if ((buf=(unsigned char *)Malloc(num)) == NULL)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding)
{
case RSA_PKCS1_PADDING:
i=RSA_padding_add_PKCS1_type_1(buf,num,from,flen);
break;
case RSA_NO_PADDING:
i=RSA_padding_add_none(buf,num,from,flen);
break;
case RSA_SSLV23_PADDING:
default:
RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) goto err;
if (BN_bin2bn(buf,num,&f) == NULL) goto err;
if ((rsa->flags & RSA_FLAG_BLINDING) && (RSA_get_thread_blinding_ptr(rsa) == NULL))
RSA_blinding_on(rsa,ctx);
if (rsa->flags & RSA_FLAG_BLINDING)
if (!BN_BLINDING_convert(&f,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err;
if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
((rsa->p != NULL) &&
(rsa->q != NULL) &&
(rsa->dmp1 != NULL) &&
(rsa->dmq1 != NULL) &&
(rsa->iqmp != NULL)) )
{ if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
else
{
if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL)) goto err;
}
if (rsa->flags & RSA_FLAG_BLINDING)
if (!BN_BLINDING_invert(&ret,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err;
/* put in leading 0 bytes if the number is less than the
* length of the modulus */
j=BN_num_bytes(&ret);
i=BN_bn2bin(&ret,&(to[num-j]));
for (k=0; k<(num-i); k++)
to[k]=0;
r=num;
err:
if (ctx != NULL) BN_CTX_free(ctx);
BN_clear_free(&ret);
BN_clear_free(&f);
if (buf != NULL)
{
memset(buf,0,num);
Free(buf);
}
return(r);
}
static int RSA_eay_public_encrypt(int flen, unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM f,ret;
int i,j,k,num=0,r= -1;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
BN_init(&f);
BN_init(&ret);
if ((ctx=BN_CTX_new()) == NULL) goto err;
num=BN_num_bytes(rsa->n);
if ((buf=(unsigned char *)Malloc(num)) == NULL)
{
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding)
{
case RSA_PKCS1_PADDING:
i=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);
break;
#ifndef _OPENSSL_APPLE_CDSA_
#ifndef NO_SHA
case RSA_PKCS1_OAEP_PADDING:
i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);
break;
#endif
#endif
case RSA_SSLV23_PADDING:
i=RSA_padding_add_SSLv23(buf,num,from,flen);
break;
case RSA_NO_PADDING:
i=RSA_padding_add_none(buf,num,from,flen);
break;
default:
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) goto err;
if (BN_bin2bn(buf,num,&f) == NULL) goto err;
if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
{
if ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)
if (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))
goto err;
}
if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
rsa->_method_mod_n)) goto err;
/* put in leading 0 bytes if the number is less than the
* length of the modulus */
j=BN_num_bytes(&ret);
i=BN_bn2bin(&ret,&(to[num-j]));
for (k=0; k<(num-i); k++)
to[k]=0;
r=num;
err:
if (ctx != NULL) BN_CTX_free(ctx);
BN_clear_free(&f);
BN_clear_free(&ret);
if (buf != NULL)
{
memset(buf,0,num);
Free(buf);
}
return(r);
}
