static int decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, const uint8_t *in, size_t in_len, int padding) { const unsigned rsa_size = RSA_size(rsa); int r = -1; uint8_t *buf = NULL; int ret = 0; if (max_out < rsa_size) { OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } buf = OPENSSL_malloc(rsa_size); if (buf == NULL) { OPENSSL_PUT_ERROR(RSA, decrypt, ERR_R_MALLOC_FAILURE); goto err; } if (in_len != rsa_size) { OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN); goto err; } if (!RSA_private_transform(rsa, buf, in, rsa_size)) { OPENSSL_PUT_ERROR(RSA, decrypt, ERR_R_INTERNAL_ERROR); goto err; } switch (padding) { case RSA_PKCS1_PADDING: r = RSA_padding_check_PKCS1_type_2(out, rsa_size, buf, rsa_size); break; case RSA_PKCS1_OAEP_PADDING: /* Use the default parameters: SHA-1 for both hashes and no label. */ r = RSA_padding_check_PKCS1_OAEP_mgf1(out, rsa_size, buf, rsa_size, NULL, 0, NULL, NULL); break; case RSA_NO_PADDING: r = RSA_padding_check_none(out, rsa_size, buf, rsa_size); break; default: OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) { OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_PADDING_CHECK_FAILED); } else { *out_len = r; ret = 1; } err: if (buf != NULL) { OPENSSL_cleanse(buf, rsa_size); OPENSSL_free(buf); } return ret; }
/* signature verification */ static int RSA_eay_public_decrypt(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { BIGNUM *f,*ret; int i,num=0,r= -1; unsigned char *p; unsigned char *buf=NULL; BN_CTX *ctx=NULL; if((ctx = BN_CTX_new()) == NULL) goto err; BN_CTX_start(ctx); f = BN_CTX_get(ctx); ret = BN_CTX_get(ctx); num=BN_num_bytes(rsa->n); buf = OPENSSL_malloc(num); if(!f || !ret || !buf) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE); goto err; } /* This check was for equality but PGP does evil things * and chops off the top '0' bytes */ if (flen > num) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN); goto err; } if (BN_bin2bn(from,flen,f) == NULL) goto err; if (BN_ucmp(f, rsa->n) >= 0) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS); goto err; } MONT_HELPER(rsa, ctx, n, rsa->flags & RSA_FLAG_CACHE_PUBLIC, goto err); if (!rsa->meth->bn_mod_exp(ret,f,rsa->e,rsa->n,ctx, rsa->_method_mod_n)) goto err; if ((padding == RSA_X931_PADDING) && ((ret->d[0] & 0xf) != 12)) BN_sub(ret, rsa->n, ret); p=buf; i=BN_bn2bin(ret,p); switch (padding) { case RSA_PKCS1_PADDING: r=RSA_padding_check_PKCS1_type_1(to,num,buf,i,num); /* Generally signatures should be at least 2/3 padding, though this isn't possible for really short keys and some standard signature schemes, so don't check if the unpadded data is small. */ if(r > 42 && 3*8*r >= BN_num_bits(rsa->n)) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_PKCS1_PADDING_TOO_SHORT); goto err; } break; case RSA_X931_PADDING: r=RSA_padding_check_X931(to,num,buf,i,num); break; case RSA_NO_PADDING: r=RSA_padding_check_none(to,num,buf,i,num); break; default: RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_PADDING_CHECK_FAILED); err: if (ctx != NULL) { BN_CTX_end(ctx); BN_CTX_free(ctx); } if (buf != NULL) { OPENSSL_cleanse(buf,num); OPENSSL_free(buf); } return(r); }
static int RSA_eay_private_decrypt(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { BIGNUM *f, *ret, *br; int j,num=0,r= -1; unsigned char *p; unsigned char *buf=NULL; BN_CTX *ctx=NULL; int local_blinding = 0; BN_BLINDING *blinding = NULL; if((ctx = BN_CTX_new()) == NULL) goto err; BN_CTX_start(ctx); f = BN_CTX_get(ctx); br = BN_CTX_get(ctx); ret = BN_CTX_get(ctx); num = BN_num_bytes(rsa->n); buf = OPENSSL_malloc(num); if(!f || !ret || !buf) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE); goto err; } /* This check was for equality but PGP does evil things * and chops off the top '0' bytes */ if (flen > num) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN); goto err; } /* make data into a big number */ if (BN_bin2bn(from,(int)flen,f) == NULL) goto err; if (BN_ucmp(f, rsa->n) >= 0) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS); goto err; } if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) { blinding = rsa_get_blinding(rsa, &br, &local_blinding, ctx); if (blinding == NULL) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, ERR_R_INTERNAL_ERROR); goto err; } } if (blinding != NULL) if (!rsa_blinding_convert(blinding, local_blinding, f, br, ctx)) goto err; /* do the decrypt */ if ( (rsa->flags & RSA_FLAG_EXT_PKEY) || ((rsa->p != NULL) && (rsa->q != NULL) && (rsa->dmp1 != NULL) && (rsa->dmq1 != NULL) && (rsa->iqmp != NULL)) ) { if (!rsa->meth->rsa_mod_exp(ret, f, rsa, ctx)) goto err; } else { BIGNUM local_d; BIGNUM *d = NULL; if (!(rsa->flags & RSA_FLAG_NO_EXP_CONSTTIME)) { d = &local_d; BN_with_flags(d, rsa->d, BN_FLG_EXP_CONSTTIME); } else d = rsa->d; MONT_HELPER(rsa, ctx, n, rsa->flags & RSA_FLAG_CACHE_PUBLIC, goto err); if (!rsa->meth->bn_mod_exp(ret,f,d,rsa->n,ctx, rsa->_method_mod_n)) goto err; } if (blinding) if (!rsa_blinding_invert(blinding, local_blinding, ret, br, ctx)) goto err; p=buf; j=BN_bn2bin(ret,p); /* j is only used with no-padding mode */ switch (padding) { case RSA_PKCS1_PADDING: r=RSA_padding_check_PKCS1_type_2(to,num,buf,j,num); break; #ifndef OPENSSL_NO_SHA case RSA_PKCS1_OAEP_PADDING: r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0); break; #endif case RSA_SSLV23_PADDING: r=RSA_padding_check_SSLv23(to,num,buf,j,num); break; case RSA_NO_PADDING: r=RSA_padding_check_none(to,num,buf,j,num); break; default: RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_PADDING_CHECK_FAILED); err: if (ctx != NULL) { BN_CTX_end(ctx); BN_CTX_free(ctx); } if (buf != NULL) { OPENSSL_cleanse(buf,num); OPENSSL_free(buf); } return(r); }
static int surewarehk_rsa_priv_dec(int flen,const unsigned char *from,unsigned char *to, RSA *rsa,int padding) { int ret=0,tlen; char *buf=NULL,*hptr=NULL; char msg[64]="ENGINE_rsa_priv_dec"; if (!p_surewarehk_Rsa_Priv_Dec) { SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,ENGINE_R_NOT_INITIALISED); } /* extract ref to private key */ else if (!(hptr=(char*)RSA_get_ex_data(rsa, rsaHndidx))) { SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,SUREWARE_R_MISSING_KEY_COMPONENTS); goto err; } /* analyse what padding we can do into the hardware */ if (padding==RSA_PKCS1_PADDING) { /* do it one shot */ ret=p_surewarehk_Rsa_Priv_Dec(msg,flen,(unsigned char *)from,&tlen,to,hptr,SUREWARE_PKCS1_PAD); surewarehk_error_handling(msg,SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,ret); if (ret!=1) goto err; ret=tlen; } else /* do with no padding into hardware */ { ret=p_surewarehk_Rsa_Priv_Dec(msg,flen,(unsigned char *)from,&tlen,to,hptr,SUREWARE_NO_PAD); surewarehk_error_handling(msg,SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,ret); if (ret!=1) goto err; /* intermediate buffer for padding */ if ((buf=(char*)OPENSSL_malloc(tlen)) == NULL) { SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,ERR_R_MALLOC_FAILURE); goto err; } TINYCLR_SSL_MEMCPY(buf,to,tlen);/* transfert to into buf */ switch (padding) /* check padding in software */ { #ifndef OPENSSL_NO_SHA case RSA_PKCS1_OAEP_PADDING: ret=RSA_padding_check_PKCS1_OAEP(to,tlen,(unsigned char *)buf,tlen,tlen,NULL,0); break; #endif case RSA_SSLV23_PADDING: ret=RSA_padding_check_SSLv23(to,tlen,(unsigned char *)buf,flen,tlen); break; case RSA_NO_PADDING: ret=RSA_padding_check_none(to,tlen,(unsigned char *)buf,flen,tlen); break; default: SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,SUREWARE_R_UNKNOWN_PADDING_TYPE); goto err; } if (ret < 0) SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,SUREWARE_R_PADDING_CHECK_FAILED); } err: if (buf) { OPENSSL_cleanse(buf,tlen); OPENSSL_free(buf); } return ret; }
/* signature verification */ static int RSA_eay_public_decrypt(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { BIGNUM *f,*ret; int i,num=0,r= -1; unsigned char *p; unsigned char *buf=NULL; BN_CTX *ctx=NULL; if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_MODULUS_TOO_LARGE); return -1; } if (BN_ucmp(rsa->n, rsa->e) <= 0) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE); return -1; } /* for large moduli, enforce exponent limit */ if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS) { if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE); return -1; } } if((ctx = BN_CTX_new()) == NULL) goto err; BN_CTX_start(ctx); f = BN_CTX_get(ctx); ret = BN_CTX_get(ctx); num=BN_num_bytes(rsa->n); buf = OPENSSL_malloc(num); if(!f || !ret || !buf) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE); goto err; } /* This check was for equality but PGP does evil things * and chops off the top '0' bytes */ if (flen > num) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN); goto err; } if (BN_bin2bn(from,flen,f) == NULL) goto err; if (BN_ucmp(f, rsa->n) >= 0) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS); goto err; } if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n, ctx)) goto err; if (!rsa->meth->bn_mod_exp(ret,f,rsa->e,rsa->n,ctx, rsa->_method_mod_n)) goto err; if ((padding == RSA_X931_PADDING) && ((ret->d[0] & 0xf) != 12)) BN_sub(ret, rsa->n, ret); p=buf; i=BN_bn2bin(ret,p); switch (padding) { case RSA_PKCS1_PADDING: r=RSA_padding_check_PKCS1_type_1(to,num,buf,i,num); break; case RSA_X931_PADDING: r=RSA_padding_check_X931(to,num,buf,i,num); break; case RSA_NO_PADDING: r=RSA_padding_check_none(to,num,buf,i,num); break; default: RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_PADDING_CHECK_FAILED); err: if (ctx != NULL) { BN_CTX_end(ctx); BN_CTX_free(ctx); } if (buf != NULL) { OPENSSL_cleanse(buf,num); OPENSSL_free(buf); } return(r); }
static int rsa_ossl_private_decrypt(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { BIGNUM *f, *ret; int j, num = 0, r = -1; unsigned char *p; unsigned char *buf = NULL; BN_CTX *ctx = NULL; int local_blinding = 0; /* * Used only if the blinding structure is shared. A non-NULL unblind * instructs rsa_blinding_convert() and rsa_blinding_invert() to store * the unblinding factor outside the blinding structure. */ BIGNUM *unblind = NULL; BN_BLINDING *blinding = NULL; if ((ctx = BN_CTX_new()) == NULL) goto err; BN_CTX_start(ctx); f = BN_CTX_get(ctx); ret = BN_CTX_get(ctx); num = BN_num_bytes(rsa->n); buf = OPENSSL_malloc(num); if (ret == NULL || buf == NULL) { RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE); goto err; } /* * This check was for equality but PGP does evil things and chops off the * top '0' bytes */ if (flen > num) { RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, RSA_R_DATA_GREATER_THAN_MOD_LEN); goto err; } /* make data into a big number */ if (BN_bin2bn(from, (int)flen, f) == NULL) goto err; if (BN_ucmp(f, rsa->n) >= 0) { RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); goto err; } if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) { blinding = rsa_get_blinding(rsa, &local_blinding, ctx); if (blinding == NULL) { RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, ERR_R_INTERNAL_ERROR); goto err; } } if (blinding != NULL) { if (!local_blinding && ((unblind = BN_CTX_get(ctx)) == NULL)) { RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE); goto err; } if (!rsa_blinding_convert(blinding, f, unblind, ctx)) goto err; } /* do the decrypt */ if ((rsa->flags & RSA_FLAG_EXT_PKEY) || (rsa->version == RSA_ASN1_VERSION_MULTI) || ((rsa->p != NULL) && (rsa->q != NULL) && (rsa->dmp1 != NULL) && (rsa->dmq1 != NULL) && (rsa->iqmp != NULL))) { if (!rsa->meth->rsa_mod_exp(ret, f, rsa, ctx)) goto err; } else { BIGNUM *d = BN_new(); if (d == NULL) { RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE); goto err; } BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME); if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) if (!BN_MONT_CTX_set_locked (&rsa->_method_mod_n, rsa->lock, rsa->n, ctx)) { BN_free(d); goto err; } if (!rsa->meth->bn_mod_exp(ret, f, d, rsa->n, ctx, rsa->_method_mod_n)) { BN_free(d); goto err; } /* We MUST free d before any further use of rsa->d */ BN_free(d); } if (blinding) if (!rsa_blinding_invert(blinding, ret, unblind, ctx)) goto err; p = buf; j = BN_bn2bin(ret, p); /* j is only used with no-padding mode */ switch (padding) { case RSA_PKCS1_PADDING: r = RSA_padding_check_PKCS1_type_2(to, num, buf, j, num); break; case RSA_PKCS1_OAEP_PADDING: r = RSA_padding_check_PKCS1_OAEP(to, num, buf, j, num, NULL, 0); break; case RSA_SSLV23_PADDING: r = RSA_padding_check_SSLv23(to, num, buf, j, num); break; case RSA_NO_PADDING: r = RSA_padding_check_none(to, num, buf, j, num); break; default: RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, RSA_R_PADDING_CHECK_FAILED); err: if (ctx != NULL) BN_CTX_end(ctx); BN_CTX_free(ctx); OPENSSL_clear_free(buf, num); return r; }
static int RSA_eay_private_decrypt(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { BIGNUM *f, *ret; int j, num = 0, r = -1; unsigned char *p; unsigned char *buf = NULL; BN_CTX *ctx = NULL; int local_blinding = 0; /* * Used only if the blinding structure is shared. A non-NULL unblind * instructs rsa_blinding_convert() and rsa_blinding_invert() to store * the unblinding factor outside the blinding structure. */ BIGNUM *unblind = NULL; BN_BLINDING *blinding = NULL; if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS) { RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_MODULUS_TOO_LARGE); return -1; } if (BN_ucmp(rsa->n, rsa->e) <= 0) { RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE); return -1; } /* for large moduli, enforce exponent limit */ if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS) { if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) { RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE); return -1; } } if ((ctx = BN_CTX_new()) == NULL) goto err; BN_CTX_start(ctx); f = BN_CTX_get(ctx); ret = BN_CTX_get(ctx); num = BN_num_bytes(rsa->n); buf = OPENSSL_malloc(num); if (!f || !ret || !buf) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE); goto err; } /* * This check was for equality but PGP does evil things and chops off the * top '0' bytes */ if (flen > num) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, RSA_R_DATA_GREATER_THAN_MOD_LEN); goto err; } /* make data into a big number */ if (BN_bin2bn(from, (int)flen, f) == NULL) goto err; if (BN_ucmp(f, rsa->n) >= 0) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); goto err; } if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) { blinding = rsa_get_blinding(rsa, &local_blinding, ctx); if (blinding == NULL) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, ERR_R_INTERNAL_ERROR); goto err; } } if (blinding != NULL) { if (!local_blinding && ((unblind = BN_CTX_get(ctx)) == NULL)) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE); goto err; } if (!rsa_blinding_convert(blinding, f, unblind, ctx)) goto err; } /* do the decrypt */ if ((rsa->flags & RSA_FLAG_EXT_PKEY) || ((rsa->p != NULL) && (rsa->q != NULL) && (rsa->dmp1 != NULL) && (rsa->dmq1 != NULL) && (rsa->iqmp != NULL))) { if (!rsa->meth->rsa_mod_exp(ret, f, rsa, ctx)) goto err; } else { BIGNUM local_d; BIGNUM *d = NULL; if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) { d = &local_d; BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME); } else d = rsa->d; if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) if (!BN_MONT_CTX_set_locked (&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n, ctx)) goto err; if (!rsa->meth->bn_mod_exp(ret, f, d, rsa->n, ctx, rsa->_method_mod_n)) goto err; } if (blinding) if (!rsa_blinding_invert(blinding, ret, unblind, ctx)) goto err; p = buf; j = BN_bn2bin(ret, p); /* j is only used with no-padding mode */ switch (padding) { case RSA_PKCS1_PADDING: r = RSA_padding_check_PKCS1_type_2(to, num, buf, j, num); break; # ifndef OPENSSL_NO_SHA case RSA_PKCS1_OAEP_PADDING: r = RSA_padding_check_PKCS1_OAEP(to, num, buf, j, num, NULL, 0); break; # endif case RSA_SSLV23_PADDING: r = RSA_padding_check_SSLv23(to, num, buf, j, num); break; case RSA_NO_PADDING: r = RSA_padding_check_none(to, num, buf, j, num); break; default: RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, RSA_R_PADDING_CHECK_FAILED); err: if (ctx != NULL) { BN_CTX_end(ctx); BN_CTX_free(ctx); } if (buf != NULL) { OPENSSL_cleanse(buf, num); OPENSSL_free(buf); } return (r); }
static int RSA_eay_public_decrypt(int flen, unsigned char *from, unsigned char *to, RSA *rsa, int padding) { BIGNUM f,ret; int i,num=0,r= -1; unsigned char *p; unsigned char *buf=NULL; BN_CTX *ctx=NULL; BN_init(&f); BN_init(&ret); ctx=BN_CTX_new(); if (ctx == NULL) goto err; num=BN_num_bytes(rsa->n); buf=(unsigned char *)Malloc(num); if (buf == NULL) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE); goto err; } /* This check was for equality but PGP does evil things * and chops off the top '0' bytes */ if (flen > num) { RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN); goto err; } if (BN_bin2bn(from,flen,&f) == NULL) goto err; /* do the decrypt */ if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC)) { if ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL) if (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx)) goto err; } if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx, rsa->_method_mod_n)) goto err; p=buf; i=BN_bn2bin(&ret,p); switch (padding) { case RSA_PKCS1_PADDING: r=RSA_padding_check_PKCS1_type_1(to,num,buf,i,num); break; case RSA_NO_PADDING: r=RSA_padding_check_none(to,num,buf,i,num); break; default: RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_PADDING_CHECK_FAILED); err: if (ctx != NULL) BN_CTX_free(ctx); BN_clear_free(&f); BN_clear_free(&ret); if (buf != NULL) { memset(buf,0,num); Free(buf); } return(r); }
static int RSA_eay_private_decrypt(int flen, unsigned char *from, unsigned char *to, RSA *rsa, int padding) { BIGNUM f,ret; int j,num=0,r= -1; unsigned char *p; unsigned char *buf=NULL; BN_CTX *ctx=NULL; BN_init(&f); BN_init(&ret); ctx=BN_CTX_new(); if (ctx == NULL) goto err; num=BN_num_bytes(rsa->n); if ((buf=(unsigned char *)Malloc(num)) == NULL) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE); goto err; } /* This check was for equality but PGP does evil things * and chops off the top '0' bytes */ if (flen > num) { RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN); goto err; } /* make data into a big number */ if (BN_bin2bn(from,(int)flen,&f) == NULL) goto err; if ((rsa->flags & RSA_FLAG_BLINDING) && (RSA_get_thread_blinding_ptr(rsa) == NULL)) RSA_blinding_on(rsa,ctx); if (rsa->flags & RSA_FLAG_BLINDING) if (!BN_BLINDING_convert(&f,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err; /* do the decrypt */ if ( (rsa->flags & RSA_FLAG_EXT_PKEY) || ((rsa->p != NULL) && (rsa->q != NULL) && (rsa->dmp1 != NULL) && (rsa->dmq1 != NULL) && (rsa->iqmp != NULL)) ) { if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; } else { if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL)) goto err; } if (rsa->flags & RSA_FLAG_BLINDING) if (!BN_BLINDING_invert(&ret,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err; p=buf; j=BN_bn2bin(&ret,p); /* j is only used with no-padding mode */ switch (padding) { case RSA_PKCS1_PADDING: r=RSA_padding_check_PKCS1_type_2(to,num,buf,j,num); break; #ifndef _OPENSSL_APPLE_CDSA_ #ifndef NO_SHA case RSA_PKCS1_OAEP_PADDING: r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0); break; #endif #endif case RSA_SSLV23_PADDING: r=RSA_padding_check_SSLv23(to,num,buf,j,num); break; case RSA_NO_PADDING: r=RSA_padding_check_none(to,num,buf,j,num); break; default: RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_PADDING_CHECK_FAILED); err: if (ctx != NULL) BN_CTX_free(ctx); BN_clear_free(&f); BN_clear_free(&ret); if (buf != NULL) { memset(buf,0,num); Free(buf); } return(r); }
static int verify_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, const uint8_t *in, size_t in_len, int padding) { const unsigned rsa_size = RSA_size(rsa); BIGNUM *f, *result; int ret = 0; int r = -1; uint8_t *buf = NULL; BN_CTX *ctx = NULL; if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS) { OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_MODULUS_TOO_LARGE); return 0; } if (BN_ucmp(rsa->n, rsa->e) <= 0) { OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_BAD_E_VALUE); return 0; } if (max_out < rsa_size) { OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } /* for large moduli, enforce exponent limit */ if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS && BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) { OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_BAD_E_VALUE); return 0; } ctx = BN_CTX_new(); if (ctx == NULL) { goto err; } BN_CTX_start(ctx); f = BN_CTX_get(ctx); result = BN_CTX_get(ctx); buf = OPENSSL_malloc(rsa_size); if (!f || !result || !buf) { OPENSSL_PUT_ERROR(RSA, verify_raw, ERR_R_MALLOC_FAILURE); goto err; } if (in_len != rsa_size) { OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN); goto err; } if (BN_bin2bn(in, in_len, f) == NULL) { goto err; } if (BN_ucmp(f, rsa->n) >= 0) { OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); goto err; } if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) { if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n, ctx)) { goto err; } } if (!rsa->meth->bn_mod_exp(result, f, rsa->e, rsa->n, ctx, rsa->_method_mod_n)) { goto err; } if (!BN_bn2bin_padded(buf, rsa_size, result)) { OPENSSL_PUT_ERROR(RSA, verify_raw, ERR_R_INTERNAL_ERROR); goto err; } switch (padding) { case RSA_PKCS1_PADDING: r = RSA_padding_check_PKCS1_type_1(out, rsa_size, buf, rsa_size); break; case RSA_NO_PADDING: r = RSA_padding_check_none(out, rsa_size, buf, rsa_size); break; default: OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) { OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_PADDING_CHECK_FAILED); } else { *out_len = r; ret = 1; } err: if (ctx != NULL) { BN_CTX_end(ctx); BN_CTX_free(ctx); } if (buf != NULL) { OPENSSL_cleanse(buf, rsa_size); OPENSSL_free(buf); } return ret; }