static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM *f, *ret, *br;
int j,num=0,r= -1;
unsigned char *p;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
int local_blinding = 0;
BN_BLINDING *blinding = NULL;
if((ctx = BN_CTX_new()) == NULL) goto err;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
br = BN_CTX_get(ctx);
ret = BN_CTX_get(ctx);
num = BN_num_bytes(rsa->n);
buf = OPENSSL_malloc(num);
if(!f || !ret || !buf)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
/* This check was for equality but PGP does evil things
* and chops off the top '0' bytes */
if (flen > num)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
goto err;
}
/* make data into a big number */
if (BN_bin2bn(from,(int)flen,f) == NULL) goto err;
if (BN_ucmp(f, rsa->n) >= 0)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (!(rsa->flags & RSA_FLAG_NO_BLINDING))
{
blinding = rsa_get_blinding(rsa, &br, &local_blinding, ctx);
if (blinding == NULL)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, ERR_R_INTERNAL_ERROR);
goto err;
}
}
if (blinding != NULL)
if (!rsa_blinding_convert(blinding, local_blinding, f, br, ctx))
goto err;
/* do the decrypt */
if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
((rsa->p != NULL) &&
(rsa->q != NULL) &&
(rsa->dmp1 != NULL) &&
(rsa->dmq1 != NULL) &&
(rsa->iqmp != NULL)) )
{
if (!rsa->meth->rsa_mod_exp(ret, f, rsa, ctx)) goto err;
}
else
{
BIGNUM local_d;
BIGNUM *d = NULL;
if (!(rsa->flags & RSA_FLAG_NO_EXP_CONSTTIME))
{
d = &local_d;
BN_with_flags(d, rsa->d, BN_FLG_EXP_CONSTTIME);
}
else
d = rsa->d;
MONT_HELPER(rsa, ctx, n, rsa->flags & RSA_FLAG_CACHE_PUBLIC, goto err);
if (!rsa->meth->bn_mod_exp(ret,f,d,rsa->n,ctx,
rsa->_method_mod_n))
goto err;
}
if (blinding)
if (!rsa_blinding_invert(blinding, local_blinding, ret, br, ctx))
goto err;
p=buf;
j=BN_bn2bin(ret,p); /* j is only used with no-padding mode */
switch (padding)
{
case RSA_PKCS1_PADDING:
r=RSA_padding_check_PKCS1_type_2(to,num,buf,j,num);
break;
#ifndef OPENSSL_NO_SHA
case RSA_PKCS1_OAEP_PADDING:
r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0);
break;
#endif
case RSA_SSLV23_PADDING:
r=RSA_padding_check_SSLv23(to,num,buf,j,num);
break;
case RSA_NO_PADDING:
r=RSA_padding_check_none(to,num,buf,j,num);
break;
default:
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0)
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
err:
if (ctx != NULL)
{
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL)
{
OPENSSL_cleanse(buf,num);
OPENSSL_free(buf);
}
return(r);
}
unsigned int OpenSSLCryptoKeyRSA::privateDecrypt(const unsigned char * inBuf,
unsigned char * plainBuf,
unsigned int inLength,
unsigned int maxOutLength,
PaddingType padding,
hashMethod hm) {
// Perform a decrypt
if (mp_rsaKey == NULL) {
throw XSECCryptoException(XSECCryptoException::RSAError,
"OpenSSL:RSA - Attempt to decrypt data with empty key");
}
#if 0
/* normally commented out code to determine endian problems */
unsigned int i;
unsigned char e[2048];
unsigned char * inBuf1 = (unsigned char *) inBuf;
if (inLength < 2048) {
memcpy(e, inBuf, inLength);
for (i = 0; i < inLength;++i) {
inBuf1[i] = e[inLength - 1 - i];
}
}
#endif
int decryptSize;
switch (padding) {
case XSECCryptoKeyRSA::PAD_PKCS_1_5 :
decryptSize = RSA_private_decrypt(inLength,
#if defined(XSEC_OPENSSL_CONST_BUFFERS)
inBuf,
#else
(unsigned char *) inBuf,
#endif
plainBuf,
mp_rsaKey,
RSA_PKCS1_PADDING);
if (decryptSize < 0) {
throw XSECCryptoException(XSECCryptoException::RSAError,
"OpenSSL:RSA privateKeyDecrypt - Error Decrypting PKCS1_5 padded RSA encrypt");
}
break;
case XSECCryptoKeyRSA::PAD_OAEP_MGFP1 :
{
unsigned char * tBuf;
int num = RSA_size(mp_rsaKey);
XSECnew(tBuf, unsigned char[num]);
ArrayJanitor<unsigned char> j_tBuf(tBuf);
const EVP_MD* evp_md = NULL;
const EVP_MD* mgf_md = NULL;
switch (hm) {
case HASH_SHA1:
evp_md = EVP_get_digestbyname("SHA1");
break;
case HASH_SHA224:
evp_md = EVP_get_digestbyname("SHA224");
break;
case HASH_SHA256:
evp_md = EVP_get_digestbyname("SHA256");
break;
case HASH_SHA384:
evp_md = EVP_get_digestbyname("SHA384");
break;
case HASH_SHA512:
evp_md = EVP_get_digestbyname("SHA512");
break;
}
if (evp_md == NULL) {
throw XSECCryptoException(XSECCryptoException::MDError,
"OpenSSL:RSA - OAEP digest algorithm not supported by this version of OpenSSL");
}
switch (m_mgf) {
case MGF1_SHA1:
mgf_md = EVP_get_digestbyname("SHA1");
break;
case MGF1_SHA224:
mgf_md = EVP_get_digestbyname("SHA224");
break;
case MGF1_SHA256:
mgf_md = EVP_get_digestbyname("SHA256");
break;
case MGF1_SHA384:
mgf_md = EVP_get_digestbyname("SHA384");
break;
case MGF1_SHA512:
mgf_md = EVP_get_digestbyname("SHA512");
break;
}
if (mgf_md == NULL) {
throw XSECCryptoException(XSECCryptoException::MDError,
"OpenSSL:RSA - MGF not supported by this version of OpenSSL");
}
decryptSize = RSA_private_decrypt(inLength,
#if defined(XSEC_OPENSSL_CONST_BUFFERS)
inBuf,
#else
(unsigned char *) inBuf,
#endif
tBuf,
mp_rsaKey,
RSA_NO_PADDING);
if (decryptSize < 0) {
throw XSECCryptoException(XSECCryptoException::RSAError,
"OpenSSL:RSA privateKeyDecrypt - Error doing raw decrypt of RSA encrypted data");
}
// Clear out the "0"s at the front
int i;
for (i = 0; i < num && tBuf[i] == 0; ++i)
--decryptSize;
decryptSize = RSA_padding_check_PKCS1_OAEP(plainBuf,
maxOutLength,
&tBuf[i],
decryptSize,
num,
mp_oaepParams,
m_oaepParamsLen,
evp_md,
mgf_md);
if (decryptSize < 0) {
throw XSECCryptoException(XSECCryptoException::RSAError,
"OpenSSL:RSA privateKeyDecrypt - Error removing OAEPadding");
}
}
break;
default :
throw XSECCryptoException(XSECCryptoException::RSAError,
"OpenSSL:RSA - Unknown padding method");
}
#if 0
/* normally commented out code to determine endian problems */
int i;
unsigned char t[512];
if (decryptSize < 512) {
memcpy(t, plainBuf, decryptSize);
for (i = 0; i < decryptSize;++i) {
plainBuf[i] = t[decryptSize - 1 - i];
}
}
#endif
return decryptSize;
}
static int surewarehk_rsa_priv_dec(int flen,const unsigned char *from,unsigned char *to,
RSA *rsa,int padding)
{
int ret=0,tlen;
char *buf=NULL,*hptr=NULL;
char msg[64]="ENGINE_rsa_priv_dec";
if (!p_surewarehk_Rsa_Priv_Dec)
{
SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,ENGINE_R_NOT_INITIALISED);
}
/* extract ref to private key */
else if (!(hptr=(char*)RSA_get_ex_data(rsa, rsaHndidx)))
{
SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,SUREWARE_R_MISSING_KEY_COMPONENTS);
goto err;
}
/* analyse what padding we can do into the hardware */
if (padding==RSA_PKCS1_PADDING)
{
/* do it one shot */
ret=p_surewarehk_Rsa_Priv_Dec(msg,flen,(unsigned char *)from,&tlen,to,hptr,SUREWARE_PKCS1_PAD);
surewarehk_error_handling(msg,SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,ret);
if (ret!=1)
goto err;
ret=tlen;
}
else /* do with no padding into hardware */
{
ret=p_surewarehk_Rsa_Priv_Dec(msg,flen,(unsigned char *)from,&tlen,to,hptr,SUREWARE_NO_PAD);
surewarehk_error_handling(msg,SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,ret);
if (ret!=1)
goto err;
/* intermediate buffer for padding */
if ((buf=(char*)OPENSSL_malloc(tlen)) == NULL)
{
SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,ERR_R_MALLOC_FAILURE);
goto err;
}
TINYCLR_SSL_MEMCPY(buf,to,tlen);/* transfert to into buf */
switch (padding) /* check padding in software */
{
#ifndef OPENSSL_NO_SHA
case RSA_PKCS1_OAEP_PADDING:
ret=RSA_padding_check_PKCS1_OAEP(to,tlen,(unsigned char *)buf,tlen,tlen,NULL,0);
break;
#endif
case RSA_SSLV23_PADDING:
ret=RSA_padding_check_SSLv23(to,tlen,(unsigned char *)buf,flen,tlen);
break;
case RSA_NO_PADDING:
ret=RSA_padding_check_none(to,tlen,(unsigned char *)buf,flen,tlen);
break;
default:
SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,SUREWARE_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (ret < 0)
SUREWAREerr(SUREWARE_F_SUREWAREHK_RSA_PRIV_DEC,SUREWARE_R_PADDING_CHECK_FAILED);
}
err:
if (buf)
{
OPENSSL_cleanse(buf,tlen);
OPENSSL_free(buf);
}
return ret;
}
static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM *f, *ret;
int j, num = 0, r = -1;
unsigned char *p;
unsigned char *buf = NULL;
BN_CTX *ctx = NULL;
int local_blinding = 0;
/*
* Used only if the blinding structure is shared. A non-NULL unblind
* instructs rsa_blinding_convert() and rsa_blinding_invert() to store
* the unblinding factor outside the blinding structure.
*/
BIGNUM *unblind = NULL;
BN_BLINDING *blinding = NULL;
if ((ctx = BN_CTX_new()) == NULL)
goto err;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
ret = BN_CTX_get(ctx);
num = BN_num_bytes(rsa->n);
buf = OPENSSL_malloc(num);
if (ret == NULL || buf == NULL) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
/*
* This check was for equality but PGP does evil things and chops off the
* top '0' bytes
*/
if (flen > num) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT,
RSA_R_DATA_GREATER_THAN_MOD_LEN);
goto err;
}
/* make data into a big number */
if (BN_bin2bn(from, (int)flen, f) == NULL)
goto err;
if (BN_ucmp(f, rsa->n) >= 0) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT,
RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) {
blinding = rsa_get_blinding(rsa, &local_blinding, ctx);
if (blinding == NULL) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, ERR_R_INTERNAL_ERROR);
goto err;
}
}
if (blinding != NULL) {
if (!local_blinding && ((unblind = BN_CTX_get(ctx)) == NULL)) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
if (!rsa_blinding_convert(blinding, f, unblind, ctx))
goto err;
}
/* do the decrypt */
if ((rsa->flags & RSA_FLAG_EXT_PKEY) ||
(rsa->version == RSA_ASN1_VERSION_MULTI) ||
((rsa->p != NULL) &&
(rsa->q != NULL) &&
(rsa->dmp1 != NULL) && (rsa->dmq1 != NULL) && (rsa->iqmp != NULL))) {
if (!rsa->meth->rsa_mod_exp(ret, f, rsa, ctx))
goto err;
} else {
BIGNUM *d = BN_new();
if (d == NULL) {
RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
if (rsa->flags & RSA_FLAG_CACHE_PUBLIC)
if (!BN_MONT_CTX_set_locked
(&rsa->_method_mod_n, rsa->lock, rsa->n, ctx)) {
BN_free(d);
goto err;
}
if (!rsa->meth->bn_mod_exp(ret, f, d, rsa->n, ctx,
rsa->_method_mod_n)) {
BN_free(d);
goto err;
}
/* We MUST free d before any further use of rsa->d */
BN_free(d);
}
if (blinding)
if (!rsa_blinding_invert(blinding, ret, unblind, ctx))
goto err;
p = buf;
j = BN_bn2bin(ret, p); /* j is only used with no-padding mode */
switch (padding) {
case RSA_PKCS1_PADDING:
r = RSA_padding_check_PKCS1_type_2(to, num, buf, j, num);
break;
case RSA_PKCS1_OAEP_PADDING:
r = RSA_padding_check_PKCS1_OAEP(to, num, buf, j, num, NULL, 0);
break;
case RSA_SSLV23_PADDING:
r = RSA_padding_check_SSLv23(to, num, buf, j, num);
break;
case RSA_NO_PADDING:
r = RSA_padding_check_none(to, num, buf, j, num);
break;
default:
RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0)
RSAerr(RSA_F_RSA_OSSL_PRIVATE_DECRYPT, RSA_R_PADDING_CHECK_FAILED);
err:
if (ctx != NULL)
BN_CTX_end(ctx);
BN_CTX_free(ctx);
OPENSSL_clear_free(buf, num);
return r;
}
static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM *f, *ret;
int j, num = 0, r = -1;
unsigned char *p;
unsigned char *buf = NULL;
BN_CTX *ctx = NULL;
int local_blinding = 0;
/*
* Used only if the blinding structure is shared. A non-NULL unblind
* instructs rsa_blinding_convert() and rsa_blinding_invert() to store
* the unblinding factor outside the blinding structure.
*/
BIGNUM *unblind = NULL;
BN_BLINDING *blinding = NULL;
if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS) {
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_MODULUS_TOO_LARGE);
return -1;
}
if (BN_ucmp(rsa->n, rsa->e) <= 0) {
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
return -1;
}
/* for large moduli, enforce exponent limit */
if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS) {
if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) {
RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
return -1;
}
}
if ((ctx = BN_CTX_new()) == NULL)
goto err;
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
ret = BN_CTX_get(ctx);
num = BN_num_bytes(rsa->n);
buf = OPENSSL_malloc(num);
if (!f || !ret || !buf) {
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
/*
* This check was for equality but PGP does evil things and chops off the
* top '0' bytes
*/
if (flen > num) {
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,
RSA_R_DATA_GREATER_THAN_MOD_LEN);
goto err;
}
/* make data into a big number */
if (BN_bin2bn(from, (int)flen, f) == NULL)
goto err;
if (BN_ucmp(f, rsa->n) >= 0) {
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,
RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) {
blinding = rsa_get_blinding(rsa, &local_blinding, ctx);
if (blinding == NULL) {
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, ERR_R_INTERNAL_ERROR);
goto err;
}
}
if (blinding != NULL) {
if (!local_blinding && ((unblind = BN_CTX_get(ctx)) == NULL)) {
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, ERR_R_MALLOC_FAILURE);
goto err;
}
if (!rsa_blinding_convert(blinding, f, unblind, ctx))
goto err;
}
/* do the decrypt */
if ((rsa->flags & RSA_FLAG_EXT_PKEY) ||
((rsa->p != NULL) &&
(rsa->q != NULL) &&
(rsa->dmp1 != NULL) && (rsa->dmq1 != NULL) && (rsa->iqmp != NULL))) {
if (!rsa->meth->rsa_mod_exp(ret, f, rsa, ctx))
goto err;
} else {
BIGNUM local_d;
BIGNUM *d = NULL;
if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {
d = &local_d;
BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
} else
d = rsa->d;
if (rsa->flags & RSA_FLAG_CACHE_PUBLIC)
if (!BN_MONT_CTX_set_locked
(&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n, ctx))
goto err;
if (!rsa->meth->bn_mod_exp(ret, f, d, rsa->n, ctx,
rsa->_method_mod_n))
goto err;
}
if (blinding)
if (!rsa_blinding_invert(blinding, ret, unblind, ctx))
goto err;
p = buf;
j = BN_bn2bin(ret, p); /* j is only used with no-padding mode */
switch (padding) {
case RSA_PKCS1_PADDING:
r = RSA_padding_check_PKCS1_type_2(to, num, buf, j, num);
break;
# ifndef OPENSSL_NO_SHA
case RSA_PKCS1_OAEP_PADDING:
r = RSA_padding_check_PKCS1_OAEP(to, num, buf, j, num, NULL, 0);
break;
# endif
case RSA_SSLV23_PADDING:
r = RSA_padding_check_SSLv23(to, num, buf, j, num);
break;
case RSA_NO_PADDING:
r = RSA_padding_check_none(to, num, buf, j, num);
break;
default:
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0)
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT, RSA_R_PADDING_CHECK_FAILED);
err:
if (ctx != NULL) {
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL) {
OPENSSL_cleanse(buf, num);
OPENSSL_free(buf);
}
return (r);
}
static int RSA_eay_private_decrypt(int flen, unsigned char *from,
unsigned char *to, RSA *rsa, int padding)
{
BIGNUM f,ret;
int j,num=0,r= -1;
unsigned char *p;
unsigned char *buf=NULL;
BN_CTX *ctx=NULL;
BN_init(&f);
BN_init(&ret);
ctx=BN_CTX_new();
if (ctx == NULL) goto err;
num=BN_num_bytes(rsa->n);
if ((buf=(unsigned char *)Malloc(num)) == NULL)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE);
goto err;
}
/* This check was for equality but PGP does evil things
* and chops off the top '0' bytes */
if (flen > num)
{
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
goto err;
}
/* make data into a big number */
if (BN_bin2bn(from,(int)flen,&f) == NULL) goto err;
if ((rsa->flags & RSA_FLAG_BLINDING) && (RSA_get_thread_blinding_ptr(rsa) == NULL))
RSA_blinding_on(rsa,ctx);
if (rsa->flags & RSA_FLAG_BLINDING)
if (!BN_BLINDING_convert(&f,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err;
/* do the decrypt */
if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
((rsa->p != NULL) &&
(rsa->q != NULL) &&
(rsa->dmp1 != NULL) &&
(rsa->dmq1 != NULL) &&
(rsa->iqmp != NULL)) )
{ if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
else
{
if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL))
goto err;
}
if (rsa->flags & RSA_FLAG_BLINDING)
if (!BN_BLINDING_invert(&ret,RSA_get_thread_blinding_ptr(rsa),ctx)) goto err;
p=buf;
j=BN_bn2bin(&ret,p); /* j is only used with no-padding mode */
switch (padding)
{
case RSA_PKCS1_PADDING:
r=RSA_padding_check_PKCS1_type_2(to,num,buf,j,num);
break;
#ifndef _OPENSSL_APPLE_CDSA_
#ifndef NO_SHA
case RSA_PKCS1_OAEP_PADDING:
r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0);
break;
#endif
#endif
case RSA_SSLV23_PADDING:
r=RSA_padding_check_SSLv23(to,num,buf,j,num);
break;
case RSA_NO_PADDING:
r=RSA_padding_check_none(to,num,buf,j,num);
break;
default:
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0)
RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
err:
if (ctx != NULL) BN_CTX_free(ctx);
BN_clear_free(&f);
BN_clear_free(&ret);
if (buf != NULL)
{
memset(buf,0,num);
Free(buf);
}
return(r);
}
static int
xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) {
xmlSecOpenSSLRsaOaepCtxPtr ctx;
xmlSecSize paramsSize;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize;
xmlSecSize keySize;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->pKey != NULL, -1);
xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1);
xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1);
keySize = RSA_size(ctx->pKey->pkey.rsa);
xmlSecAssert2(keySize > 0, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
/* the encoded size is equal to the keys size so we could not
* process more than that */
if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"%d when expected less than %d", inSize, keySize);
return(-1);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"%d when expected %d", inSize, keySize);
return(-1);
}
outSize = keySize;
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams));
if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) {
/* encode w/o OAEPParams --> simple */
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) {
xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1);
/* add space for padding */
ret = xmlSecBufferSetMaxSize(in, keySize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", keySize);
return(-1);
}
/* add padding */
ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(&(ctx->oaepParams)),
paramsSize);
if(ret != 1) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_padding_add_PKCS1_OAEP",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
inSize = keySize;
/* encode with OAEPParams */
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_NO_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_public_encrypt(RSA_NO_PADDING)",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) {
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) {
BIGNUM bn;
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_NO_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_private_decrypt(RSA_NO_PADDING)",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
/*
* the private decrypt w/o padding adds '0's at the begginning.
* it's not clear for me can I simply skip all '0's from the
* beggining so I have to do decode it back to BIGNUM and dump
* buffer again
*/
BN_init(&bn);
if(BN_bin2bn(xmlSecBufferGetData(out), outSize, &bn) == NULL) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"BN_bin2bn",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
"size=%d", outSize);
BN_clear_free(&bn);
return(-1);
}
ret = BN_bn2bin(&bn, xmlSecBufferGetData(out));
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"BN_bn2bin",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
BN_clear_free(&bn);
return(-1);
}
BN_clear_free(&bn);
outSize = ret;
ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize,
xmlSecBufferGetData(out), outSize,
keySize,
xmlSecBufferGetData(&(ctx->oaepParams)),
paramsSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_padding_check_PKCS1_OAEP",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
} else {
xmlSecAssert2("we could not be here" == NULL, -1);
return(-1);
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferRemoveHead",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", inSize);
return(-1);
}
return(0);
}
static int
xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) {
xmlSecOpenSSLRsaOaepCtxPtr ctx;
xmlSecSize paramsSize;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize;
xmlSecSize keySize;
RSA *rsa;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->pKey != NULL, -1);
xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1);
rsa = EVP_PKEY_get0_RSA(ctx->pKey);
xmlSecAssert2(rsa != NULL, -1);
keySize = RSA_size(rsa);
xmlSecAssert2(keySize > 0, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
/* the encoded size is equal to the keys size so we could not
* process more than that */
if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
xmlSecInvalidSizeLessThanError("Input data", inSize, keySize,
xmlSecTransformGetName(transform));
return(-1);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
xmlSecInvalidSizeError("Input data", inSize, keySize,
xmlSecTransformGetName(transform));
return(-1);
}
outSize = keySize;
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferSetMaxSize",
xmlSecTransformGetName(transform),
"size=%d", outSize);
return(-1);
}
paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams));
if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) {
/* encode w/o OAEPParams --> simple */
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError("RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)",
xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) {
xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1);
/* add space for padding */
ret = xmlSecBufferSetMaxSize(in, keySize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferSetMaxSize",
xmlSecTransformGetName(transform),
"size=%d", keySize);
return(-1);
}
/* add padding */
ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(&(ctx->oaepParams)),
paramsSize);
if(ret != 1) {
xmlSecOpenSSLError("RSA_padding_add_PKCS1_OAEP",
xmlSecTransformGetName(transform));
return(-1);
}
inSize = keySize;
/* encode with OAEPParams */
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_NO_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError("RSA_public_encrypt(RSA_NO_PADDING)",
xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) {
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError("RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)",
xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) {
BIGNUM * bn;
bn = BN_new();
if(bn == NULL) {
xmlSecOpenSSLError("BN_new()",
xmlSecTransformGetName(transform));
return(-1);
}
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_NO_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError("RSA_private_decrypt(RSA_NO_PADDING)",
xmlSecTransformGetName(transform));
BN_free(bn);
return(-1);
}
outSize = ret;
/*
* the private decrypt w/o padding adds '0's at the begginning.
* it's not clear for me can I simply skip all '0's from the
* beggining so I have to do decode it back to BIGNUM and dump
* buffer again
*/
if(BN_bin2bn(xmlSecBufferGetData(out), outSize, bn) == NULL) {
xmlSecOpenSSLError2("BN_bin2bn",
xmlSecTransformGetName(transform),
"size=%lu", (unsigned long)outSize);
BN_free(bn);
return(-1);
}
ret = BN_bn2bin(bn, xmlSecBufferGetData(out));
if(ret <= 0) {
xmlSecOpenSSLError("BN_bn2bin",
xmlSecTransformGetName(transform));
BN_free(bn);
return(-1);
}
BN_free(bn);
outSize = ret;
ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize,
xmlSecBufferGetData(out), outSize,
keySize,
xmlSecBufferGetData(&(ctx->oaepParams)),
paramsSize);
if(ret < 0) {
xmlSecOpenSSLError("RSA_padding_check_PKCS1_OAEP",
xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else {
xmlSecOtherError3(XMLSEC_ERRORS_R_INVALID_OPERATION,
xmlSecTransformGetName(transform),
"Unexpected transform operation: %ld; paramsSize: %ld",
(long int)transform->operation, (long int)paramsSize);
return(-1);
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferSetSize",
xmlSecTransformGetName(transform),
"size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferRemoveHead",
xmlSecTransformGetName(transform),
"size=%d", inSize);
return(-1);
}
return(0);
}
