static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
unsigned char *out, size_t *outlen,
const unsigned char *in, size_t inlen)
{
int ret;
RSA_PKEY_CTX *rctx = ctx->data;
if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
int klen = RSA_size(ctx->pkey->pkey.rsa);
if (!setup_tbuf(rctx, ctx))
return -1;
if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
in, inlen,
rctx->oaep_label,
rctx->oaep_labellen,
rctx->md, rctx->mgf1md))
return -1;
ret = RSA_public_encrypt(klen, rctx->tbuf, out,
ctx->pkey->pkey.rsa, RSA_NO_PADDING);
} else
ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
rctx->pad_mode);
if (ret < 0)
return ret;
*outlen = ret;
return 1;
}
static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen,
const uint8_t *in, size_t inlen) {
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
const size_t key_len = EVP_PKEY_size(ctx->pkey);
if (!out) {
*outlen = key_len;
return 1;
}
if (*outlen < key_len) {
OPENSSL_PUT_ERROR(EVP, pkey_rsa_encrypt, EVP_R_BUFFER_TOO_SMALL);
return 0;
}
if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
if (!setup_tbuf(rctx, ctx) ||
!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, key_len, in, inlen,
rctx->oaep_label, rctx->oaep_labellen,
rctx->md, rctx->mgf1md) ||
!RSA_encrypt(rsa, outlen, out, *outlen, rctx->tbuf, key_len,
RSA_NO_PADDING)) {
return 0;
}
return 1;
}
return RSA_encrypt(rsa, outlen, out, *outlen, in, inlen, rctx->pad_mode);
}
int rsa_default_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
const unsigned rsa_size = RSA_size(rsa);
BIGNUM *f, *result;
uint8_t *buf = NULL;
BN_CTX *ctx = NULL;
int i, ret = 0;
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
if (!check_modulus_and_exponent_sizes(rsa)) {
return 0;
}
ctx = BN_CTX_new();
if (ctx == NULL) {
goto err;
}
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
result = BN_CTX_get(ctx);
buf = OPENSSL_malloc(rsa_size);
if (!f || !result || !buf) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
i = RSA_padding_add_PKCS1_type_2(buf, rsa_size, in, in_len);
break;
case RSA_PKCS1_OAEP_PADDING:
/* Use the default parameters: SHA-1 for both hashes and no label. */
i = RSA_padding_add_PKCS1_OAEP_mgf1(buf, rsa_size, in, in_len,
NULL, 0, NULL, NULL);
break;
case RSA_NO_PADDING:
i = RSA_padding_add_none(buf, rsa_size, in, in_len);
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) {
goto err;
}
if (BN_bin2bn(buf, rsa_size, f) == NULL) {
goto err;
}
if (BN_ucmp(f, rsa->n) >= 0) {
/* usually the padding functions would catch this */
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (!BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx) ||
!BN_mod_exp_mont(result, f, rsa->e, rsa->n, ctx, rsa->mont_n)) {
goto err;
}
/* put in leading 0 bytes if the number is less than the length of the
* modulus */
if (!BN_bn2bin_padded(out, rsa_size, result)) {
OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
goto err;
}
*out_len = rsa_size;
ret = 1;
err:
if (ctx != NULL) {
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
return ret;
}
static int my_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
const unsigned rsa_size = RSA_size(rsa);
BIGNUM *f, *result;
uint8_t *buf = NULL;
BN_CTX *ctx = NULL;
int i, ret = 0;
printf("my encrypt\n");
if (rsa_size > OPENSSL_RSA_MAX_MODULUS_BITS) {
OPENSSL_PUT_ERROR(RSA, RSA_R_MODULUS_TOO_LARGE);
return 0;
}
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
if (BN_ucmp(rsa->n, rsa->e) <= 0) {
OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_E_VALUE);
return 0;
}
/* for large moduli, enforce exponent limit */
if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS &&
BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) {
OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_E_VALUE);
return 0;
}
ctx = BN_CTX_new();
if (ctx == NULL) {
goto err;
}
BN_CTX_start(ctx);
f = BN_CTX_get(ctx);
result = BN_CTX_get(ctx);
buf = OPENSSL_malloc(rsa_size);
if (!f || !result || !buf) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
i = RSA_padding_add_PKCS1_type_2(buf, rsa_size, in, in_len);
break;
case RSA_PKCS1_OAEP_PADDING:
/* Use the default parameters: SHA-1 for both hashes and no label. */
i = RSA_padding_add_PKCS1_OAEP_mgf1(buf, rsa_size, in, in_len,
NULL, 0, NULL, NULL);
break;
case RSA_NO_PADDING:
i = RSA_padding_add_none(buf, rsa_size, in, in_len);
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) {
goto err;
}
if (BN_bin2bn(buf, rsa_size, f) == NULL) {
goto err;
}
if (BN_ucmp(f, rsa->n) >= 0) {
/* usually the padding functions would catch this */
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
goto err;
}
if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) {
if (BN_MONT_CTX_set_locked(&rsa->_method_mod_n, &rsa->lock, rsa->n, ctx) ==
NULL) {
goto err;
}
}
printf("before my bn_mod_exp\n");
if (!rsa->meth->bn_mod_exp(result, f, rsa->e, rsa->n, ctx,
rsa->_method_mod_n)) {
goto err;
}
printf("after my bn_mod_exp\n");
/* put in leading 0 bytes if the number is less than the length of the
* modulus */
if (!BN_bn2bin_padded(out, rsa_size, result)) {
OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
goto err;
}
*out_len = rsa_size;
ret = 1;
err:
if (ctx != NULL) {
BN_CTX_end(ctx);
BN_CTX_free(ctx);
}
if (buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
return ret;
}
