static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen, const unsigned char *in, size_t inlen) { int ret; RSA_PKEY_CTX *rctx = ctx->data; if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) { int klen = RSA_size(ctx->pkey->pkey.rsa); if (!setup_tbuf(rctx, ctx)) return -1; if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen, in, inlen, rctx->oaep_label, rctx->oaep_labellen, rctx->md, rctx->mgf1md)) return -1; ret = RSA_public_encrypt(klen, rctx->tbuf, out, ctx->pkey->pkey.rsa, RSA_NO_PADDING); } else ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa, rctx->pad_mode); if (ret < 0) return ret; *outlen = ret; return 1; }
static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen, const uint8_t *in, size_t inlen) { RSA_PKEY_CTX *rctx = ctx->data; RSA *rsa = ctx->pkey->pkey.rsa; const size_t key_len = EVP_PKEY_size(ctx->pkey); if (!out) { *outlen = key_len; return 1; } if (*outlen < key_len) { OPENSSL_PUT_ERROR(EVP, pkey_rsa_encrypt, EVP_R_BUFFER_TOO_SMALL); return 0; } if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) { if (!setup_tbuf(rctx, ctx) || !RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, key_len, in, inlen, rctx->oaep_label, rctx->oaep_labellen, rctx->md, rctx->mgf1md) || !RSA_encrypt(rsa, outlen, out, *outlen, rctx->tbuf, key_len, RSA_NO_PADDING)) { return 0; } return 1; } return RSA_encrypt(rsa, outlen, out, *outlen, in, inlen, rctx->pad_mode); }
int rsa_default_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, const uint8_t *in, size_t in_len, int padding) { const unsigned rsa_size = RSA_size(rsa); BIGNUM *f, *result; uint8_t *buf = NULL; BN_CTX *ctx = NULL; int i, ret = 0; if (max_out < rsa_size) { OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } if (!check_modulus_and_exponent_sizes(rsa)) { return 0; } ctx = BN_CTX_new(); if (ctx == NULL) { goto err; } BN_CTX_start(ctx); f = BN_CTX_get(ctx); result = BN_CTX_get(ctx); buf = OPENSSL_malloc(rsa_size); if (!f || !result || !buf) { OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE); goto err; } switch (padding) { case RSA_PKCS1_PADDING: i = RSA_padding_add_PKCS1_type_2(buf, rsa_size, in, in_len); break; case RSA_PKCS1_OAEP_PADDING: /* Use the default parameters: SHA-1 for both hashes and no label. */ i = RSA_padding_add_PKCS1_OAEP_mgf1(buf, rsa_size, in, in_len, NULL, 0, NULL, NULL); break; case RSA_NO_PADDING: i = RSA_padding_add_none(buf, rsa_size, in, in_len); break; default: OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (i <= 0) { goto err; } if (BN_bin2bn(buf, rsa_size, f) == NULL) { goto err; } if (BN_ucmp(f, rsa->n) >= 0) { /* usually the padding functions would catch this */ OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); goto err; } if (!BN_MONT_CTX_set_locked(&rsa->mont_n, &rsa->lock, rsa->n, ctx) || !BN_mod_exp_mont(result, f, rsa->e, rsa->n, ctx, rsa->mont_n)) { goto err; } /* put in leading 0 bytes if the number is less than the length of the * modulus */ if (!BN_bn2bin_padded(out, rsa_size, result)) { OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR); goto err; } *out_len = rsa_size; ret = 1; err: if (ctx != NULL) { BN_CTX_end(ctx); BN_CTX_free(ctx); } if (buf != NULL) { OPENSSL_cleanse(buf, rsa_size); OPENSSL_free(buf); } return ret; }
static int my_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, const uint8_t *in, size_t in_len, int padding) { const unsigned rsa_size = RSA_size(rsa); BIGNUM *f, *result; uint8_t *buf = NULL; BN_CTX *ctx = NULL; int i, ret = 0; printf("my encrypt\n"); if (rsa_size > OPENSSL_RSA_MAX_MODULUS_BITS) { OPENSSL_PUT_ERROR(RSA, RSA_R_MODULUS_TOO_LARGE); return 0; } if (max_out < rsa_size) { OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } if (BN_ucmp(rsa->n, rsa->e) <= 0) { OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_E_VALUE); return 0; } /* for large moduli, enforce exponent limit */ if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS && BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) { OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_E_VALUE); return 0; } ctx = BN_CTX_new(); if (ctx == NULL) { goto err; } BN_CTX_start(ctx); f = BN_CTX_get(ctx); result = BN_CTX_get(ctx); buf = OPENSSL_malloc(rsa_size); if (!f || !result || !buf) { OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE); goto err; } switch (padding) { case RSA_PKCS1_PADDING: i = RSA_padding_add_PKCS1_type_2(buf, rsa_size, in, in_len); break; case RSA_PKCS1_OAEP_PADDING: /* Use the default parameters: SHA-1 for both hashes and no label. */ i = RSA_padding_add_PKCS1_OAEP_mgf1(buf, rsa_size, in, in_len, NULL, 0, NULL, NULL); break; case RSA_NO_PADDING: i = RSA_padding_add_none(buf, rsa_size, in, in_len); break; default: OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (i <= 0) { goto err; } if (BN_bin2bn(buf, rsa_size, f) == NULL) { goto err; } if (BN_ucmp(f, rsa->n) >= 0) { /* usually the padding functions would catch this */ OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); goto err; } if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) { if (BN_MONT_CTX_set_locked(&rsa->_method_mod_n, &rsa->lock, rsa->n, ctx) == NULL) { goto err; } } printf("before my bn_mod_exp\n"); if (!rsa->meth->bn_mod_exp(result, f, rsa->e, rsa->n, ctx, rsa->_method_mod_n)) { goto err; } printf("after my bn_mod_exp\n"); /* put in leading 0 bytes if the number is less than the length of the * modulus */ if (!BN_bn2bin_padded(out, rsa_size, result)) { OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR); goto err; } *out_len = rsa_size; ret = 1; err: if (ctx != NULL) { BN_CTX_end(ctx); BN_CTX_free(ctx); } if (buf != NULL) { OPENSSL_cleanse(buf, rsa_size); OPENSSL_free(buf); } return ret; }