static int decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
const unsigned rsa_size = RSA_size(rsa);
int r = -1;
uint8_t *buf = NULL;
int ret = 0;
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
buf = OPENSSL_malloc(rsa_size);
if (buf == NULL) {
OPENSSL_PUT_ERROR(RSA, decrypt, ERR_R_MALLOC_FAILURE);
goto err;
}
if (in_len != rsa_size) {
OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN);
goto err;
}
if (!RSA_private_transform(rsa, buf, in, rsa_size)) {
OPENSSL_PUT_ERROR(RSA, decrypt, ERR_R_INTERNAL_ERROR);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
r = RSA_padding_check_PKCS1_type_2(out, rsa_size, buf, rsa_size);
break;
case RSA_PKCS1_OAEP_PADDING:
/* Use the default parameters: SHA-1 for both hashes and no label. */
r = RSA_padding_check_PKCS1_OAEP_mgf1(out, rsa_size, buf, rsa_size,
NULL, 0, NULL, NULL);
break;
case RSA_NO_PADDING:
r = RSA_padding_check_none(out, rsa_size, buf, rsa_size);
break;
default:
OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (r < 0) {
OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_PADDING_CHECK_FAILED);
} else {
*out_len = r;
ret = 1;
}
err:
if (buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
return ret;
}
/* signing */
int rsa_default_sign_raw(RSA *rsa, size_t *out_len, uint8_t *out,
size_t max_out, const uint8_t *in, size_t in_len,
int padding) {
const unsigned rsa_size = RSA_size(rsa);
uint8_t *buf = NULL;
int i, ret = 0;
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
buf = OPENSSL_malloc(rsa_size);
if (buf == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
i = RSA_padding_add_PKCS1_type_1(buf, rsa_size, in, in_len);
break;
case RSA_NO_PADDING:
i = RSA_padding_add_none(buf, rsa_size, in, in_len);
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (i <= 0) {
goto err;
}
if (!RSA_private_transform(rsa, out, buf, rsa_size)) {
goto err;
}
*out_len = rsa_size;
ret = 1;
err:
if (buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
return ret;
}
static int my_decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
const unsigned rsa_size = RSA_size(rsa);
int r = -1;
uint8_t *buf = NULL;
int ret = 0;
printf("my decrypt, file:%s line:%d \n", __FILE__, __LINE__);
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
if (padding == RSA_NO_PADDING) {
buf = out;
} else {
/* Allocate a temporary buffer to hold the padded plaintext. */
buf = OPENSSL_malloc(rsa_size);
if (buf == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
}
if (in_len != rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN);
goto err;
}
printf("my decrypt, file:%s line:%d \n", __FILE__, __LINE__);
if (!RSA_private_transform(rsa, buf, in, rsa_size)) {
goto err;
}
printf("my decrypt, file:%s line:%d \n", __FILE__, __LINE__);
switch (padding) {
case RSA_PKCS1_PADDING:
r = RSA_padding_check_PKCS1_type_2(out, rsa_size, buf, rsa_size);
break;
case RSA_PKCS1_OAEP_PADDING:
/* Use the default parameters: SHA-1 for both hashes and no label. */
r = RSA_padding_check_PKCS1_OAEP_mgf1(out, rsa_size, buf, rsa_size,
NULL, 0, NULL, NULL);
break;
case RSA_NO_PADDING:
r = rsa_size;
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
printf("my decrypt, file:%s line:%d \n", __FILE__, __LINE__);
if (r < 0) {
OPENSSL_PUT_ERROR(RSA, RSA_R_PADDING_CHECK_FAILED);
} else {
*out_len = r;
ret = 1;
}
err:
if (padding != RSA_NO_PADDING && buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
return ret;
}
int rsa_default_decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
const unsigned rsa_size = RSA_size(rsa);
uint8_t *buf = NULL;
int ret = 0;
if (max_out < rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL);
return 0;
}
if (padding == RSA_NO_PADDING) {
buf = out;
} else {
// Allocate a temporary buffer to hold the padded plaintext.
buf = OPENSSL_malloc(rsa_size);
if (buf == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err;
}
}
if (in_len != rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN);
goto err;
}
if (!RSA_private_transform(rsa, buf, in, rsa_size)) {
goto err;
}
switch (padding) {
case RSA_PKCS1_PADDING:
ret =
RSA_padding_check_PKCS1_type_2(out, out_len, rsa_size, buf, rsa_size);
break;
case RSA_PKCS1_OAEP_PADDING:
// Use the default parameters: SHA-1 for both hashes and no label.
ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, out_len, rsa_size, buf,
rsa_size, NULL, 0, NULL, NULL);
break;
case RSA_NO_PADDING:
*out_len = rsa_size;
ret = 1;
break;
default:
OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
if (!ret) {
OPENSSL_PUT_ERROR(RSA, RSA_R_PADDING_CHECK_FAILED);
}
err:
if (padding != RSA_NO_PADDING) {
OPENSSL_free(buf);
}
return ret;
}
