static int decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, const uint8_t *in, size_t in_len, int padding) { const unsigned rsa_size = RSA_size(rsa); int r = -1; uint8_t *buf = NULL; int ret = 0; if (max_out < rsa_size) { OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } buf = OPENSSL_malloc(rsa_size); if (buf == NULL) { OPENSSL_PUT_ERROR(RSA, decrypt, ERR_R_MALLOC_FAILURE); goto err; } if (in_len != rsa_size) { OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN); goto err; } if (!RSA_private_transform(rsa, buf, in, rsa_size)) { OPENSSL_PUT_ERROR(RSA, decrypt, ERR_R_INTERNAL_ERROR); goto err; } switch (padding) { case RSA_PKCS1_PADDING: r = RSA_padding_check_PKCS1_type_2(out, rsa_size, buf, rsa_size); break; case RSA_PKCS1_OAEP_PADDING: /* Use the default parameters: SHA-1 for both hashes and no label. */ r = RSA_padding_check_PKCS1_OAEP_mgf1(out, rsa_size, buf, rsa_size, NULL, 0, NULL, NULL); break; case RSA_NO_PADDING: r = RSA_padding_check_none(out, rsa_size, buf, rsa_size); break; default: OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (r < 0) { OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_PADDING_CHECK_FAILED); } else { *out_len = r; ret = 1; } err: if (buf != NULL) { OPENSSL_cleanse(buf, rsa_size); OPENSSL_free(buf); } return ret; }
/* signing */ int rsa_default_sign_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, const uint8_t *in, size_t in_len, int padding) { const unsigned rsa_size = RSA_size(rsa); uint8_t *buf = NULL; int i, ret = 0; if (max_out < rsa_size) { OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } buf = OPENSSL_malloc(rsa_size); if (buf == NULL) { OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE); goto err; } switch (padding) { case RSA_PKCS1_PADDING: i = RSA_padding_add_PKCS1_type_1(buf, rsa_size, in, in_len); break; case RSA_NO_PADDING: i = RSA_padding_add_none(buf, rsa_size, in, in_len); break; default: OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (i <= 0) { goto err; } if (!RSA_private_transform(rsa, out, buf, rsa_size)) { goto err; } *out_len = rsa_size; ret = 1; err: if (buf != NULL) { OPENSSL_cleanse(buf, rsa_size); OPENSSL_free(buf); } return ret; }
static int my_decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, const uint8_t *in, size_t in_len, int padding) { const unsigned rsa_size = RSA_size(rsa); int r = -1; uint8_t *buf = NULL; int ret = 0; printf("my decrypt, file:%s line:%d \n", __FILE__, __LINE__); if (max_out < rsa_size) { OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } if (padding == RSA_NO_PADDING) { buf = out; } else { /* Allocate a temporary buffer to hold the padded plaintext. */ buf = OPENSSL_malloc(rsa_size); if (buf == NULL) { OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE); goto err; } } if (in_len != rsa_size) { OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN); goto err; } printf("my decrypt, file:%s line:%d \n", __FILE__, __LINE__); if (!RSA_private_transform(rsa, buf, in, rsa_size)) { goto err; } printf("my decrypt, file:%s line:%d \n", __FILE__, __LINE__); switch (padding) { case RSA_PKCS1_PADDING: r = RSA_padding_check_PKCS1_type_2(out, rsa_size, buf, rsa_size); break; case RSA_PKCS1_OAEP_PADDING: /* Use the default parameters: SHA-1 for both hashes and no label. */ r = RSA_padding_check_PKCS1_OAEP_mgf1(out, rsa_size, buf, rsa_size, NULL, 0, NULL, NULL); break; case RSA_NO_PADDING: r = rsa_size; break; default: OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } printf("my decrypt, file:%s line:%d \n", __FILE__, __LINE__); if (r < 0) { OPENSSL_PUT_ERROR(RSA, RSA_R_PADDING_CHECK_FAILED); } else { *out_len = r; ret = 1; } err: if (padding != RSA_NO_PADDING && buf != NULL) { OPENSSL_cleanse(buf, rsa_size); OPENSSL_free(buf); } return ret; }
int rsa_default_decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, const uint8_t *in, size_t in_len, int padding) { const unsigned rsa_size = RSA_size(rsa); uint8_t *buf = NULL; int ret = 0; if (max_out < rsa_size) { OPENSSL_PUT_ERROR(RSA, RSA_R_OUTPUT_BUFFER_TOO_SMALL); return 0; } if (padding == RSA_NO_PADDING) { buf = out; } else { // Allocate a temporary buffer to hold the padded plaintext. buf = OPENSSL_malloc(rsa_size); if (buf == NULL) { OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE); goto err; } } if (in_len != rsa_size) { OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN); goto err; } if (!RSA_private_transform(rsa, buf, in, rsa_size)) { goto err; } switch (padding) { case RSA_PKCS1_PADDING: ret = RSA_padding_check_PKCS1_type_2(out, out_len, rsa_size, buf, rsa_size); break; case RSA_PKCS1_OAEP_PADDING: // Use the default parameters: SHA-1 for both hashes and no label. ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, out_len, rsa_size, buf, rsa_size, NULL, 0, NULL, NULL); break; case RSA_NO_PADDING: *out_len = rsa_size; ret = 1; break; default: OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (!ret) { OPENSSL_PUT_ERROR(RSA, RSA_R_PADDING_CHECK_FAILED); } err: if (padding != RSA_NO_PADDING) { OPENSSL_free(buf); } return ret; }