int RSA_verify_pss_mgf1(RSA *rsa, const uint8_t *msg, size_t msg_len,
const EVP_MD *md, const EVP_MD *mgf1_md, int salt_len,
const uint8_t *sig, size_t sig_len) {
if (msg_len != EVP_MD_size(md)) {
OPENSSL_PUT_ERROR(RSA, RSA_R_INVALID_MESSAGE_LENGTH);
return 0;
}
size_t em_len = RSA_size(rsa);
uint8_t *em = OPENSSL_malloc(em_len);
if (em == NULL) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
return 0;
}
int ret = 0;
if (!RSA_verify_raw(rsa, &em_len, em, em_len, sig, sig_len, RSA_NO_PADDING)) {
goto err;
}
if (em_len != RSA_size(rsa)) {
OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
goto err;
}
ret = RSA_verify_PKCS1_PSS_mgf1(rsa, msg, md, mgf1_md, em, salt_len);
err:
OPENSSL_free(em);
return ret;
}
int RSA_public_decrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
int padding) {
size_t out_len;
if (!RSA_verify_raw(rsa, &out_len, to, RSA_size(rsa), from, flen, padding)) {
return -1;
}
return out_len;
}
int RSA_verify(int hash_nid, const uint8_t *msg, size_t msg_len,
const uint8_t *sig, size_t sig_len, RSA *rsa) {
const size_t rsa_size = RSA_size(rsa);
uint8_t *buf = NULL;
int ret = 0;
uint8_t *signed_msg = NULL;
size_t signed_msg_len, len;
int signed_msg_is_alloced = 0;
if (rsa->meth->verify) {
return rsa->meth->verify(hash_nid, msg, msg_len, sig, sig_len, rsa);
}
if (sig_len != rsa_size) {
OPENSSL_PUT_ERROR(RSA, RSA_verify, RSA_R_WRONG_SIGNATURE_LENGTH);
return 0;
}
if (hash_nid == NID_md5_sha1 && msg_len != SSL_SIG_LENGTH) {
OPENSSL_PUT_ERROR(RSA, RSA_verify, RSA_R_INVALID_MESSAGE_LENGTH);
return 0;
}
buf = OPENSSL_malloc(rsa_size);
if (!buf) {
OPENSSL_PUT_ERROR(RSA, RSA_verify, ERR_R_MALLOC_FAILURE);
return 0;
}
if (!RSA_verify_raw(rsa, &len, buf, rsa_size, sig, sig_len,
RSA_PKCS1_PADDING)) {
goto out;
}
if (!pkcs1_prefixed_msg(&signed_msg, &signed_msg_len, &signed_msg_is_alloced,
hash_nid, msg, msg_len)) {
goto out;
}
if (len != signed_msg_len || CRYPTO_memcmp(buf, signed_msg, len) != 0) {
OPENSSL_PUT_ERROR(RSA, RSA_verify, RSA_R_BAD_SIGNATURE);
goto out;
}
ret = 1;
out:
if (buf != NULL) {
OPENSSL_free(buf);
}
if (signed_msg_is_alloced) {
OPENSSL_free(signed_msg);
}
return ret;
}
int RSA_verify(int hash_nid, const uint8_t *msg, size_t msg_len,
const uint8_t *sig, size_t sig_len, RSA *rsa) {
if (rsa->n == NULL || rsa->e == NULL) {
OPENSSL_PUT_ERROR(RSA, RSA_R_VALUE_MISSING);
return 0;
}
const size_t rsa_size = RSA_size(rsa);
uint8_t *buf = NULL;
int ret = 0;
uint8_t *signed_msg = NULL;
size_t signed_msg_len = 0, len;
int signed_msg_is_alloced = 0;
if (hash_nid == NID_md5_sha1 && msg_len != SSL_SIG_LENGTH) {
OPENSSL_PUT_ERROR(RSA, RSA_R_INVALID_MESSAGE_LENGTH);
return 0;
}
buf = OPENSSL_malloc(rsa_size);
if (!buf) {
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
return 0;
}
if (!RSA_verify_raw(rsa, &len, buf, rsa_size, sig, sig_len,
RSA_PKCS1_PADDING)) {
goto out;
}
if (!RSA_add_pkcs1_prefix(&signed_msg, &signed_msg_len,
&signed_msg_is_alloced, hash_nid, msg, msg_len)) {
goto out;
}
/* Check that no other information follows the hash value (FIPS 186-4 Section
* 5.5) and it matches the expected hash. */
if (len != signed_msg_len || OPENSSL_memcmp(buf, signed_msg, len) != 0) {
OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_SIGNATURE);
goto out;
}
ret = 1;
out:
OPENSSL_free(buf);
if (signed_msg_is_alloced) {
OPENSSL_free(signed_msg);
}
return ret;
}
static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig,
size_t siglen, const uint8_t *tbs,
size_t tbslen) {
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
size_t rslen;
const size_t key_len = EVP_PKEY_size(ctx->pkey);
if (rctx->md) {
switch (rctx->pad_mode) {
case RSA_PKCS1_PADDING:
return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen, sig, siglen, rsa);
case RSA_PKCS1_PSS_PADDING:
if (!setup_tbuf(rctx, ctx) ||
!RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen,
RSA_NO_PADDING) ||
!RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md, rctx->mgf1md,
rctx->tbuf, rctx->saltlen)) {
return 0;
}
return 1;
default:
return 0;
}
}
if (!setup_tbuf(rctx, ctx) ||
!RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen,
rctx->pad_mode) ||
rslen != tbslen ||
CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) {
return 0;
}
return 1;
}
int RSA_public_decrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
int padding) {
size_t out_len;
if (!RSA_verify_raw(rsa, &out_len, to, RSA_size(rsa), from, flen, padding)) {
return -1;
}
if (out_len > INT_MAX) {
OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
return -1;
}
return out_len;
}
static int pkey_rsa_verify_recover(EVP_PKEY_CTX *ctx, uint8_t *out,
size_t *out_len, const uint8_t *sig,
size_t sig_len) {
RSA_PKEY_CTX *rctx = ctx->data;
RSA *rsa = ctx->pkey->pkey.rsa;
const size_t key_len = EVP_PKEY_size(ctx->pkey);
if (out == NULL) {
*out_len = key_len;
return 1;
}
if (*out_len < key_len) {
OPENSSL_PUT_ERROR(EVP, EVP_R_BUFFER_TOO_SMALL);
return 0;
}
if (!setup_tbuf(rctx, ctx)) {
return 0;
}
if (rctx->md == NULL) {
const int ret = RSA_public_decrypt(sig_len, sig, rctx->tbuf,
ctx->pkey->pkey.rsa, rctx->pad_mode);
if (ret < 0) {
return 0;
}
*out_len = ret;
memcpy(out, rctx->tbuf, *out_len);
return 1;
}
if (rctx->pad_mode != RSA_PKCS1_PADDING) {
return 0;
}
uint8_t *asn1_prefix;
size_t asn1_prefix_len;
int asn1_prefix_allocated;
if (!RSA_add_pkcs1_prefix(&asn1_prefix, &asn1_prefix_len,
&asn1_prefix_allocated, EVP_MD_type(rctx->md), NULL,
0)) {
return 0;
}
size_t rslen;
int ok = 1;
if (!RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, sig_len,
RSA_PKCS1_PADDING) ||
rslen < asn1_prefix_len ||
CRYPTO_memcmp(rctx->tbuf, asn1_prefix, asn1_prefix_len) != 0) {
ok = 0;
}
if (asn1_prefix_allocated) {
OPENSSL_free(asn1_prefix);
}
if (!ok) {
return 0;
}
const size_t result_len = rslen - asn1_prefix_len;
if (result_len != EVP_MD_size(rctx->md)) {
return 0;
}
if (out != NULL) {
memcpy(out, rctx->tbuf + asn1_prefix_len, result_len);
}
*out_len = result_len;
return 1;
}