enum ssl_private_key_result_t ssl_private_key_sign(
SSL *ssl, uint8_t *out, size_t *out_len, size_t max_out,
uint16_t signature_algorithm, const uint8_t *in, size_t in_len) {
if (ssl->cert->key_method != NULL) {
if (ssl->cert->key_method->sign != NULL) {
return ssl->cert->key_method->sign(ssl, out, out_len, max_out,
signature_algorithm, in, in_len);
}
/* TODO(davidben): Remove support for |sign_digest|-only
* |SSL_PRIVATE_KEY_METHOD|s. */
const EVP_MD *md;
int curve;
if (!is_rsa_pkcs1(&md, signature_algorithm) &&
!is_ecdsa(&curve, &md, signature_algorithm)) {
OPENSSL_PUT_ERROR(SSL, SSL_R_UNSUPPORTED_PROTOCOL_FOR_CUSTOM_KEY);
return ssl_private_key_failure;
}
uint8_t hash[EVP_MAX_MD_SIZE];
unsigned hash_len;
if (!EVP_Digest(in, in_len, hash, &hash_len, md, NULL)) {
return ssl_private_key_failure;
}
return ssl->cert->key_method->sign_digest(ssl, out, out_len, max_out, md,
hash, hash_len);
}
const EVP_MD *md;
if (is_rsa_pkcs1(&md, signature_algorithm) &&
ssl3_protocol_version(ssl) < TLS1_3_VERSION) {
return ssl_sign_rsa_pkcs1(ssl, out, out_len, max_out, md, in, in_len)
? ssl_private_key_success
: ssl_private_key_failure;
}
int curve;
if (is_ecdsa(&curve, &md, signature_algorithm)) {
return ssl_sign_ecdsa(ssl, out, out_len, max_out, curve, md, in, in_len)
? ssl_private_key_success
: ssl_private_key_failure;
}
if (is_rsa_pss(&md, signature_algorithm)) {
return ssl_sign_rsa_pss(ssl, out, out_len, max_out, md, in, in_len)
? ssl_private_key_success
: ssl_private_key_failure;
}
OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
return ssl_private_key_failure;
}
enum ssl_private_key_result_t ssl_private_key_sign(
SSL *ssl, uint8_t *out, size_t *out_len, size_t max_out,
uint16_t signature_algorithm, const uint8_t *in, size_t in_len) {
if (ssl->cert->key_method != NULL) {
/* For now, custom private keys can only handle pre-TLS-1.3 signature
* algorithms.
*
* TODO(davidben): Switch SSL_PRIVATE_KEY_METHOD to message-based APIs. */
const EVP_MD *md;
int curve;
if (!is_rsa_pkcs1(&md, signature_algorithm) &&
!is_ecdsa(&curve, &md, signature_algorithm)) {
OPENSSL_PUT_ERROR(SSL, SSL_R_UNSUPPORTED_PROTOCOL_FOR_CUSTOM_KEY);
return ssl_private_key_failure;
}
uint8_t hash[EVP_MAX_MD_SIZE];
unsigned hash_len;
if (!EVP_Digest(in, in_len, hash, &hash_len, md, NULL)) {
return ssl_private_key_failure;
}
return ssl->cert->key_method->sign(ssl, out, out_len, max_out, md, hash,
hash_len);
}
const EVP_MD *md;
if (is_rsa_pkcs1(&md, signature_algorithm)) {
return ssl_sign_rsa_pkcs1(ssl, out, out_len, max_out, md, in, in_len)
? ssl_private_key_success
: ssl_private_key_failure;
}
int curve;
if (is_ecdsa(&curve, &md, signature_algorithm)) {
return ssl_sign_ecdsa(ssl, out, out_len, max_out, curve, md, in, in_len)
? ssl_private_key_success
: ssl_private_key_failure;
}
if (is_rsa_pss(&md, signature_algorithm) &&
ssl3_protocol_version(ssl) >= TLS1_3_VERSION) {
return ssl_sign_rsa_pss(ssl, out, out_len, max_out, md, in, in_len)
? ssl_private_key_success
: ssl_private_key_failure;
}
OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
return ssl_private_key_failure;
}
int ssl_private_key_supports_signature_algorithm(SSL *ssl,
uint16_t signature_algorithm) {
const EVP_MD *md;
if (is_rsa_pkcs1(&md, signature_algorithm) &&
ssl3_protocol_version(ssl) < TLS1_3_VERSION) {
return ssl_private_key_type(ssl) == NID_rsaEncryption;
}
int curve;
if (is_ecdsa(&curve, &md, signature_algorithm)) {
int type = ssl_private_key_type(ssl);
if (!ssl_is_ecdsa_key_type(type)) {
return 0;
}
/* Prior to TLS 1.3, ECDSA curves did not match the signature algorithm. */
if (ssl3_protocol_version(ssl) < TLS1_3_VERSION) {
return 1;
}
return curve != NID_undef && type == curve;
}
if (is_rsa_pss(&md, signature_algorithm)) {
if (ssl_private_key_type(ssl) != NID_rsaEncryption) {
return 0;
}
/* Ensure the RSA key is large enough for the hash. RSASSA-PSS requires that
* emLen be at least hLen + sLen + 2. Both hLen and sLen are the size of the
* hash in TLS. Reasonable RSA key sizes are large enough for the largest
* defined RSASSA-PSS algorithm, but 1024-bit RSA is slightly too large for
* SHA-512. 1024-bit RSA is sometimes used for test credentials, so check
* the size to fall back to another algorithm. */
if (ssl_private_key_max_signature_len(ssl) < 2 * EVP_MD_size(md) + 2) {
return 0;
}
/* RSA-PSS is only supported by message-based private keys. */
if (ssl->cert->key_method != NULL && ssl->cert->key_method->sign == NULL) {
return 0;
}
return 1;
}
return 0;
}
int ssl_private_key_supports_signature_algorithm(SSL *ssl,
uint16_t signature_algorithm) {
const EVP_MD *md;
if (is_rsa_pkcs1(&md, signature_algorithm)) {
return ssl_private_key_type(ssl) == EVP_PKEY_RSA;
}
int curve;
if (is_ecdsa(&curve, &md, signature_algorithm)) {
if (ssl_private_key_type(ssl) != EVP_PKEY_EC) {
return 0;
}
/* For non-custom keys, also check the curve matches. Custom private keys
* must instead configure the signature algorithms accordingly. */
if (ssl3_protocol_version(ssl) >= TLS1_3_VERSION &&
ssl->cert->key_method == NULL) {
EC_KEY *ec_key = EVP_PKEY_get0_EC_KEY(ssl->cert->privatekey);
if (curve == NID_undef ||
EC_GROUP_get_curve_name(EC_KEY_get0_group(ec_key)) != curve) {
return 0;
}
}
return 1;
}
if (is_rsa_pss(&md, signature_algorithm)) {
if (ssl3_protocol_version(ssl) < TLS1_3_VERSION ||
ssl_private_key_type(ssl) != EVP_PKEY_RSA) {
return 0;
}
/* Ensure the RSA key is large enough for the hash. RSASSA-PSS requires that
* emLen be at least hLen + sLen + 2. Both hLen and sLen are the size of the
* hash in TLS. Reasonable RSA key sizes are large enough for the largest
* defined RSASSA-PSS algorithm, but 1024-bit RSA is slightly too large for
* SHA-512. 1024-bit RSA is sometimes used for test credentials, so check
* the size to fall back to another algorithm. */
if (ssl_private_key_max_signature_len(ssl) < 2 * EVP_MD_size(md) + 2) {
return 0;
}
return 1;
}
return 0;
}
int ssl_public_key_verify(SSL *ssl, const uint8_t *signature,
size_t signature_len, uint16_t signature_algorithm,
EVP_PKEY *pkey, const uint8_t *in, size_t in_len) {
const EVP_MD *md;
if (is_rsa_pkcs1(&md, signature_algorithm) &&
ssl3_protocol_version(ssl) < TLS1_3_VERSION) {
return ssl_verify_rsa_pkcs1(ssl, signature, signature_len, md, pkey, in,
in_len);
}
int curve;
if (is_ecdsa(&curve, &md, signature_algorithm)) {
return ssl_verify_ecdsa(ssl, signature, signature_len, curve, md, pkey, in,
in_len);
}
if (is_rsa_pss(&md, signature_algorithm)) {
return ssl_verify_rsa_pss(ssl, signature, signature_len, md, pkey, in,
in_len);
}
OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
return 0;
}