static isc_result_t opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) { dst_key_t *key = dctx->key; RSA *rsa = key->opaque; isc_region_t r; /* note: ISC_SHA1_DIGESTLENGTH > ISC_MD5_DIGESTLENGTH */ unsigned char digest[ISC_SHA1_DIGESTLENGTH]; unsigned int siglen = 0; int status; int type; unsigned int digestlen; char *message; unsigned long err; const char* file; int line; REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 || dctx->key->key_alg == DST_ALG_RSASHA1); isc_buffer_availableregion(sig, &r); if (r.length < (unsigned int) RSA_size(rsa)) return (ISC_R_NOSPACE); if (dctx->key->key_alg == DST_ALG_RSAMD5) { isc_md5_t *md5ctx = dctx->opaque; isc_md5_final(md5ctx, digest); type = NID_md5; digestlen = ISC_MD5_DIGESTLENGTH; } else { isc_sha1_t *sha1ctx = dctx->opaque; isc_sha1_final(sha1ctx, digest); type = NID_sha1; digestlen = ISC_SHA1_DIGESTLENGTH; } status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa); if (status == 0) { err = ERR_peek_error_line(&file, &line); if (err != 0U) { message = ERR_error_string(err, NULL); fprintf(stderr, "%s:%s:%d\n", message, file ? file : "", line); } return (dst__openssl_toresult(DST_R_OPENSSLFAILURE)); } isc_buffer_add(sig, siglen); return (ISC_R_SUCCESS); }
static isc_result_t opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) { dst_key_t *key = dctx->key; isc_region_t r; unsigned int siglen = 0; #if USE_EVP EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx; EVP_PKEY *pkey = key->keydata.pkey; #else RSA *rsa = key->keydata.rsa; /* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */ unsigned char digest[PREFIXLEN + ISC_SHA512_DIGESTLENGTH]; int status; int type = 0; unsigned int digestlen = 0; char *message; unsigned long err; const char* file; int line; #if OPENSSL_VERSION_NUMBER < 0x00908000L unsigned int prefixlen = 0; const unsigned char *prefix = NULL; #endif #endif REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 || dctx->key->key_alg == DST_ALG_RSASHA1 || dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 || dctx->key->key_alg == DST_ALG_RSASHA256 || dctx->key->key_alg == DST_ALG_RSASHA512); isc_buffer_availableregion(sig, &r); #if USE_EVP if (r.length < (unsigned int) EVP_PKEY_size(pkey)) return (ISC_R_NOSPACE); if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) { return (ISC_R_FAILURE); } #else if (r.length < (unsigned int) RSA_size(rsa)) return (ISC_R_NOSPACE); switch (dctx->key->key_alg) { case DST_ALG_RSAMD5: { isc_md5_t *md5ctx = dctx->ctxdata.md5ctx; isc_md5_final(md5ctx, digest); type = NID_md5; digestlen = ISC_MD5_DIGESTLENGTH; } break; case DST_ALG_RSASHA1: case DST_ALG_NSEC3RSASHA1: { isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx; isc_sha1_final(sha1ctx, digest); type = NID_sha1; digestlen = ISC_SHA1_DIGESTLENGTH; } break; case DST_ALG_RSASHA256: { isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx; isc_sha256_final(digest, sha256ctx); digestlen = ISC_SHA256_DIGESTLENGTH; #if OPENSSL_VERSION_NUMBER < 0x00908000L prefix = sha256_prefix; prefixlen = sizeof(sha256_prefix); #else type = NID_sha256; #endif } break; case DST_ALG_RSASHA512: { isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx; isc_sha512_final(digest, sha512ctx); digestlen = ISC_SHA512_DIGESTLENGTH; #if OPENSSL_VERSION_NUMBER < 0x00908000L prefix = sha512_prefix; prefixlen = sizeof(sha512_prefix); #else type = NID_sha512; #endif } break; default: INSIST(0); } #if OPENSSL_VERSION_NUMBER < 0x00908000L switch (dctx->key->key_alg) { case DST_ALG_RSAMD5: case DST_ALG_RSASHA1: case DST_ALG_NSEC3RSASHA1: INSIST(type != 0); status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa); break; case DST_ALG_RSASHA256: case DST_ALG_RSASHA512: INSIST(prefix != NULL); INSIST(prefixlen != 0); INSIST(prefixlen + digestlen <= sizeof(digest)); memmove(digest + prefixlen, digest, digestlen); memcpy(digest, prefix, prefixlen); status = RSA_private_encrypt(digestlen + prefixlen, digest, r.base, rsa, RSA_PKCS1_PADDING); if (status < 0) status = 0; else siglen = status; break; default: INSIST(0); } #else INSIST(type != 0); status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa); #endif if (status == 0) { err = ERR_peek_error_line(&file, &line); if (err != 0U) { message = ERR_error_string(err, NULL); } return (dst__openssl_toresult(DST_R_OPENSSLFAILURE)); } #endif isc_buffer_add(sig, siglen); return (ISC_R_SUCCESS); }