int rsa_greater_than_pow2(const BIGNUM *b, int n) {
if (BN_is_negative(b) || n == INT_MAX) {
return 0;
}
int b_bits = BN_num_bits(b);
return b_bits > n + 1 || (b_bits == n + 1 && !BN_is_pow2(b));
}
// generate_prime sets |out| to a prime with length |bits| such that |out|-1 is
// relatively prime to |e|. If |p| is non-NULL, |out| will also not be close to
// |p|. |sqrt2| must be ⌊2^(bits-1)×√2⌋ (or a slightly overestimate for large
// sizes), and |pow2_bits_100| must be 2^(bits-100).
static int generate_prime(BIGNUM *out, int bits, const BIGNUM *e,
const BIGNUM *p, const BIGNUM *sqrt2,
const BIGNUM *pow2_bits_100, BN_CTX *ctx,
BN_GENCB *cb) {
if (bits < 128 || (bits % BN_BITS2) != 0) {
OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
return 0;
}
assert(BN_is_pow2(pow2_bits_100));
assert(BN_is_bit_set(pow2_bits_100, bits - 100));
// See FIPS 186-4 appendix B.3.3, steps 4 and 5. Note |bits| here is nlen/2.
// Use the limit from steps 4.7 and 5.8 for most values of |e|. When |e| is 3,
// the 186-4 limit is too low, so we use a higher one. Note this case is not
// reachable from |RSA_generate_key_fips|.
if (bits >= INT_MAX/32) {
OPENSSL_PUT_ERROR(RSA, RSA_R_MODULUS_TOO_LARGE);
return 0;
}
int limit = BN_is_word(e, 3) ? bits * 32 : bits * 5;
int ret = 0, tries = 0, rand_tries = 0;
BN_CTX_start(ctx);
BIGNUM *tmp = BN_CTX_get(ctx);
if (tmp == NULL) {
goto err;
}
for (;;) {
// Generate a random number of length |bits| where the bottom bit is set
// (steps 4.2, 4.3, 5.2 and 5.3) and the top bit is set (implied by the
// bound checked below in steps 4.4 and 5.5).
if (!BN_rand(out, bits, BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ODD) ||
!BN_GENCB_call(cb, BN_GENCB_GENERATED, rand_tries++)) {
goto err;
}
if (p != NULL) {
// If |p| and |out| are too close, try again (step 5.4).
if (!bn_abs_sub_consttime(tmp, out, p, ctx)) {
goto err;
}
if (BN_cmp(tmp, pow2_bits_100) <= 0) {
continue;
}
}
// If out < 2^(bits-1)×√2, try again (steps 4.4 and 5.5). This is equivalent
// to out <= ⌊2^(bits-1)×√2⌋, or out <= sqrt2 for FIPS key sizes.
//
// For larger keys, the comparison is approximate, leaning towards
// retrying. That is, we reject a negligible fraction of primes that are
// within the FIPS bound, but we will never accept a prime outside the
// bound, ensuring the resulting RSA key is the right size.
if (BN_cmp(out, sqrt2) <= 0) {
continue;
}
// RSA key generation's bottleneck is discarding composites. If it fails
// trial division, do not bother computing a GCD or performing Rabin-Miller.
if (!bn_odd_number_is_obviously_composite(out)) {
// Check gcd(out-1, e) is one (steps 4.5 and 5.6).
int relatively_prime;
if (!BN_sub(tmp, out, BN_value_one()) ||
!bn_is_relatively_prime(&relatively_prime, tmp, e, ctx)) {
goto err;
}
if (relatively_prime) {
// Test |out| for primality (steps 4.5.1 and 5.6.1).
int is_probable_prime;
if (!BN_primality_test(&is_probable_prime, out, BN_prime_checks, ctx, 0,
cb)) {
goto err;
}
if (is_probable_prime) {
ret = 1;
goto err;
}
}
}
// If we've tried too many times to find a prime, abort (steps 4.7 and
// 5.8).
tries++;
if (tries >= limit) {
OPENSSL_PUT_ERROR(RSA, RSA_R_TOO_MANY_ITERATIONS);
goto err;
}
if (!BN_GENCB_call(cb, 2, tries)) {
goto err;
}
}
err:
BN_CTX_end(ctx);
return ret;
}
