int HMAC_Init_ex(HMAC_CTX *ctx, const void *key, size_t key_len, const EVP_MD *md, ENGINE *impl) { if (md == NULL) { md = ctx->md; } // If either |key| is non-NULL or |md| has changed, initialize with a new key // rather than rewinding the previous one. // // TODO(davidben,eroman): Passing the previous |md| with a NULL |key| is // ambiguous between using the empty key and reusing the previous key. There // exist callers which intend the latter, but the former is an awkward edge // case. Fix to API to avoid this. if (md != ctx->md || key != NULL) { uint8_t pad[EVP_MAX_MD_BLOCK_SIZE]; uint8_t key_block[EVP_MAX_MD_BLOCK_SIZE]; unsigned key_block_len; size_t block_size = EVP_MD_block_size(md); assert(block_size <= sizeof(key_block)); if (block_size < key_len) { // Long keys are hashed. if (!EVP_DigestInit_ex(&ctx->md_ctx, md, impl) || !EVP_DigestUpdate(&ctx->md_ctx, key, key_len) || !EVP_DigestFinal_ex(&ctx->md_ctx, key_block, &key_block_len)) { return 0; } } else { assert(key_len <= sizeof(key_block)); OPENSSL_memcpy(key_block, key, key_len); key_block_len = (unsigned)key_len; } // Keys are then padded with zeros. if (key_block_len != EVP_MAX_MD_BLOCK_SIZE) { OPENSSL_memset(&key_block[key_block_len], 0, sizeof(key_block) - key_block_len); } for (size_t i = 0; i < EVP_MAX_MD_BLOCK_SIZE; i++) { pad[i] = 0x36 ^ key_block[i]; } if (!EVP_DigestInit_ex(&ctx->i_ctx, md, impl) || !EVP_DigestUpdate(&ctx->i_ctx, pad, EVP_MD_block_size(md))) { return 0; } for (size_t i = 0; i < EVP_MAX_MD_BLOCK_SIZE; i++) { pad[i] = 0x5c ^ key_block[i]; } if (!EVP_DigestInit_ex(&ctx->o_ctx, md, impl) || !EVP_DigestUpdate(&ctx->o_ctx, pad, EVP_MD_block_size(md))) { return 0; } ctx->md = md; } if (!EVP_MD_CTX_copy_ex(&ctx->md_ctx, &ctx->i_ctx)) { return 0; } return 1; }
void EVP_tls_cbc_copy_mac(uint8_t *out, size_t md_size, const uint8_t *in, size_t in_len, size_t orig_len) { uint8_t rotated_mac1[EVP_MAX_MD_SIZE], rotated_mac2[EVP_MAX_MD_SIZE]; uint8_t *rotated_mac = rotated_mac1; uint8_t *rotated_mac_tmp = rotated_mac2; // mac_end is the index of |in| just after the end of the MAC. size_t mac_end = in_len; size_t mac_start = mac_end - md_size; assert(orig_len >= in_len); assert(in_len >= md_size); assert(md_size <= EVP_MAX_MD_SIZE); // scan_start contains the number of bytes that we can ignore because // the MAC's position can only vary by 255 bytes. size_t scan_start = 0; // This information is public so it's safe to branch based on it. if (orig_len > md_size + 255 + 1) { scan_start = orig_len - (md_size + 255 + 1); } size_t rotate_offset = 0; uint8_t mac_started = 0; OPENSSL_memset(rotated_mac, 0, md_size); for (size_t i = scan_start, j = 0; i < orig_len; i++, j++) { if (j >= md_size) { j -= md_size; } crypto_word_t is_mac_start = constant_time_eq_w(i, mac_start); mac_started |= is_mac_start; uint8_t mac_ended = constant_time_ge_8(i, mac_end); rotated_mac[j] |= in[i] & mac_started & ~mac_ended; // Save the offset that |mac_start| is mapped to. rotate_offset |= j & is_mac_start; } // Now rotate the MAC. We rotate in log(md_size) steps, one for each bit // position. for (size_t offset = 1; offset < md_size; offset <<= 1, rotate_offset >>= 1) { // Rotate by |offset| iff the corresponding bit is set in // |rotate_offset|, placing the result in |rotated_mac_tmp|. const uint8_t skip_rotate = (rotate_offset & 1) - 1; for (size_t i = 0, j = offset; i < md_size; i++, j++) { if (j >= md_size) { j -= md_size; } rotated_mac_tmp[i] = constant_time_select_8(skip_rotate, rotated_mac[i], rotated_mac[j]); } // Swap pointers so |rotated_mac| contains the (possibly) rotated value. // Note the number of iterations and thus the identity of these pointers is // public information. uint8_t *tmp = rotated_mac; rotated_mac = rotated_mac_tmp; rotated_mac_tmp = tmp; } OPENSSL_memcpy(out, rotated_mac, md_size); }
static int aead_aes_gcm_seal_scatter(const EVP_AEAD_CTX *ctx, uint8_t *out, uint8_t *out_tag, size_t *out_tag_len, size_t max_out_tag_len, const uint8_t *nonce, size_t nonce_len, const uint8_t *in, size_t in_len, const uint8_t *extra_in, size_t extra_in_len, const uint8_t *ad, size_t ad_len) { struct aead_aes_gcm_ctx *gcm_ctx = (struct aead_aes_gcm_ctx *) &ctx->state; if (extra_in_len + ctx->tag_len < ctx->tag_len) { OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TOO_LARGE); return 0; } if (max_out_tag_len < extra_in_len + ctx->tag_len) { OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BUFFER_TOO_SMALL); return 0; } if (nonce_len == 0) { OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_INVALID_NONCE_SIZE); return 0; } const AES_KEY *key = &gcm_ctx->ks.ks; GCM128_CONTEXT gcm; OPENSSL_memset(&gcm, 0, sizeof(gcm)); OPENSSL_memcpy(&gcm.gcm_key, &gcm_ctx->gcm_key, sizeof(gcm.gcm_key)); CRYPTO_gcm128_setiv(&gcm, key, nonce, nonce_len); if (ad_len > 0 && !CRYPTO_gcm128_aad(&gcm, ad, ad_len)) { return 0; } if (gcm_ctx->ctr) { if (!CRYPTO_gcm128_encrypt_ctr32(&gcm, key, in, out, in_len, gcm_ctx->ctr)) { return 0; } } else { if (!CRYPTO_gcm128_encrypt(&gcm, key, in, out, in_len)) { return 0; } } if (extra_in_len) { if (gcm_ctx->ctr) { if (!CRYPTO_gcm128_encrypt_ctr32(&gcm, key, extra_in, out_tag, extra_in_len, gcm_ctx->ctr)) { return 0; } } else { if (!CRYPTO_gcm128_encrypt(&gcm, key, extra_in, out_tag, extra_in_len)) { return 0; } } } CRYPTO_gcm128_tag(&gcm, out_tag + extra_in_len, ctx->tag_len); *out_tag_len = ctx->tag_len + extra_in_len; return 1; }
int BORINGSSL_self_test(void) { static const uint8_t kAESKey[16] = "BoringCrypto Key"; static const uint8_t kAESIV[16] = {0}; static const uint8_t kPlaintext[64] = "BoringCryptoModule FIPS KAT Encryption and Decryption Plaintext!"; static const uint8_t kAESCBCCiphertext[64] = { 0x87, 0x2d, 0x98, 0xc2, 0xcc, 0x31, 0x5b, 0x41, 0xe0, 0xfa, 0x7b, 0x0a, 0x71, 0xc0, 0x42, 0xbf, 0x4f, 0x61, 0xd0, 0x0d, 0x58, 0x8c, 0xf7, 0x05, 0xfb, 0x94, 0x89, 0xd3, 0xbc, 0xaa, 0x1a, 0x50, 0x45, 0x1f, 0xc3, 0x8c, 0xb8, 0x98, 0x86, 0xa3, 0xe3, 0x6c, 0xfc, 0xad, 0x3a, 0xb5, 0x59, 0x27, 0x7d, 0x21, 0x07, 0xca, 0x4c, 0x1d, 0x55, 0x34, 0xdd, 0x5a, 0x2d, 0xc4, 0xb4, 0xf5, 0xa8, #if !defined(BORINGSSL_FIPS_BREAK_AES_CBC) 0x35 #else 0x00 #endif }; static const uint8_t kAESGCMCiphertext[80] = { 0x4a, 0xd8, 0xe7, 0x7d, 0x78, 0xd7, 0x7d, 0x5e, 0xb2, 0x11, 0xb6, 0xc9, 0xa4, 0xbc, 0xb2, 0xae, 0xbe, 0x93, 0xd1, 0xb7, 0xfe, 0x65, 0xc1, 0x82, 0x2a, 0xb6, 0x71, 0x5f, 0x1a, 0x7c, 0xe0, 0x1b, 0x2b, 0xe2, 0x53, 0xfa, 0xa0, 0x47, 0xfa, 0xd7, 0x8f, 0xb1, 0x4a, 0xc4, 0xdc, 0x89, 0xf9, 0xb4, 0x14, 0x4d, 0xde, 0x95, 0xea, 0x29, 0x69, 0x76, 0x81, 0xa3, 0x5c, 0x33, 0xd8, 0x37, 0xd8, 0xfa, 0x47, 0x19, 0x46, 0x2f, 0xf1, 0x90, 0xb7, 0x61, 0x8f, 0x6f, 0xdd, 0x31, 0x3f, 0x6a, 0x64, #if !defined(BORINGSSL_FIPS_BREAK_AES_GCM) 0x0d #else 0x00 #endif }; static const DES_cblock kDESKey1 = {"BCMDESK1"}; static const DES_cblock kDESKey2 = {"BCMDESK2"}; static const DES_cblock kDESKey3 = {"BCMDESK3"}; static const DES_cblock kDESIV = {"BCMDESIV"}; static const uint8_t kDESCiphertext[64] = { 0xa4, 0x30, 0x7a, 0x4c, 0x1f, 0x60, 0x16, 0xd7, 0x4f, 0x41, 0xe1, 0xbb, 0x27, 0xc4, 0x27, 0x37, 0xd4, 0x7f, 0xb9, 0x10, 0xf8, 0xbc, 0xaf, 0x93, 0x91, 0xb8, 0x88, 0x24, 0xb1, 0xf6, 0xf8, 0xbd, 0x31, 0x96, 0x06, 0x76, 0xde, 0x32, 0xcd, 0x29, 0x29, 0xba, 0x70, 0x5f, 0xea, 0xc0, 0xcb, 0xde, 0xc7, 0x75, 0x90, 0xe0, 0x0f, 0x5e, 0x2c, 0x0d, 0x49, 0x20, 0xd5, 0x30, 0x83, 0xf8, 0x08, #if !defined(BORINGSSL_FIPS_BREAK_DES) 0x5a #else 0x00 #endif }; static const uint8_t kPlaintextSHA1[20] = { 0xc6, 0xf8, 0xc9, 0x63, 0x1c, 0x14, 0x23, 0x62, 0x9b, 0xbd, 0x55, 0x82, 0xf4, 0xd6, 0x1d, 0xf2, 0xab, 0x7d, 0xc8, #if !defined(BORINGSSL_FIPS_BREAK_SHA_1) 0x28 #else 0x00 #endif }; static const uint8_t kPlaintextSHA256[32] = { 0x37, 0xbd, 0x70, 0x53, 0x72, 0xfc, 0xd4, 0x03, 0x79, 0x70, 0xfb, 0x06, 0x95, 0xb1, 0x2a, 0x82, 0x48, 0xe1, 0x3e, 0xf2, 0x33, 0xfb, 0xef, 0x29, 0x81, 0x22, 0x45, 0x40, 0x43, 0x70, 0xce, #if !defined(BORINGSSL_FIPS_BREAK_SHA_256) 0x0f #else 0x00 #endif }; static const uint8_t kPlaintextSHA512[64] = { 0x08, 0x6a, 0x1c, 0x84, 0x61, 0x9d, 0x8e, 0xb3, 0xc0, 0x97, 0x4e, 0xa1, 0x9f, 0x9c, 0xdc, 0xaf, 0x3b, 0x5c, 0x31, 0xf0, 0xf2, 0x74, 0xc3, 0xbd, 0x6e, 0xd6, 0x1e, 0xb2, 0xbb, 0x34, 0x74, 0x72, 0x5c, 0x51, 0x29, 0x8b, 0x87, 0x3a, 0xa3, 0xf2, 0x25, 0x23, 0xd4, 0x1c, 0x82, 0x1b, 0xfe, 0xd3, 0xc6, 0xee, 0xb5, 0xd6, 0xaf, 0x07, 0x7b, 0x98, 0xca, 0xa7, 0x01, 0xf3, 0x94, 0xf3, 0x68, #if !defined(BORINGSSL_FIPS_BREAK_SHA_512) 0x14 #else 0x00 #endif }; static const uint8_t kRSASignature[256] = { 0x62, 0x66, 0x4b, 0xe3, 0xb1, 0xd2, 0x83, 0xf1, 0xa8, 0x56, 0x2b, 0x33, 0x60, 0x1e, 0xdb, 0x1e, 0x06, 0xf7, 0xa7, 0x1e, 0xa8, 0xef, 0x03, 0x4d, 0x0c, 0xf6, 0x83, 0x75, 0x7a, 0xf0, 0x14, 0xc7, 0xe2, 0x94, 0x3a, 0xb5, 0x67, 0x56, 0xa5, 0x48, 0x7f, 0x3a, 0xa5, 0xbf, 0xf7, 0x1d, 0x44, 0xa6, 0x34, 0xed, 0x9b, 0xd6, 0x51, 0xaa, 0x2c, 0x4e, 0xce, 0x60, 0x5f, 0xe9, 0x0e, 0xd5, 0xcd, 0xeb, 0x23, 0x27, 0xf8, 0xfb, 0x45, 0xe5, 0x34, 0x63, 0x77, 0x7f, 0x2e, 0x80, 0xcf, 0x9d, 0x2e, 0xfc, 0xe2, 0x50, 0x75, 0x29, 0x46, 0xf4, 0xaf, 0x91, 0xed, 0x36, 0xe1, 0x5e, 0xef, 0x66, 0xa1, 0xff, 0x27, 0xfc, 0x87, 0x7e, 0x60, 0x84, 0x0f, 0x54, 0x51, 0x56, 0x0f, 0x68, 0x99, 0xc0, 0x3f, 0xeb, 0xa5, 0xa0, 0x46, 0xb0, 0x86, 0x02, 0xb0, 0xc8, 0xe8, 0x46, 0x13, 0x06, 0xcd, 0xb7, 0x8a, 0xd0, 0x3b, 0x46, 0xd0, 0x14, 0x64, 0x53, 0x9b, 0x5b, 0x5e, 0x02, 0x45, 0xba, 0x6e, 0x7e, 0x0a, 0xb9, 0x9e, 0x62, 0xb7, 0xd5, 0x7a, 0x87, 0xea, 0xd3, 0x24, 0xa5, 0xef, 0xb3, 0xdc, 0x05, 0x9c, 0x04, 0x60, 0x4b, 0xde, 0xa8, 0x90, 0x08, 0x7b, 0x6a, 0x5f, 0xb4, 0x3f, 0xda, 0xc5, 0x1f, 0x6e, 0xd6, 0x15, 0xde, 0x65, 0xa4, 0x6e, 0x62, 0x9d, 0x8f, 0xa8, 0xbe, 0x86, 0xf6, 0x09, 0x90, 0x40, 0xa5, 0xf4, 0x23, 0xc5, 0xf6, 0x38, 0x86, 0x0d, 0x1c, 0xed, 0x4a, 0x0a, 0xae, 0xa4, 0x26, 0xc2, 0x2e, 0xd3, 0x13, 0x66, 0x61, 0xea, 0x35, 0x01, 0x0e, 0x13, 0xda, 0x78, 0x20, 0xae, 0x59, 0x5f, 0x9b, 0xa9, 0x6c, 0xf9, 0x1b, 0xdf, 0x76, 0x53, 0xc8, 0xa7, 0xf5, 0x63, 0x6d, 0xf3, 0xff, 0xfd, 0xaf, 0x75, 0x4b, 0xac, 0x67, 0xb1, 0x3c, 0xbf, 0x5e, 0xde, 0x73, 0x02, 0x6d, 0xd2, 0x0c, 0xb1, #if !defined(BORINGSSL_FIPS_BREAK_RSA_SIG) 0x64 #else 0x00 #endif }; const uint8_t kDRBGEntropy[48] = "BCM Known Answer Test DBRG Initial Entropy "; const uint8_t kDRBGPersonalization[18] = "BCMPersonalization"; const uint8_t kDRBGAD[16] = "BCM DRBG KAT AD "; const uint8_t kDRBGOutput[64] = { 0x1d, 0x63, 0xdf, 0x05, 0x51, 0x49, 0x22, 0x46, 0xcd, 0x9b, 0xc5, 0xbb, 0xf1, 0x5d, 0x44, 0xae, 0x13, 0x78, 0xb1, 0xe4, 0x7c, 0xf1, 0x96, 0x33, 0x3d, 0x60, 0xb6, 0x29, 0xd4, 0xbb, 0x6b, 0x44, 0xf9, 0xef, 0xd9, 0xf4, 0xa2, 0xba, 0x48, 0xea, 0x39, 0x75, 0x59, 0x32, 0xf7, 0x31, 0x2c, 0x98, 0x14, 0x2b, 0x49, 0xdf, 0x02, 0xb6, 0x5d, 0x71, 0x09, 0x50, 0xdb, 0x23, 0xdb, 0xe5, 0x22, #if !defined(BORINGSSL_FIPS_BREAK_DRBG) 0x95 #else 0x00 #endif }; const uint8_t kDRBGEntropy2[48] = "BCM Known Answer Test DBRG Reseed Entropy "; const uint8_t kDRBGReseedOutput[64] = { 0xa4, 0x77, 0x05, 0xdb, 0x14, 0x11, 0x76, 0x71, 0x42, 0x5b, 0xd8, 0xd7, 0xa5, 0x4f, 0x8b, 0x39, 0xf2, 0x10, 0x4a, 0x50, 0x5b, 0xa2, 0xc8, 0xf0, 0xbb, 0x3e, 0xa1, 0xa5, 0x90, 0x7d, 0x54, 0xd9, 0xc6, 0xb0, 0x96, 0xc0, 0x2b, 0x7e, 0x9b, 0xc9, 0xa1, 0xdd, 0x78, 0x2e, 0xd5, 0xa8, 0x66, 0x16, 0xbd, 0x18, 0x3c, 0xf2, 0xaa, 0x7a, 0x2b, 0x37, 0xf9, 0xab, 0x35, 0x64, 0x15, 0x01, 0x3f, 0xc4, }; const uint8_t kECDSASigR[32] = { 0x67, 0x80, 0xc5, 0xfc, 0x70, 0x27, 0x5e, 0x2c, 0x70, 0x61, 0xa0, 0xe7, 0x87, 0x7b, 0xb1, 0x74, 0xde, 0xad, 0xeb, 0x98, 0x87, 0x02, 0x7f, 0x3f, 0xa8, 0x36, 0x54, 0x15, 0x8b, 0xa7, 0xf5, #if !defined(BORINGSSL_FIPS_BREAK_ECDSA_SIG) 0x0c, #else 0x00, #endif }; const uint8_t kECDSASigS[32] = { 0xa5, 0x93, 0xe0, 0x23, 0x91, 0xe7, 0x4b, 0x8d, 0x77, 0x25, 0xa6, 0xba, 0x4d, 0xd9, 0x86, 0x77, 0xda, 0x7d, 0x8f, 0xef, 0xc4, 0x1a, 0xf0, 0xcc, 0x81, 0xe5, 0xea, 0x3f, 0xc2, 0x41, 0x7f, 0xd8, }; EVP_AEAD_CTX aead_ctx; EVP_AEAD_CTX_zero(&aead_ctx); RSA *rsa_key = NULL; EC_KEY *ec_key = NULL; ECDSA_SIG *sig = NULL; int ret = 0; AES_KEY aes_key; uint8_t aes_iv[16]; uint8_t output[256]; // AES-CBC Encryption KAT memcpy(aes_iv, kAESIV, sizeof(kAESIV)); if (AES_set_encrypt_key(kAESKey, 8 * sizeof(kAESKey), &aes_key) != 0) { goto err; } AES_cbc_encrypt(kPlaintext, output, sizeof(kPlaintext), &aes_key, aes_iv, AES_ENCRYPT); if (!check_test(kAESCBCCiphertext, output, sizeof(kAESCBCCiphertext), "AES-CBC Encryption KAT")) { goto err; } // AES-CBC Decryption KAT memcpy(aes_iv, kAESIV, sizeof(kAESIV)); if (AES_set_decrypt_key(kAESKey, 8 * sizeof(kAESKey), &aes_key) != 0) { goto err; } AES_cbc_encrypt(kAESCBCCiphertext, output, sizeof(kAESCBCCiphertext), &aes_key, aes_iv, AES_DECRYPT); if (!check_test(kPlaintext, output, sizeof(kPlaintext), "AES-CBC Decryption KAT")) { goto err; } size_t out_len; uint8_t nonce[EVP_AEAD_MAX_NONCE_LENGTH]; OPENSSL_memset(nonce, 0, sizeof(nonce)); if (!EVP_AEAD_CTX_init(&aead_ctx, EVP_aead_aes_128_gcm(), kAESKey, sizeof(kAESKey), 0, NULL)) { goto err; } // AES-GCM Encryption KAT if (!EVP_AEAD_CTX_seal(&aead_ctx, output, &out_len, sizeof(output), nonce, EVP_AEAD_nonce_length(EVP_aead_aes_128_gcm()), kPlaintext, sizeof(kPlaintext), NULL, 0) || !check_test(kAESGCMCiphertext, output, sizeof(kAESGCMCiphertext), "AES-GCM Encryption KAT")) { goto err; } // AES-GCM Decryption KAT if (!EVP_AEAD_CTX_open(&aead_ctx, output, &out_len, sizeof(output), nonce, EVP_AEAD_nonce_length(EVP_aead_aes_128_gcm()), kAESGCMCiphertext, sizeof(kAESGCMCiphertext), NULL, 0) || !check_test(kPlaintext, output, sizeof(kPlaintext), "AES-GCM Decryption KAT")) { goto err; } DES_key_schedule des1, des2, des3; DES_cblock des_iv; DES_set_key(&kDESKey1, &des1); DES_set_key(&kDESKey2, &des2); DES_set_key(&kDESKey3, &des3); // 3DES Encryption KAT memcpy(&des_iv, &kDESIV, sizeof(des_iv)); DES_ede3_cbc_encrypt(kPlaintext, output, sizeof(kPlaintext), &des1, &des2, &des3, &des_iv, DES_ENCRYPT); if (!check_test(kDESCiphertext, output, sizeof(kDESCiphertext), "3DES Encryption KAT")) { goto err; } // 3DES Decryption KAT memcpy(&des_iv, &kDESIV, sizeof(des_iv)); DES_ede3_cbc_encrypt(kDESCiphertext, output, sizeof(kDESCiphertext), &des1, &des2, &des3, &des_iv, DES_DECRYPT); if (!check_test(kPlaintext, output, sizeof(kPlaintext), "3DES Decryption KAT")) { goto err; } // SHA-1 KAT SHA1(kPlaintext, sizeof(kPlaintext), output); if (!check_test(kPlaintextSHA1, output, sizeof(kPlaintextSHA1), "SHA-1 KAT")) { goto err; } // SHA-256 KAT SHA256(kPlaintext, sizeof(kPlaintext), output); if (!check_test(kPlaintextSHA256, output, sizeof(kPlaintextSHA256), "SHA-256 KAT")) { goto err; } // SHA-512 KAT SHA512(kPlaintext, sizeof(kPlaintext), output); if (!check_test(kPlaintextSHA512, output, sizeof(kPlaintextSHA512), "SHA-512 KAT")) { goto err; } rsa_key = self_test_rsa_key(); if (rsa_key == NULL) { fprintf(stderr, "RSA KeyGen failed\n"); goto err; } // RSA Sign KAT unsigned sig_len; // Disable blinding for the power-on tests because it's not needed and // triggers an entropy draw. rsa_key->flags |= RSA_FLAG_NO_BLINDING; if (!RSA_sign(NID_sha256, kPlaintextSHA256, sizeof(kPlaintextSHA256), output, &sig_len, rsa_key) || !check_test(kRSASignature, output, sizeof(kRSASignature), "RSA Sign KAT")) { goto err; } // RSA Verify KAT if (!RSA_verify(NID_sha256, kPlaintextSHA256, sizeof(kPlaintextSHA256), kRSASignature, sizeof(kRSASignature), rsa_key)) { fprintf(stderr, "RSA Verify KAT failed.\n"); goto err; } ec_key = self_test_ecdsa_key(); if (ec_key == NULL) { fprintf(stderr, "ECDSA KeyGen failed\n"); goto err; } // ECDSA Sign/Verify PWCT // The 'k' value for ECDSA is fixed to avoid an entropy draw. ec_key->fixed_k = BN_new(); if (ec_key->fixed_k == NULL || !BN_set_word(ec_key->fixed_k, 42)) { fprintf(stderr, "Out of memory\n"); goto err; } sig = ECDSA_do_sign(kPlaintextSHA256, sizeof(kPlaintextSHA256), ec_key); uint8_t ecdsa_r_bytes[sizeof(kECDSASigR)]; uint8_t ecdsa_s_bytes[sizeof(kECDSASigS)]; if (sig == NULL || BN_num_bytes(sig->r) != sizeof(ecdsa_r_bytes) || !BN_bn2bin(sig->r, ecdsa_r_bytes) || BN_num_bytes(sig->s) != sizeof(ecdsa_s_bytes) || !BN_bn2bin(sig->s, ecdsa_s_bytes) || !check_test(kECDSASigR, ecdsa_r_bytes, sizeof(kECDSASigR), "ECDSA R") || !check_test(kECDSASigS, ecdsa_s_bytes, sizeof(kECDSASigS), "ECDSA S")) { fprintf(stderr, "ECDSA KAT failed.\n"); goto err; } // DBRG KAT CTR_DRBG_STATE drbg; if (!CTR_DRBG_init(&drbg, kDRBGEntropy, kDRBGPersonalization, sizeof(kDRBGPersonalization)) || !CTR_DRBG_generate(&drbg, output, sizeof(kDRBGOutput), kDRBGAD, sizeof(kDRBGAD)) || !check_test(kDRBGOutput, output, sizeof(kDRBGOutput), "DBRG Generate KAT") || !CTR_DRBG_reseed(&drbg, kDRBGEntropy2, kDRBGAD, sizeof(kDRBGAD)) || !CTR_DRBG_generate(&drbg, output, sizeof(kDRBGReseedOutput), kDRBGAD, sizeof(kDRBGAD)) || !check_test(kDRBGReseedOutput, output, sizeof(kDRBGReseedOutput), "DRBG Reseed KAT")) { goto err; } CTR_DRBG_clear(&drbg); CTR_DRBG_STATE kZeroDRBG; memset(&kZeroDRBG, 0, sizeof(kZeroDRBG)); if (!check_test(&kZeroDRBG, &drbg, sizeof(drbg), "DRBG Clear KAT")) { goto err; } ret = 1; err: EVP_AEAD_CTX_cleanup(&aead_ctx); RSA_free(rsa_key); EC_KEY_free(ec_key); ECDSA_SIG_free(sig); return ret; }
// bn_mul_part_recursive sets |r| to |a| * |b|, using |t| as scratch space. |r| // has length 4*|n|, |a| has length |n| + |tna|, |b| has length |n| + |tnb|, and // |t| has length 8*|n|. |n| must be a power of two. Additionally, we must have // 0 <= tna < n and 0 <= tnb < n, and |tna| and |tnb| must differ by at most // one. // // TODO(davidben): Make this take |size_t| and perhaps the actual lengths of |a| // and |b|. static void bn_mul_part_recursive(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, int n, int tna, int tnb, BN_ULONG *t) { // |n| is a power of two. assert(n != 0 && (n & (n - 1)) == 0); // Check |tna| and |tnb| are in range. assert(0 <= tna && tna < n); assert(0 <= tnb && tnb < n); assert(-1 <= tna - tnb && tna - tnb <= 1); int n2 = n * 2; if (n < 8) { bn_mul_normal(r, a, n + tna, b, n + tnb); OPENSSL_memset(r + n2 + tna + tnb, 0, n2 - tna - tnb); return; } // Split |a| and |b| into a0,a1 and b0,b1, where a0 and b0 have size |n|. |a1| // and |b1| have size |tna| and |tnb|, respectively. // Split |t| into t0,t1,t2,t3, each of size |n|, with the remaining 4*|n| used // for recursive calls. // Split |r| into r0,r1,r2,r3. We must contribute a0*b0 to r0,r1, a0*a1+b0*b1 // to r1,r2, and a1*b1 to r2,r3. The middle term we will compute as: // // a0*a1 + b0*b1 = (a0 - a1)*(b1 - b0) + a1*b1 + a0*b0 // t0 = a0 - a1 and t1 = b1 - b0. The result will be multiplied, so we XOR // their sign masks, giving the sign of (a0 - a1)*(b1 - b0). t0 and t1 // themselves store the absolute value. BN_ULONG neg = bn_abs_sub_part_words(t, a, &a[n], tna, n - tna, &t[n2]); neg ^= bn_abs_sub_part_words(&t[n], &b[n], b, tnb, tnb - n, &t[n2]); // Compute: // t2,t3 = t0 * t1 = |(a0 - a1)*(b1 - b0)| // r0,r1 = a0 * b0 // r2,r3 = a1 * b1 if (n == 8) { bn_mul_comba8(&t[n2], t, &t[n]); bn_mul_comba8(r, a, b); bn_mul_normal(&r[n2], &a[n], tna, &b[n], tnb); // |bn_mul_normal| only writes |tna| + |tna| words. Zero the rest. OPENSSL_memset(&r[n2 + tna + tnb], 0, sizeof(BN_ULONG) * (n2 - tna - tnb)); } else { BN_ULONG *p = &t[n2 * 2]; bn_mul_recursive(&t[n2], t, &t[n], n, 0, 0, p); bn_mul_recursive(r, a, b, n, 0, 0, p); OPENSSL_memset(&r[n2], 0, sizeof(BN_ULONG) * n2); if (tna < BN_MUL_RECURSIVE_SIZE_NORMAL && tnb < BN_MUL_RECURSIVE_SIZE_NORMAL) { bn_mul_normal(&r[n2], &a[n], tna, &b[n], tnb); } else { int i = n; for (;;) { i /= 2; if (i < tna || i < tnb) { // E.g., n == 16, i == 8 and tna == 11. |tna| and |tnb| are within one // of each other, so if |tna| is larger and tna > i, then we know // tnb >= i, and this call is valid. bn_mul_part_recursive(&r[n2], &a[n], &b[n], i, tna - i, tnb - i, p); break; } if (i == tna || i == tnb) { // If there is only a bottom half to the number, just do it. We know // the larger of |tna - i| and |tnb - i| is zero. The other is zero or // -1 by because of |tna| and |tnb| differ by at most one. bn_mul_recursive(&r[n2], &a[n], &b[n], i, tna - i, tnb - i, p); break; } // This loop will eventually terminate when |i| falls below // |BN_MUL_RECURSIVE_SIZE_NORMAL| because we know one of |tna| and |tnb| // exceeds that. } } } // t0,t1,c = r0,r1 + r2,r3 = a0*b0 + a1*b1 BN_ULONG c = bn_add_words(t, r, &r[n2], n2); // t2,t3,c = t0,t1,c + neg*t2,t3 = (a0 - a1)*(b1 - b0) + a1*b1 + a0*b0. // The second term is stored as the absolute value, so we do this with a // constant-time select. BN_ULONG c_neg = c - bn_sub_words(&t[n2 * 2], t, &t[n2], n2); BN_ULONG c_pos = c + bn_add_words(&t[n2], t, &t[n2], n2); bn_select_words(&t[n2], neg, &t[n2 * 2], &t[n2], n2); OPENSSL_COMPILE_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t), crypto_word_t_too_small); c = constant_time_select_w(neg, c_neg, c_pos); // We now have our three components. Add them together. // r1,r2,c = r1,r2 + t2,t3,c c += bn_add_words(&r[n], &r[n], &t[n2], n2); // Propagate the carry bit to the end. for (int i = n + n2; i < n2 + n2; i++) { BN_ULONG old = r[i]; r[i] = old + c; c = r[i] < old; } // The product should fit without carries. assert(c == 0); }
// bn_mul_recursive sets |r| to |a| * |b|, using |t| as scratch space. |r| has // length 2*|n2|, |a| has length |n2| + |dna|, |b| has length |n2| + |dnb|, and // |t| has length 4*|n2|. |n2| must be a power of two. Finally, we must have // -|BN_MUL_RECURSIVE_SIZE_NORMAL|/2 <= |dna| <= 0 and // -|BN_MUL_RECURSIVE_SIZE_NORMAL|/2 <= |dnb| <= 0. // // TODO(davidben): Simplify and |size_t| the calling convention around lengths // here. static void bn_mul_recursive(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, int n2, int dna, int dnb, BN_ULONG *t) { // |n2| is a power of two. assert(n2 != 0 && (n2 & (n2 - 1)) == 0); // Check |dna| and |dnb| are in range. assert(-BN_MUL_RECURSIVE_SIZE_NORMAL/2 <= dna && dna <= 0); assert(-BN_MUL_RECURSIVE_SIZE_NORMAL/2 <= dnb && dnb <= 0); // Only call bn_mul_comba 8 if n2 == 8 and the // two arrays are complete [steve] if (n2 == 8 && dna == 0 && dnb == 0) { bn_mul_comba8(r, a, b); return; } // Else do normal multiply if (n2 < BN_MUL_RECURSIVE_SIZE_NORMAL) { bn_mul_normal(r, a, n2 + dna, b, n2 + dnb); if (dna + dnb < 0) { OPENSSL_memset(&r[2 * n2 + dna + dnb], 0, sizeof(BN_ULONG) * -(dna + dnb)); } return; } // Split |a| and |b| into a0,a1 and b0,b1, where a0 and b0 have size |n|. // Split |t| into t0,t1,t2,t3, each of size |n|, with the remaining 4*|n| used // for recursive calls. // Split |r| into r0,r1,r2,r3. We must contribute a0*b0 to r0,r1, a0*a1+b0*b1 // to r1,r2, and a1*b1 to r2,r3. The middle term we will compute as: // // a0*a1 + b0*b1 = (a0 - a1)*(b1 - b0) + a1*b1 + a0*b0 // // Note that we know |n| >= |BN_MUL_RECURSIVE_SIZE_NORMAL|/2 above, so // |tna| and |tnb| are non-negative. int n = n2 / 2, tna = n + dna, tnb = n + dnb; // t0 = a0 - a1 and t1 = b1 - b0. The result will be multiplied, so we XOR // their sign masks, giving the sign of (a0 - a1)*(b1 - b0). t0 and t1 // themselves store the absolute value. BN_ULONG neg = bn_abs_sub_part_words(t, a, &a[n], tna, n - tna, &t[n2]); neg ^= bn_abs_sub_part_words(&t[n], &b[n], b, tnb, tnb - n, &t[n2]); // Compute: // t2,t3 = t0 * t1 = |(a0 - a1)*(b1 - b0)| // r0,r1 = a0 * b0 // r2,r3 = a1 * b1 if (n == 4 && dna == 0 && dnb == 0) { bn_mul_comba4(&t[n2], t, &t[n]); bn_mul_comba4(r, a, b); bn_mul_comba4(&r[n2], &a[n], &b[n]); } else if (n == 8 && dna == 0 && dnb == 0) { bn_mul_comba8(&t[n2], t, &t[n]); bn_mul_comba8(r, a, b); bn_mul_comba8(&r[n2], &a[n], &b[n]); } else { BN_ULONG *p = &t[n2 * 2]; bn_mul_recursive(&t[n2], t, &t[n], n, 0, 0, p); bn_mul_recursive(r, a, b, n, 0, 0, p); bn_mul_recursive(&r[n2], &a[n], &b[n], n, dna, dnb, p); } // t0,t1,c = r0,r1 + r2,r3 = a0*b0 + a1*b1 BN_ULONG c = bn_add_words(t, r, &r[n2], n2); // t2,t3,c = t0,t1,c + neg*t2,t3 = (a0 - a1)*(b1 - b0) + a1*b1 + a0*b0. // The second term is stored as the absolute value, so we do this with a // constant-time select. BN_ULONG c_neg = c - bn_sub_words(&t[n2 * 2], t, &t[n2], n2); BN_ULONG c_pos = c + bn_add_words(&t[n2], t, &t[n2], n2); bn_select_words(&t[n2], neg, &t[n2 * 2], &t[n2], n2); OPENSSL_COMPILE_ASSERT(sizeof(BN_ULONG) <= sizeof(crypto_word_t), crypto_word_t_too_small); c = constant_time_select_w(neg, c_neg, c_pos); // We now have our three components. Add them together. // r1,r2,c = r1,r2 + t2,t3,c c += bn_add_words(&r[n], &r[n], &t[n2], n2); // Propagate the carry bit to the end. for (int i = n + n2; i < n2 + n2; i++) { BN_ULONG old = r[i]; r[i] = old + c; c = r[i] < old; } // The product should fit without carries. assert(c == 0); }
/* r is 2*n words in size, * a and b are both n words in size. (There's not actually a 'b' here ...) * n must be a power of 2. * We multiply and return the result. * t must be 2*n words in size * We calculate * a[0]*b[0] * a[0]*b[0]+a[1]*b[1]+(a[0]-a[1])*(b[1]-b[0]) * a[1]*b[1] */ static void bn_sqr_recursive(BN_ULONG *r, const BN_ULONG *a, int n2, BN_ULONG *t) { int n = n2 / 2; int zero, c1; BN_ULONG ln, lo, *p; if (n2 == 4) { bn_sqr_comba4(r, a); return; } else if (n2 == 8) { bn_sqr_comba8(r, a); return; } if (n2 < BN_SQR_RECURSIVE_SIZE_NORMAL) { bn_sqr_normal(r, a, n2, t); return; } /* r=(a[0]-a[1])*(a[1]-a[0]) */ c1 = bn_cmp_words(a, &(a[n]), n); zero = 0; if (c1 > 0) { bn_sub_words(t, a, &(a[n]), n); } else if (c1 < 0) { bn_sub_words(t, &(a[n]), a, n); } else { zero = 1; } /* The result will always be negative unless it is zero */ p = &(t[n2 * 2]); if (!zero) { bn_sqr_recursive(&(t[n2]), t, n, p); } else { OPENSSL_memset(&(t[n2]), 0, n2 * sizeof(BN_ULONG)); } bn_sqr_recursive(r, a, n, p); bn_sqr_recursive(&(r[n2]), &(a[n]), n, p); /* t[32] holds (a[0]-a[1])*(a[1]-a[0]), it is negative or zero * r[10] holds (a[0]*b[0]) * r[32] holds (b[1]*b[1]) */ c1 = (int)(bn_add_words(t, r, &(r[n2]), n2)); /* t[32] is negative */ c1 -= (int)(bn_sub_words(&(t[n2]), t, &(t[n2]), n2)); /* t[32] holds (a[0]-a[1])*(a[1]-a[0])+(a[0]*a[0])+(a[1]*a[1]) * r[10] holds (a[0]*a[0]) * r[32] holds (a[1]*a[1]) * c1 holds the carry bits */ c1 += (int)(bn_add_words(&(r[n]), &(r[n]), &(t[n2]), n2)); if (c1) { p = &(r[n + n2]); lo = *p; ln = (lo + c1) & BN_MASK2; *p = ln; /* The overflow will stop before we over write * words we should not overwrite */ if (ln < (BN_ULONG)c1) { do { p++; lo = *p; ln = (lo + 1) & BN_MASK2; *p = ln; } while (ln == 0); } } }
/* tnX may not be negative but less than n */ static void bn_mul_part_recursive(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n, int tna, int tnb, BN_ULONG *t) { int i, j, n2 = n * 2; int c1, c2, neg; BN_ULONG ln, lo, *p; if (n < 8) { bn_mul_normal(r, a, n + tna, b, n + tnb); return; } /* r=(a[0]-a[1])*(b[1]-b[0]) */ c1 = bn_cmp_part_words(a, &(a[n]), tna, n - tna); c2 = bn_cmp_part_words(&(b[n]), b, tnb, tnb - n); neg = 0; switch (c1 * 3 + c2) { case -4: bn_sub_part_words(t, &(a[n]), a, tna, tna - n); /* - */ bn_sub_part_words(&(t[n]), b, &(b[n]), tnb, n - tnb); /* - */ break; case -3: /* break; */ case -2: bn_sub_part_words(t, &(a[n]), a, tna, tna - n); /* - */ bn_sub_part_words(&(t[n]), &(b[n]), b, tnb, tnb - n); /* + */ neg = 1; break; case -1: case 0: case 1: /* break; */ case 2: bn_sub_part_words(t, a, &(a[n]), tna, n - tna); /* + */ bn_sub_part_words(&(t[n]), b, &(b[n]), tnb, n - tnb); /* - */ neg = 1; break; case 3: /* break; */ case 4: bn_sub_part_words(t, a, &(a[n]), tna, n - tna); bn_sub_part_words(&(t[n]), &(b[n]), b, tnb, tnb - n); break; } if (n == 8) { bn_mul_comba8(&(t[n2]), t, &(t[n])); bn_mul_comba8(r, a, b); bn_mul_normal(&(r[n2]), &(a[n]), tna, &(b[n]), tnb); OPENSSL_memset(&(r[n2 + tna + tnb]), 0, sizeof(BN_ULONG) * (n2 - tna - tnb)); } else { p = &(t[n2 * 2]); bn_mul_recursive(&(t[n2]), t, &(t[n]), n, 0, 0, p); bn_mul_recursive(r, a, b, n, 0, 0, p); i = n / 2; /* If there is only a bottom half to the number, * just do it */ if (tna > tnb) { j = tna - i; } else { j = tnb - i; } if (j == 0) { bn_mul_recursive(&(r[n2]), &(a[n]), &(b[n]), i, tna - i, tnb - i, p); OPENSSL_memset(&(r[n2 + i * 2]), 0, sizeof(BN_ULONG) * (n2 - i * 2)); } else if (j > 0) { /* eg, n == 16, i == 8 and tn == 11 */ bn_mul_part_recursive(&(r[n2]), &(a[n]), &(b[n]), i, tna - i, tnb - i, p); OPENSSL_memset(&(r[n2 + tna + tnb]), 0, sizeof(BN_ULONG) * (n2 - tna - tnb)); } else { /* (j < 0) eg, n == 16, i == 8 and tn == 5 */ OPENSSL_memset(&(r[n2]), 0, sizeof(BN_ULONG) * n2); if (tna < BN_MUL_RECURSIVE_SIZE_NORMAL && tnb < BN_MUL_RECURSIVE_SIZE_NORMAL) { bn_mul_normal(&(r[n2]), &(a[n]), tna, &(b[n]), tnb); } else { for (;;) { i /= 2; /* these simplified conditions work * exclusively because difference * between tna and tnb is 1 or 0 */ if (i < tna || i < tnb) { bn_mul_part_recursive(&(r[n2]), &(a[n]), &(b[n]), i, tna - i, tnb - i, p); break; } else if (i == tna || i == tnb) { bn_mul_recursive(&(r[n2]), &(a[n]), &(b[n]), i, tna - i, tnb - i, p); break; } } } } } /* t[32] holds (a[0]-a[1])*(b[1]-b[0]), c1 is the sign * r[10] holds (a[0]*b[0]) * r[32] holds (b[1]*b[1]) */ c1 = (int)(bn_add_words(t, r, &(r[n2]), n2)); if (neg) { /* if t[32] is negative */ c1 -= (int)(bn_sub_words(&(t[n2]), t, &(t[n2]), n2)); } else { /* Might have a carry */ c1 += (int)(bn_add_words(&(t[n2]), &(t[n2]), t, n2)); } /* t[32] holds (a[0]-a[1])*(b[1]-b[0])+(a[0]*b[0])+(a[1]*b[1]) * r[10] holds (a[0]*b[0]) * r[32] holds (b[1]*b[1]) * c1 holds the carry bits */ c1 += (int)(bn_add_words(&(r[n]), &(r[n]), &(t[n2]), n2)); if (c1) { p = &(r[n + n2]); lo = *p; ln = (lo + c1) & BN_MASK2; *p = ln; /* The overflow will stop before we over write * words we should not overwrite */ if (ln < (BN_ULONG)c1) { do { p++; lo = *p; ln = (lo + 1) & BN_MASK2; *p = ln; } while (ln == 0); } } }
/* dnX may not be positive, but n2/2+dnX has to be */ static void bn_mul_recursive(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n2, int dna, int dnb, BN_ULONG *t) { int n = n2 / 2, c1, c2; int tna = n + dna, tnb = n + dnb; unsigned int neg, zero; BN_ULONG ln, lo, *p; /* Only call bn_mul_comba 8 if n2 == 8 and the * two arrays are complete [steve] */ if (n2 == 8 && dna == 0 && dnb == 0) { bn_mul_comba8(r, a, b); return; } /* Else do normal multiply */ if (n2 < BN_MUL_RECURSIVE_SIZE_NORMAL) { bn_mul_normal(r, a, n2 + dna, b, n2 + dnb); if ((dna + dnb) < 0) { OPENSSL_memset(&r[2 * n2 + dna + dnb], 0, sizeof(BN_ULONG) * -(dna + dnb)); } return; } /* r=(a[0]-a[1])*(b[1]-b[0]) */ c1 = bn_cmp_part_words(a, &(a[n]), tna, n - tna); c2 = bn_cmp_part_words(&(b[n]), b, tnb, tnb - n); zero = neg = 0; switch (c1 * 3 + c2) { case -4: bn_sub_part_words(t, &(a[n]), a, tna, tna - n); /* - */ bn_sub_part_words(&(t[n]), b, &(b[n]), tnb, n - tnb); /* - */ break; case -3: zero = 1; break; case -2: bn_sub_part_words(t, &(a[n]), a, tna, tna - n); /* - */ bn_sub_part_words(&(t[n]), &(b[n]), b, tnb, tnb - n); /* + */ neg = 1; break; case -1: case 0: case 1: zero = 1; break; case 2: bn_sub_part_words(t, a, &(a[n]), tna, n - tna); /* + */ bn_sub_part_words(&(t[n]), b, &(b[n]), tnb, n - tnb); /* - */ neg = 1; break; case 3: zero = 1; break; case 4: bn_sub_part_words(t, a, &(a[n]), tna, n - tna); bn_sub_part_words(&(t[n]), &(b[n]), b, tnb, tnb - n); break; } if (n == 4 && dna == 0 && dnb == 0) { /* XXX: bn_mul_comba4 could take extra args to do this well */ if (!zero) { bn_mul_comba4(&(t[n2]), t, &(t[n])); } else { OPENSSL_memset(&(t[n2]), 0, 8 * sizeof(BN_ULONG)); } bn_mul_comba4(r, a, b); bn_mul_comba4(&(r[n2]), &(a[n]), &(b[n])); } else if (n == 8 && dna == 0 && dnb == 0) { /* XXX: bn_mul_comba8 could take extra args to do this well */ if (!zero) { bn_mul_comba8(&(t[n2]), t, &(t[n])); } else { OPENSSL_memset(&(t[n2]), 0, 16 * sizeof(BN_ULONG)); } bn_mul_comba8(r, a, b); bn_mul_comba8(&(r[n2]), &(a[n]), &(b[n])); } else { p = &(t[n2 * 2]); if (!zero) { bn_mul_recursive(&(t[n2]), t, &(t[n]), n, 0, 0, p); } else { OPENSSL_memset(&(t[n2]), 0, n2 * sizeof(BN_ULONG)); } bn_mul_recursive(r, a, b, n, 0, 0, p); bn_mul_recursive(&(r[n2]), &(a[n]), &(b[n]), n, dna, dnb, p); } /* t[32] holds (a[0]-a[1])*(b[1]-b[0]), c1 is the sign * r[10] holds (a[0]*b[0]) * r[32] holds (b[1]*b[1]) */ c1 = (int)(bn_add_words(t, r, &(r[n2]), n2)); if (neg) { /* if t[32] is negative */ c1 -= (int)(bn_sub_words(&(t[n2]), t, &(t[n2]), n2)); } else { /* Might have a carry */ c1 += (int)(bn_add_words(&(t[n2]), &(t[n2]), t, n2)); } /* t[32] holds (a[0]-a[1])*(b[1]-b[0])+(a[0]*b[0])+(a[1]*b[1]) * r[10] holds (a[0]*b[0]) * r[32] holds (b[1]*b[1]) * c1 holds the carry bits */ c1 += (int)(bn_add_words(&(r[n]), &(r[n]), &(t[n2]), n2)); if (c1) { p = &(r[n + n2]); lo = *p; ln = (lo + c1) & BN_MASK2; *p = ln; /* The overflow will stop before we over write * words we should not overwrite */ if (ln < (BN_ULONG)c1) { do { p++; lo = *p; ln = (lo + 1) & BN_MASK2; *p = ln; } while (ln == 0); } } }