void RTMP::ComputeRC4Keys(Crypto& crypto,const UInt8* pubKey,UInt32 pubKeySize,const UInt8* farPubKey,UInt32 farPubKeySize,const Buffer& sharedSecret,RC4_KEY& decryptKey,RC4_KEY& encryptKey) {
UInt8 hash[HMAC_KEY_SIZE];
RC4_set_key(&decryptKey, 16, crypto.hmac(EVP_sha256(),sharedSecret.data(),sharedSecret.size(),pubKey,pubKeySize,hash));
RC4_set_key(&encryptKey, 16, crypto.hmac(EVP_sha256(),sharedSecret.data(),sharedSecret.size(),farPubKey,farPubKeySize,hash));
//bring the keys to correct cursor
RC4(&encryptKey, 1536, AlignData, AlignData);
}
void RTMFP::ComputeAsymetricKeys(const Buffer& sharedSecret, const UInt8* initiatorNonce,UInt16 initNonceSize,
const UInt8* responderNonce,UInt16 respNonceSize,
UInt8* requestKey,UInt8* responseKey) {
UInt8 mdp1[HMAC_KEY_SIZE];
UInt8 mdp2[HMAC_KEY_SIZE];
Crypto crypto;
// doing HMAC-SHA256 of one side
crypto.hmac(EVP_sha256(),responderNonce,respNonceSize,initiatorNonce,initNonceSize,mdp1);
// doing HMAC-SHA256 of the other side
crypto.hmac(EVP_sha256(),initiatorNonce,initNonceSize,responderNonce,respNonceSize,mdp2);
// now doing HMAC-sha256 of both result with the shared secret DH key
crypto.hmac(EVP_sha256(),sharedSecret.data(),sharedSecret.size(),mdp1,HMAC_KEY_SIZE,requestKey);
crypto.hmac(EVP_sha256(),sharedSecret.data(),sharedSecret.size(),mdp2,HMAC_KEY_SIZE,responseKey);
}
void RTMP::WriteDigestAndKey(Crypto& crypto,UInt8* data,const UInt8* challengeKey,bool middleKey) {
UInt16 serverDigestOffset = RTMP::GetDigestPos(data, middleKey);
UInt8 content[1504];
memcpy(content, data+1, serverDigestOffset-1);
memcpy(content + serverDigestOffset-1, data + serverDigestOffset + HMAC_KEY_SIZE,1505 - serverDigestOffset);
UInt8 hash[HMAC_KEY_SIZE];
crypto.hmac(EVP_sha256(),FMSKey,36,content,sizeof(content),hash);
//put the digest in place
memcpy(data+serverDigestOffset,hash,sizeof(hash));
//compute the key
crypto.hmac(EVP_sha256(),FMSKey,sizeof(FMSKey),challengeKey,HMAC_KEY_SIZE,hash);
//generate the hash
crypto.hmac(EVP_sha256(),hash,HMAC_KEY_SIZE,data + 1537,1504,data+3041);
}