void* CPDF_CryptoHandler::CryptStart(uint32_t objnum,
uint32_t gennum,
bool bEncrypt) {
if (m_Cipher == FXCIPHER_NONE) {
return this;
}
if (m_Cipher == FXCIPHER_AES && m_KeyLen == 32) {
AESCryptContext* pContext = FX_Alloc(AESCryptContext, 1);
pContext->m_bIV = true;
pContext->m_BlockOffset = 0;
CRYPT_AESSetKey(pContext->m_Context, 16, m_EncryptKey, 32, bEncrypt);
if (bEncrypt) {
for (int i = 0; i < 16; i++) {
pContext->m_Block[i] = (uint8_t)rand();
}
CRYPT_AESSetIV(pContext->m_Context, pContext->m_Block);
}
return pContext;
}
uint8_t key1[48];
PopulateKey(objnum, gennum, key1);
if (m_Cipher == FXCIPHER_AES) {
FXSYS_memcpy(key1 + m_KeyLen + 5, "sAlT", 4);
}
uint8_t realkey[16];
CRYPT_MD5Generate(
key1, m_Cipher == FXCIPHER_AES ? m_KeyLen + 9 : m_KeyLen + 5, realkey);
int realkeylen = m_KeyLen + 5;
if (realkeylen > 16) {
realkeylen = 16;
}
if (m_Cipher == FXCIPHER_AES) {
AESCryptContext* pContext = FX_Alloc(AESCryptContext, 1);
pContext->m_bIV = true;
pContext->m_BlockOffset = 0;
CRYPT_AESSetKey(pContext->m_Context, 16, realkey, 16, bEncrypt);
if (bEncrypt) {
for (int i = 0; i < 16; i++) {
pContext->m_Block[i] = (uint8_t)rand();
}
CRYPT_AESSetIV(pContext->m_Context, pContext->m_Block);
}
return pContext;
}
CRYPT_rc4_context* pContext = FX_Alloc(CRYPT_rc4_context, 1);
CRYPT_ArcFourSetup(pContext, realkey, realkeylen);
return pContext;
}
void CPDF_SecurityHandler::AES256_SetPerms(CPDF_Dictionary* pEncryptDict,
uint32_t permissions,
FX_BOOL bEncryptMetadata,
const uint8_t* key) {
uint8_t buf[16];
buf[0] = (uint8_t)permissions;
buf[1] = (uint8_t)(permissions >> 8);
buf[2] = (uint8_t)(permissions >> 16);
buf[3] = (uint8_t)(permissions >> 24);
buf[4] = 0xff;
buf[5] = 0xff;
buf[6] = 0xff;
buf[7] = 0xff;
buf[8] = bEncryptMetadata ? 'T' : 'F';
buf[9] = 'a';
buf[10] = 'd';
buf[11] = 'b';
uint8_t* aes = FX_Alloc(uint8_t, 2048);
CRYPT_AESSetKey(aes, 16, key, 32, TRUE);
uint8_t iv[16], buf1[16];
FXSYS_memset(iv, 0, 16);
CRYPT_AESSetIV(aes, iv);
CRYPT_AESEncrypt(aes, buf1, buf, 16);
FX_Free(aes);
pEncryptDict->SetAtString("Perms", CFX_ByteString(buf1, 16));
}
void CPDF_StandardSecurityHandler::AES256_SetPerms(CPDF_Dictionary* pEncryptDict, FX_DWORD permissions,
FX_BOOL bEncryptMetadata, FX_LPCBYTE key)
{
FX_BYTE buf[16];
buf[0] = (FX_BYTE)permissions;
buf[1] = (FX_BYTE)(permissions >> 8);
buf[2] = (FX_BYTE)(permissions >> 16);
buf[3] = (FX_BYTE)(permissions >> 24);
buf[4] = 0xff;
buf[5] = 0xff;
buf[6] = 0xff;
buf[7] = 0xff;
buf[8] = bEncryptMetadata ? 'T' : 'F';
buf[9] = 'a';
buf[10] = 'd';
buf[11] = 'b';
FX_BYTE* aes = FX_Alloc(FX_BYTE, 2048);
CRYPT_AESSetKey(aes, 16, key, 32, TRUE);
FX_BYTE iv[16], buf1[16];
FXSYS_memset32(iv, 0, 16);
CRYPT_AESSetIV(aes, iv);
CRYPT_AESEncrypt(aes, buf1, buf, 16);
FX_Free(aes);
pEncryptDict->SetAtString(FX_BSTRC("Perms"), CFX_ByteString(buf1, 16));
}
void CPDF_CryptoHandler::CryptBlock(bool bEncrypt,
uint32_t objnum,
uint32_t gennum,
const uint8_t* src_buf,
uint32_t src_size,
uint8_t* dest_buf,
uint32_t& dest_size) {
if (m_Cipher == FXCIPHER_NONE) {
FXSYS_memcpy(dest_buf, src_buf, src_size);
return;
}
uint8_t realkey[16];
int realkeylen = 16;
if (m_Cipher != FXCIPHER_AES || m_KeyLen != 32) {
uint8_t key1[32];
PopulateKey(objnum, gennum, key1);
if (m_Cipher == FXCIPHER_AES) {
FXSYS_memcpy(key1 + m_KeyLen + 5, "sAlT", 4);
}
CRYPT_MD5Generate(
key1, m_Cipher == FXCIPHER_AES ? m_KeyLen + 9 : m_KeyLen + 5, realkey);
realkeylen = m_KeyLen + 5;
if (realkeylen > 16) {
realkeylen = 16;
}
}
if (m_Cipher == FXCIPHER_AES) {
CRYPT_AESSetKey(m_pAESContext, 16, m_KeyLen == 32 ? m_EncryptKey : realkey,
m_KeyLen, bEncrypt);
if (bEncrypt) {
uint8_t iv[16];
for (int i = 0; i < 16; i++) {
iv[i] = (uint8_t)rand();
}
CRYPT_AESSetIV(m_pAESContext, iv);
FXSYS_memcpy(dest_buf, iv, 16);
int nblocks = src_size / 16;
CRYPT_AESEncrypt(m_pAESContext, dest_buf + 16, src_buf, nblocks * 16);
uint8_t padding[16];
FXSYS_memcpy(padding, src_buf + nblocks * 16, src_size % 16);
FXSYS_memset(padding + src_size % 16, 16 - src_size % 16,
16 - src_size % 16);
CRYPT_AESEncrypt(m_pAESContext, dest_buf + nblocks * 16 + 16, padding,
16);
dest_size = 32 + nblocks * 16;
} else {
CRYPT_AESSetIV(m_pAESContext, src_buf);
CRYPT_AESDecrypt(m_pAESContext, dest_buf, src_buf + 16, src_size - 16);
dest_size = src_size - 16;
dest_size -= dest_buf[dest_size - 1];
}
} else {
ASSERT(dest_size == src_size);
if (dest_buf != src_buf) {
FXSYS_memcpy(dest_buf, src_buf, src_size);
}
CRYPT_ArcFourCryptBlock(dest_buf, dest_size, realkey, realkeylen);
}
}
void CPDF_StandardSecurityHandler::AES256_SetPassword(
CPDF_Dictionary* pEncryptDict,
const uint8_t* password,
FX_DWORD size,
FX_BOOL bOwner,
const uint8_t* key) {
uint8_t sha[128];
CRYPT_SHA1Start(sha);
CRYPT_SHA1Update(sha, key, 32);
CRYPT_SHA1Update(sha, (uint8_t*)"hello", 5);
uint8_t digest[20];
CRYPT_SHA1Finish(sha, digest);
CFX_ByteString ukey = pEncryptDict->GetString(FX_BSTRC("U"));
uint8_t digest1[48];
if (m_Revision >= 6) {
Revision6_Hash(password, size, digest,
(bOwner ? (const uint8_t*)ukey : NULL), digest1);
} else {
CRYPT_SHA256Start(sha);
CRYPT_SHA256Update(sha, password, size);
CRYPT_SHA256Update(sha, digest, 8);
if (bOwner) {
CRYPT_SHA256Update(sha, ukey, ukey.GetLength());
}
CRYPT_SHA256Finish(sha, digest1);
}
FXSYS_memcpy(digest1 + 32, digest, 16);
pEncryptDict->SetAtString(bOwner ? FX_BSTRC("O") : FX_BSTRC("U"),
CFX_ByteString(digest1, 48));
if (m_Revision >= 6) {
Revision6_Hash(password, size, digest + 8,
(bOwner ? (const uint8_t*)ukey : NULL), digest1);
} else {
CRYPT_SHA256Start(sha);
CRYPT_SHA256Update(sha, password, size);
CRYPT_SHA256Update(sha, digest + 8, 8);
if (bOwner) {
CRYPT_SHA256Update(sha, ukey, ukey.GetLength());
}
CRYPT_SHA256Finish(sha, digest1);
}
uint8_t* aes = FX_Alloc(uint8_t, 2048);
CRYPT_AESSetKey(aes, 16, digest1, 32, TRUE);
uint8_t iv[16];
FXSYS_memset(iv, 0, 16);
CRYPT_AESSetIV(aes, iv);
CRYPT_AESEncrypt(aes, digest1, key, 32);
FX_Free(aes);
pEncryptDict->SetAtString(bOwner ? FX_BSTRC("OE") : FX_BSTRC("UE"),
CFX_ByteString(digest1, 32));
}
FX_BOOL CPDF_SecurityHandler::AES256_CheckPassword(const uint8_t* password,
uint32_t size,
FX_BOOL bOwner,
uint8_t* key) {
CFX_ByteString okey =
m_pEncryptDict ? m_pEncryptDict->GetStringBy("O") : CFX_ByteString();
if (okey.GetLength() < 48) {
return FALSE;
}
CFX_ByteString ukey =
m_pEncryptDict ? m_pEncryptDict->GetStringBy("U") : CFX_ByteString();
if (ukey.GetLength() < 48) {
return FALSE;
}
const uint8_t* pkey = (bOwner ? okey : ukey).raw_str();
uint8_t sha[128];
uint8_t digest[32];
if (m_Revision >= 6) {
Revision6_Hash(password, size, (const uint8_t*)pkey + 32,
bOwner ? ukey.raw_str() : nullptr, digest);
} else {
CRYPT_SHA256Start(sha);
CRYPT_SHA256Update(sha, password, size);
CRYPT_SHA256Update(sha, pkey + 32, 8);
if (bOwner) {
CRYPT_SHA256Update(sha, ukey.raw_str(), 48);
}
CRYPT_SHA256Finish(sha, digest);
}
if (FXSYS_memcmp(digest, pkey, 32) != 0) {
return FALSE;
}
if (!key) {
return TRUE;
}
if (m_Revision >= 6) {
Revision6_Hash(password, size, (const uint8_t*)pkey + 40,
bOwner ? ukey.raw_str() : nullptr, digest);
} else {
CRYPT_SHA256Start(sha);
CRYPT_SHA256Update(sha, password, size);
CRYPT_SHA256Update(sha, pkey + 40, 8);
if (bOwner) {
CRYPT_SHA256Update(sha, ukey.raw_str(), 48);
}
CRYPT_SHA256Finish(sha, digest);
}
CFX_ByteString ekey = m_pEncryptDict
? m_pEncryptDict->GetStringBy(bOwner ? "OE" : "UE")
: CFX_ByteString();
if (ekey.GetLength() < 32) {
return FALSE;
}
uint8_t* aes = FX_Alloc(uint8_t, 2048);
CRYPT_AESSetKey(aes, 16, digest, 32, FALSE);
uint8_t iv[16];
FXSYS_memset(iv, 0, 16);
CRYPT_AESSetIV(aes, iv);
CRYPT_AESDecrypt(aes, key, ekey.raw_str(), 32);
CRYPT_AESSetKey(aes, 16, key, 32, FALSE);
CRYPT_AESSetIV(aes, iv);
CFX_ByteString perms = m_pEncryptDict->GetStringBy("Perms");
if (perms.IsEmpty()) {
return FALSE;
}
uint8_t perms_buf[16];
FXSYS_memset(perms_buf, 0, sizeof(perms_buf));
uint32_t copy_len = sizeof(perms_buf);
if (copy_len > (uint32_t)perms.GetLength()) {
copy_len = perms.GetLength();
}
FXSYS_memcpy(perms_buf, perms.raw_str(), copy_len);
uint8_t buf[16];
CRYPT_AESDecrypt(aes, buf, perms_buf, 16);
FX_Free(aes);
if (buf[9] != 'a' || buf[10] != 'd' || buf[11] != 'b') {
return FALSE;
}
if (FXDWORD_GET_LSBFIRST(buf) != m_Permissions) {
return FALSE;
}
if ((buf[8] == 'T' && !IsMetadataEncrypted()) ||
(buf[8] == 'F' && IsMetadataEncrypted())) {
return FALSE;
}
return TRUE;
}
void Revision6_Hash(const uint8_t* password,
uint32_t size,
const uint8_t* salt,
const uint8_t* vector,
uint8_t* hash) {
int iBlockSize = 32;
uint8_t sha[128];
CRYPT_SHA256Start(sha);
CRYPT_SHA256Update(sha, password, size);
CRYPT_SHA256Update(sha, salt, 8);
if (vector) {
CRYPT_SHA256Update(sha, vector, 48);
}
uint8_t digest[32];
CRYPT_SHA256Finish(sha, digest);
CFX_ByteTextBuf buf;
uint8_t* input = digest;
uint8_t* key = input;
uint8_t* iv = input + 16;
uint8_t* E = buf.GetBuffer();
int iBufLen = buf.GetLength();
CFX_ByteTextBuf interDigest;
int i = 0;
uint8_t* aes = FX_Alloc(uint8_t, 2048);
while (i < 64 || i < E[iBufLen - 1] + 32) {
int iRoundSize = size + iBlockSize;
if (vector) {
iRoundSize += 48;
}
iBufLen = iRoundSize * 64;
buf.EstimateSize(iBufLen);
E = buf.GetBuffer();
CFX_ByteTextBuf content;
for (int j = 0; j < 64; ++j) {
content.AppendBlock(password, size);
content.AppendBlock(input, iBlockSize);
if (vector) {
content.AppendBlock(vector, 48);
}
}
CRYPT_AESSetKey(aes, 16, key, 16, TRUE);
CRYPT_AESSetIV(aes, iv);
CRYPT_AESEncrypt(aes, E, content.GetBuffer(), iBufLen);
int iHash = 0;
switch (BigOrder64BitsMod3(E)) {
case 0:
iHash = 0;
iBlockSize = 32;
break;
case 1:
iHash = 1;
iBlockSize = 48;
break;
default:
iHash = 2;
iBlockSize = 64;
break;
}
interDigest.EstimateSize(iBlockSize);
input = interDigest.GetBuffer();
if (iHash == 0) {
CRYPT_SHA256Generate(E, iBufLen, input);
} else if (iHash == 1) {
CRYPT_SHA384Generate(E, iBufLen, input);
} else if (iHash == 2) {
CRYPT_SHA512Generate(E, iBufLen, input);
}
key = input;
iv = input + 16;
++i;
}
FX_Free(aes);
if (hash) {
FXSYS_memcpy(hash, input, 32);
}
}
FX_BOOL CPDF_StandardSecurityHandler::AES256_CheckPassword(FX_LPCBYTE password, FX_DWORD size,
FX_BOOL bOwner, FX_LPBYTE key)
{
CFX_ByteString okey = m_pEncryptDict ? m_pEncryptDict->GetString(FX_BSTRC("O")) : CFX_ByteString();
if (okey.GetLength() < 48) {
return FALSE;
}
CFX_ByteString ukey = m_pEncryptDict ? m_pEncryptDict->GetString(FX_BSTRC("U")) : CFX_ByteString();
if (ukey.GetLength() < 48) {
return FALSE;
}
FX_LPCBYTE pkey = bOwner ? (FX_LPCBYTE)okey : (FX_LPCBYTE)ukey;
FX_BYTE sha[128];
FX_BYTE digest[32];
if (m_Revision >= 6) {
Revision6_Hash(password, size, (FX_LPCBYTE)pkey + 32, (bOwner ? (FX_LPCBYTE)ukey : NULL), digest);
} else {
CRYPT_SHA256Start(sha);
CRYPT_SHA256Update(sha, password, size);
CRYPT_SHA256Update(sha, pkey + 32, 8);
if (bOwner) {
CRYPT_SHA256Update(sha, ukey, 48);
}
CRYPT_SHA256Finish(sha, digest);
}
if (FXSYS_memcmp32(digest, pkey, 32) != 0) {
return FALSE;
}
if (key == NULL) {
return TRUE;
}
if (m_Revision >= 6) {
Revision6_Hash(password, size, (FX_LPCBYTE)pkey + 40, (bOwner ? (FX_LPCBYTE)ukey : NULL), digest);
} else {
CRYPT_SHA256Start(sha);
CRYPT_SHA256Update(sha, password, size);
CRYPT_SHA256Update(sha, pkey + 40, 8);
if (bOwner) {
CRYPT_SHA256Update(sha, ukey, 48);
}
CRYPT_SHA256Finish(sha, digest);
}
CFX_ByteString ekey = m_pEncryptDict ? m_pEncryptDict->GetString(bOwner ? FX_BSTRC("OE") : FX_BSTRC("UE")) : CFX_ByteString();
if (ekey.GetLength() < 32) {
return FALSE;
}
FX_BYTE* aes = FX_Alloc(FX_BYTE, 2048);
CRYPT_AESSetKey(aes, 16, digest, 32, FALSE);
FX_BYTE iv[16];
FXSYS_memset32(iv, 0, 16);
CRYPT_AESSetIV(aes, iv);
CRYPT_AESDecrypt(aes, key, ekey, 32);
CRYPT_AESSetKey(aes, 16, key, 32, FALSE);
CRYPT_AESSetIV(aes, iv);
CFX_ByteString perms = m_pEncryptDict->GetString(FX_BSTRC("Perms"));
if (perms.IsEmpty()) {
return FALSE;
}
FX_BYTE perms_buf[16];
FXSYS_memset32(perms_buf, 0, sizeof(perms_buf));
FX_DWORD copy_len = sizeof(perms_buf);
if (copy_len > (FX_DWORD)perms.GetLength()) {
copy_len = perms.GetLength();
}
FXSYS_memcpy32(perms_buf, (FX_LPCBYTE)perms, copy_len);
FX_BYTE buf[16];
CRYPT_AESDecrypt(aes, buf, perms_buf, 16);
FX_Free(aes);
if (buf[9] != 'a' || buf[10] != 'd' || buf[11] != 'b') {
return FALSE;
}
if (FXDWORD_GET_LSBFIRST(buf) != m_Permissions) {
return FALSE;
}
if ((buf[8] == 'T' && !IsMetadataEncrypted()) || (buf[8] == 'F' && IsMetadataEncrypted())) {
return FALSE;
}
return TRUE;
}
