MD5_state MD5_from_string(XString msg)
{
MD5_state state;
MD5_init(&state);
MD5_block block;
const uint64_t msg_full_len = msg.size();
while (msg.size() >= 64)
{
for (int i = 0; i < 0x10; i++)
X[i] = msg[4 * i + 0] | msg[4 * i + 1] << 8 | msg[4 * i + 2] << 16 | msg[4 * i + 3] << 24;
MD5_do_block(&state, block);
msg = msg.xslice_t(64);
}
// now pad 1-512 bits + the 64-bit length - may be two blocks
uint8_t buf[0x40] = {};
really_memcpy(buf, reinterpret_cast<const uint8_t *>(msg.data()), msg.size());
buf[msg.size()] = 0x80; // a single one bit
if (64 - msg.size() > 8)
{
for (int i = 0; i < 8; i++)
buf[0x38 + i] = (msg_full_len * 8) >> (i * 8);
}
for (int i = 0; i < 0x10; i++)
X[i] = buf[4 * i + 0] | buf[4 * i + 1] << 8 | buf[4 * i + 2] << 16 | buf[4 * i + 3] << 24;
MD5_do_block(&state, block);
if (64 - msg.size() <= 8)
{
really_memset0(buf, 0x38);
for (int i = 0; i < 8; i++)
buf[0x38 + i] = (msg_full_len * 8) >> (i * 8);
for (int i = 0; i < 0x10; i++)
X[i] = buf[4 * i + 0] | buf[4 * i + 1] << 8 | buf[4 * i + 2] << 16 | buf[4 * i + 3] << 24;
MD5_do_block(&state, block);
}
/**
* Computes the MD5 on the contents of the provided file.
* @param filename The name of the file to process.
*/
void hash_file_md5(char* filename) {
int file = open(filename, O_RDONLY | O_SHLOCK);
if (file == -1) {
fprintf(stderr, "Unable to open file %s: %s\n", filename, strerror(errno));
return;
}
MD5_Context md4;
MD5_init(&md4);
int bytesRead;
unsigned char data[1024];
while ((bytesRead = read(file, data, 1024)) != 0) {
MD5_update(&md4, data, bytesRead);
}
unsigned char result[16];
MD5_final(&md4, result);
close(file);
printf(" ");
for (int i = 0; i < 16; ++i) {
printf("%02x", result[i]);
}
printf(" %s\n", filename);
}
int MD5_testRun(void)
{
int i;
MD5_Context context;
MD5_init(&context);
hash_begin(&context.h);
hash_update(&context.h, "abc", 3);
ASSERT(memcmp(hash_final(&context.h), "\x90\x01\x50\x98\x3C\xD2\x4F\xB0\xD6\x96\x3F\x7D\x28\xE1\x7F\x72", 16) == 0);
hash_begin(&context.h);
hash_update(&context.h, "aaa", 3);
ASSERT(memcmp(hash_final(&context.h), "\x47\xBC\xE5\xC7\x4F\x58\x9F\x48\x67\xDB\xD5\x7E\x9C\xA9\xF8\x08", 16) == 0);
hash_begin(&context.h);
hash_update(&context.h, "abcdefghijklmnopqrstuvwxyz", 26);
ASSERT(memcmp(hash_final(&context.h), "\xC3\xFC\xD3\xD7\x61\x92\xE4\x00\x7D\xFB\x49\x6C\xCA\x67\xE1\x3B", 16) == 0);
hash_begin(&context.h);
hash_update(&context.h, "0123456789", 10);
ASSERT(memcmp(hash_final(&context.h), "\x78\x1E\x5E\x24\x5D\x69\xB5\x66\x97\x9B\x86\xE2\x8D\x23\xF2\xC7", 16) == 0);
hash_begin(&context.h);
for (i = 0; i < 1000; i++)
hash_update(&context.h, "a", 1);
ASSERT(memcmp(hash_final(&context.h), "\xCA\xBE\x45\xDC\xC9\xAE\x5B\x66\xBA\x86\x60\x0C\xCA\x6B\x8B\xA8", 16) == 0);
hash_begin(&context.h);
for (i = 0; i < 1000000; i++)
hash_update(&context.h, "a", 1);
ASSERT(memcmp(hash_final(&context.h), "\x77\x07\xd6\xae\x4e\x02\x7c\x70\xee\xa2\xa9\x35\xc2\x29\x6f\x21", 16) == 0);
return 0;
}
void MD5_buffer(const char *buffer, const unsigned int buf_len, void *signature)
{
md5_t md5;
/* initialize the computation context */
MD5_init(&md5);
/* process whole buffer but last buf_len % MD5_BLOCK bytes */
MD5_process(&md5, buffer, buf_len);
/* put result in desired memory area */
MD5_finish(&md5, signature);
}
std::string lc_md5(const char * str, size_t len)
{
int32_t i;
uint8_t PADDING[64] = {
0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
uint32_t context_state[4]; /* state (ABCD) */
uint32_t context_count[2]; /* number of bits, modulo 2^64 (lsb first) */
uint8_t context_buffer[64]; /* input buffer */
uint8_t hash[16];
MD5_init();
MD5_update(str, len);
MD5_fini(hash);
std::string ret;
for ( i = 0 ; i < 16 ; i++ )
{
std::string rettmp = lc_itoa16((uint8_t)hash[i]);
if (rettmp.size() == 1)
{
ret += "0";
ret += rettmp;
}
else
{
ret += rettmp;
}
}
return ret;
}
int ProcessAsyncRequest(JobDetails* job) {
/* Variables to hold our options. */
char doCRC32 = 0;
char doMD5 = 0;
char doSHA1 = 0;
char doED2k = 0;
/* Standard variables (same between platforms) */
CRC32_Context crc32;
MD4_Context ed2k;
MD5_Context md5;
SHA1_Context sha1;
Block* blocks = job->blocks;
uint8_t block = 0;
uint32_t ed2kBlockIdx = 0;
uint32_t ed2kBlocks = 0;
unsigned char* ed2kHashes = NULL;
uint32_t ed2kHashLength = 0;
uint8_t ed2kLoopIdx = 0;
uint64_t position = 0;
uint32_t progressLoopCount = 0;
uint64_t totalBytesRead = 0;
BOOL result = FALSE;
/* Set our options */
doCRC32 = job->request->options & OPTION_CRC32;
doMD5 = job->request->options & OPTION_MD5;
doSHA1 = job->request->options & OPTION_SHA1;
doED2k = job->request->options & OPTION_ED2K;
if (doED2k) {
/* Determine the number of blocks needed for calculating the file's
* ED2k's hash. If the file isn't an even multiple of BLOCKSIZE then we
* add one more block. */
ed2kBlocks = (uint32_t)((uint64_t)job->size / BLOCKSIZE);
if (job->size % BLOCKSIZE > 0) {
++ed2kBlocks;
}
/* Only allocate an array if we have more than one block to hash.
* Files that are smaller in size than BLOCKSIZE simply use the normal
* computed MD4 hash. */
if (ed2kBlocks > 1) {
ed2kHashLength = ed2kBlocks * 16;
ed2kHashes = (unsigned char*)HeapAlloc(
job->heap,
HEAP_ZERO_MEMORY,
ed2kHashLength);
if (ed2kHashes == NULL) {
return -6;
}
}
}
if (doCRC32) { CRC32_init(&crc32); }
if (doMD5) { MD5_init(&md5); }
if (doSHA1) { SHA1_init(&sha1); }
/* Issue our initial read requests. */
for (block = 0; block < MAX_REQUESTS && position <= job->size; ++block) {
blocks[block].overlapped.Offset = LOPOS(position);
blocks[block].overlapped.OffsetHigh = HIPOS(position);
result = ReadFile(
job->file,
blocks[block].data,
BUFFERSIZE,
NULL,
&blocks[block].overlapped);
if (!result && GetLastError() != ERROR_IO_PENDING) {
if (ed2kHashes) {
HeapFree(job->heap, 0, ed2kHashes);
}
return -8;
}
position += BUFFERSIZE;
}
block = 0;
while (TRUE) {
uint32_t bytesRead = 0;
int getLastError = 0;
/* Use the mask to determine which block we'll process next. */
block = block % MAX_REQUESTS;
/* Get the results of the asynchronous read. */
result = GetOverlappedResult(
job->file, &blocks[block].overlapped, &bytesRead, TRUE);
getLastError = GetLastError();
/* Check to see if we reached the end of the file. */
if (!result && getLastError == ERROR_HANDLE_EOF) {
break;
}
/* This scenario should never happen, but check for it anyway. */
if (!result && getLastError == ERROR_IO_INCOMPLETE) {
continue;
}
/* Any other error is a failure, so bail out. */
if (!result) {
if (ed2kHashes) {
HeapFree(job->heap, 0, ed2kHashes);
}
return -8;
}
/* Update the total bytes read and inform the callback of our progress
* if it's time. Don't forget to bail out if they request it. */
totalBytesRead += bytesRead;
if (job->callback && progressLoopCount % 10 == 0) {
int32_t result = job->callback(job->request->tag, totalBytesRead);
if (result != 0) {
if (ed2kHashes) {
HeapFree(job->heap, 0, ed2kHashes);
}
return -9;
}
}
++progressLoopCount;
/* Update the hashes with the file data. */
if (doED2k) {
/* If we've looped 10 times, finish the current MD4 hash, update
* the block counter and set the hash to be initialized again.
* Also, if the BUFFERSIZE is ever changed, the loop index will
* need to be adjusted as well. */
if (ed2kLoopIdx == 10) {
MD4_final(&ed2k, &ed2kHashes[ed2kBlockIdx * 16]);
++ed2kBlockIdx;
ed2kLoopIdx = 0;
}
/* If this is the first loop (or we've just finished a hash)
* initialize the MD4 hash for the next block. */
if (ed2kLoopIdx == 0) {
MD4_init(&ed2k);
}
++ed2kLoopIdx;
MD4_update(&ed2k, blocks[block].data, bytesRead);
}
if (doCRC32) { CRC32_update(&crc32, blocks[block].data, bytesRead); }
if (doMD5) { MD5_update(&md5, blocks[block].data, bytesRead); }
if (doSHA1) { SHA1_update(&sha1, blocks[block].data, bytesRead); }
/* Update our position in the file and issue a new request if there's
* still more data to read. */
blocks[block].overlapped.Offset = LOPOS(position);
blocks[block].overlapped.OffsetHigh = HIPOS(position);
result = ReadFile(
job->file,
blocks[block].data,
BUFFERSIZE,
NULL,
&blocks[block].overlapped);
if (!result && GetLastError() != ERROR_IO_PENDING) {
if (ed2kHashes) {
HeapFree(job->heap, 0, ed2kHashes);
}
return -8;
}
/* Update our position in the file and the next block to read. */
position += BUFFERSIZE;
++block;
}
if (job->callback) {
job->callback(job->request->tag, totalBytesRead);
}
/* Finalize all of the hashes that were selected and store the results in
* the request result buffer. The order of the hashes are:
* 0 - 15: ED2k
* 16 - 19: CRC32
* 20 - 35: MD5
* 36 - 55: SHA1 */
if (doED2k) {
/* If we just had one block to process directly store the result of the
* block in the result buffer. */
if (ed2kBlocks == 1) {
MD4_final(&ed2k, &job->request->result[0]);
} else {
/* Check to see if we were in the middle of a loop and finalize the
* final pending block if we were. */
if (ed2kLoopIdx > 0) {
MD4_final(&ed2k, &ed2kHashes[ed2kBlockIdx * 16]);
}
/* Calculate the MD4 hash of the concatenated hashes from each ED2k
* block. The resulting hash is the final ED2k hash. */
MD4_init(&ed2k);
MD4_update(&ed2k, ed2kHashes, ed2kHashLength);
MD4_final(&ed2k, &job->request->result[0]);
/* Don't forget to free our ED2k buffer. */
HeapFree(GetProcessHeap(), 0, ed2kHashes);
}
}
if (doCRC32) { CRC32_final(&crc32, &job->request->result[16]); }
if (doMD5) { MD5_final(&md5, &job->request->result[20]); }
if (doSHA1) { SHA1_final(&sha1, &job->request->result[36]); }
return 0;
}
/**
* Accepts a HashRequest structure and attempts to calculate the requested hash
* of the provided file using synchronous IO.
*/
int HashFileWithSyncIO(HashRequest* request, HashProgressCallback* callback) {
/* Variables to hold our options. */
char doCRC32 = 0;
char doMD5 = 0;
char doSHA1 = 0;
char doED2k = 0;
/* Standard variables (same between platforms) */
CRC32_Context crc32;
MD4_Context ed2k;
MD5_Context md5;
SHA1_Context sha1;
uint32_t bytesRead = 0;
uint32_t ed2kBlockIdx = 0;
uint32_t ed2kBlocks = 0;
uint32_t ed2kHashLength = 0;
uint8_t ed2kLoopIdx = 0;
uint32_t progressLoopCount = 0;
uint64_t totalBytesRead = 0;
unsigned char* ed2kHashes = NULL;
unsigned char* fileData = NULL;
/* Platform specific variables */
HANDLE file = NULL;
WIN32_FILE_ATTRIBUTE_DATA fileAttributeData = { 0 };
BOOL readFailed = FALSE;
FILE_IO_PRIORITY_HINT_INFO priorityHint = { 0 };
/* Simple guard condition. If we have no request, we can't process. */
if (request == NULL) {
return -1;
}
/* clear the result buffer */
SecureZeroMemory(&request->result, 56);
/* Set our options */
doCRC32 = request->options & OPTION_CRC32;
doMD5 = request->options & OPTION_MD5;
doSHA1 = request->options & OPTION_SHA1;
doED2k = request->options & OPTION_ED2K;
/* If they didn't pass any valid options (or passed 0) for the options,
* return since we can't calculate a hash without knowing which hashing
* algorithm(s) to use. */
if (!doCRC32 && !doMD5 && !doSHA1 && !doED2k) {
return -2;
}
file = CreateFileW(
request->filename,
GENERIC_READ,
FILE_SHARE_READ,
NULL,
OPEN_EXISTING,
FILE_FLAG_SEQUENTIAL_SCAN,
NULL);
if (file == INVALID_HANDLE_VALUE) {
return -4;
}
/* Set our file priority hint to background. This is only a hint to Windows
* that our file IO should be secondary to other scheduled IO. If there is
* no other IO at the time, it will run at full speed. */
priorityHint.PriorityHint = IoPriorityHintVeryLow;
SetFileInformationByHandle(
file,
FileIoPriorityHintInfo,
&priorityHint,
sizeof(priorityHint));
/* Set up our hashes, also calculate the space needed for the ED2k hash if
* we were requested to produce an ED2k hash result. */
if (doED2k) {
BOOL result = 0;
uint64_t fileSize = 0;
result = GetFileAttributesExW(
request->filename,
GetFileExInfoStandard,
&fileAttributeData);
if (!result) {
CloseHandle(file);
return -5;
}
/* Convert the file size from two 32-bit DWORDS to a uint64_t. */
fileSize =
(((uint64_t)fileAttributeData.nFileSizeHigh) << 32) |
fileAttributeData.nFileSizeLow;
/* Determine the number of blocks needed for calculating the file's
* ED2k's hash. If the file isn't an even multiple of BLOCKSIZE then we
* add one more block. */
ed2kBlocks = (uint32_t)((uint64_t)fileSize / BLOCKSIZE);
if (fileSize % BLOCKSIZE > 0) {
++ed2kBlocks;
}
/* Only allocate an array if we have more than one block to hash.
* Files that are smaller in size than BLOCKSIZE simply use the normal
* computed MD4 hash. */
if (ed2kBlocks > 1) {
ed2kHashLength = ed2kBlocks * 16;
ed2kHashes = (unsigned char*)HeapAlloc(
GetProcessHeap(),
HEAP_ZERO_MEMORY,
ed2kHashLength);
if (ed2kHashes == NULL) {
CloseHandle(file);
return -6;
}
}
}
if (doCRC32) { CRC32_init(&crc32); }
if (doMD5) { MD5_init(&md5); }
if (doSHA1) { SHA1_init(&sha1); }
/* Allocate the file buffer. The BUFFERSIZE constant is a clean multiple of
* the BLOCKSIZE for ED2k hashing, which will make for easier looping. */
fileData = (unsigned char*)HeapAlloc(
GetProcessHeap(),
HEAP_ZERO_MEMORY,
BUFFERSIZE);
if (fileData == NULL) {
CloseHandle(file);
if (doED2k && ed2kHashes) {
HeapFree(GetProcessHeap(), 0, ed2kHashes);
}
return -7;
}
do {
if (!ReadFile(file, fileData, BUFFERSIZE, &bytesRead, NULL)) {
readFailed = TRUE;
break;
}
if (bytesRead == 0) {
break;
}
/* Update the total bytes read and inform the callback of our progress
* if it's time. Don't forget to bail out if they request it. */
totalBytesRead += bytesRead;
if (callback && progressLoopCount % 10 == 0) {
int32_t result = callback(request->tag, totalBytesRead);
if (result != 0) {
CloseHandle(file);
HeapFree(GetProcessHeap(), 0, fileData);
if (doED2k && ed2kHashes) {
HeapFree(GetProcessHeap(), 0, ed2kHashes);
}
return -9;
}
}
++progressLoopCount;
/* Update the hashes with the file data. */
if (doED2k) {
/* If we've looped 10 times, finish the current MD4 hash, update
* the block counter and set the hash to be initialized again.
* Also, if the BUFFERSIZE is ever changed, the loop index will
* need to be adjusted as well. */
if (ed2kLoopIdx == 10) {
MD4_final(&ed2k, &ed2kHashes[ed2kBlockIdx * 16]);
++ed2kBlockIdx;
ed2kLoopIdx = 0;
}
/* If this is the first loop (or we've just finished a hash)
* initialize the MD4 hash for the next block. */
if (ed2kLoopIdx == 0) {
MD4_init(&ed2k);
}
++ed2kLoopIdx;
MD4_update(&ed2k, fileData, bytesRead);
}
if (doCRC32) { CRC32_update(&crc32, fileData, bytesRead); }
if (doMD5) { MD5_update(&md5, fileData, bytesRead); }
if (doSHA1) { SHA1_update(&sha1, fileData, bytesRead); }
} while (bytesRead != 0);
/* Free our file buffer and close the file since we're done with it. */
HeapFree(GetProcessHeap(), 0, fileData);
CloseHandle(file);
/* If we have a callback, call them one last time informing them of our
* final progress (even if we failed). We'll ignore the request to cancel
* since we'll be done in no time anyway. */
if (callback) {
callback(request->tag, totalBytesRead);
}
/* If we had a read failure, free up the ed2k hash (if we have one) and
* inform our caller that we had a problem. */
if (readFailed) {
if (doED2k && ed2kHashes) {
HeapFree(GetProcessHeap(), 0, ed2kHashes);
}
return -8;
}
/* Finalize all of the hashes that were selected and store the results in
* the request result buffer. The order of the hashes are:
* 0 - 15: ED2k
* 16 - 19: CRC32
* 20 - 35: MD5
* 36 - 55: SHA1 */
if (doED2k) {
/* If we just had one block to process directly store the result of the
* block in the result buffer. */
if (ed2kBlocks == 1) {
MD4_final(&ed2k, &request->result[0]);
} else {
/* Check to see if we were in the middle of a loop and finalize the
* final pending block if we were. */
if (ed2kLoopIdx > 0) {
MD4_final(&ed2k, &ed2kHashes[ed2kBlockIdx * 16]);
}
/* Calculate the MD4 hash of the concatenated hashes from each ED2k
* block. The resulting hash is the final ED2k hash. */
MD4_init(&ed2k);
MD4_update(&ed2k, ed2kHashes, ed2kHashLength);
MD4_final(&ed2k, &request->result[0]);
/* Don't forget to free our ED2k buffer. */
HeapFree(GetProcessHeap(), 0, ed2kHashes);
}
}
if (doCRC32) { CRC32_final(&crc32, &request->result[16]); }
if (doMD5) { MD5_final(&md5, &request->result[20]); }
if (doSHA1) { SHA1_final(&sha1, &request->result[36]); }
return 0;
}