uint64_t insert_hash(uint16_t chunk_length, uint16_t *packed_upto,
unsigned char *payload,
uint16_t last_marker,
bool *dedup_flag,
time_t current_timestamp) {
uint16_t local_packed_upto = (*packed_upto);
uint32_t left = 0, right = 0;
hashlittle2((void*)(payload + last_marker), chunk_length, &right, &left);
printlog(logfile, system_loglevel, LOG_DEBUG, "Hashing chunk"
" from %d to %d\n", local_packed_upto, local_packed_upto +
chunk_length);
uint64_t hash_value = right + (((uint64_t)left)<<32);
uint16_t advance_by = 0;
if(regular_cache.find(hash_value) != regular_cache.end()) {
printlog(logfile, system_loglevel, LOG_DEBUG, "Putting regular hash %llx for chunk length %d\n", hash_value, chunk_length);
advance_by += pack_hash_value(new_packet + local_packed_upto, left, right);
*dedup_flag = true;
printlog(logfile, system_loglevel, LOG_DEBUG, "Normal hit: "
"%.llx\n", (unsigned long long)hash_value);
} else if (prefetch_cache.find(hash_value) !=
prefetch_cache.end()) {
printlog(logfile, system_loglevel, LOG_DEBUG, "Putting feedback hash %llx for chunk length %d\n", hash_value, chunk_length);
advance_by += pack_hash_value(new_packet + local_packed_upto, left, right);
*dedup_flag = true;
printlog(logfile, system_loglevel, LOG_DEBUG, "Advert hit\
%.llx\n", (unsigned long long)hash_value);
prefetch_cache.erase(hash_value);
} else {
void Resolver::resolve(const size_t &length, const uint &depth) {
for (char letter : alphabet) {
buffer[length] = letter;
for (const char *extension: extensions) {
strcpy(&buffer[length + 1], extension);
hashHighDw = 0;
hashLowDw = 0;
hashlittle2(buffer, length + 5, &hashHighDw, &hashLowDw);
if (unknownHashes.contain(((uint64_t) hashHighDw << 32) | hashLowDw)) {
adder(std::string(buffer, 0, length + 5));
}
}
if (depth > 1) {
buffer[length + 1] = 0;
resolve(length + 1, depth - 1);
}
}
buffer[length] = 0;
}
//------------------------------------------------------------------------
//------------------------------------------------------------------------
void insert(const void *rhs)
{
h1_ = h2_ = 0;
hashlittle2(rhs, size_of_size_t, &h1_, &h2_);
bit_vector1_.set( h1_ & bitwiseAndOp );
bit_vector2_.set( h2_ & bitwiseAndOp );
}
//------------------------------------------------------------------------
//------------------------------------------------------------------------
bool exists(const void *rhs)
{
h1_ = h2_ = 0;
hashlittle2(rhs, size_of_size_t, &h1_, &h2_);
return bit_vector1_.test( h1_ & bitwiseAndOp ) &&
bit_vector2_.test( h2_ & bitwiseAndOp );
}
T CuckooHashMap<T>::get(std::string key) {
// Hash to two buckets.
uint32_t h1, h2;
h1 = 0;
h2 = 0;
hashlittle2(key.c_str(), key.length(), &h1, &h2);
h1 = h1 % m_num_buckets;
h1 *= SLOTS_PER_BUCKET;
// Look at the first bucket.
for (unsigned int i = h1; i < h1 + SLOTS_PER_BUCKET; i++) {
HashEntry* hash_entry = m_table[i];
if (hash_entry != NULL) {
if (key == hash_entry->key)
return hash_entry->val;
}
}
h2 = h2 % m_num_buckets;
h2 *= SLOTS_PER_BUCKET;
// Look at the second bucket.
for (unsigned int i = h2; i < h2 + SLOTS_PER_BUCKET; i++) {
HashEntry* hash_entry = m_table[i];
if (hash_entry != NULL) {
if (key == hash_entry->key)
return hash_entry->val;
}
}
throw KeyNotFoundError(key.c_str());
}
uint64_t HashData( void* data, const size_t dataSize )
{
uint64_t hash;
*( ((uint32_t*)&hash) + 0 ) = 13;
*( ((uint32_t*)&hash) + 1 ) = 37;
hashlittle2( data, dataSize, (((uint32_t*)&hash)+0), (((uint32_t*)&hash)+1) );
return hash;
}
uint64_t compute_hash(unsigned char *payload, uint16_t chunked_upto,
uint16_t chunk_length) {
uint32_t left = 0, right = 0;
hashlittle2((void*)(payload + chunked_upto), chunk_length, &right, &left);
printlog(logfile, system_loglevel, LOG_DEBUG,
"Hashing chunk from %d to %d\n",
chunked_upto, chunked_upto + chunk_length);
return right + (((uint64_t)left)<<32);
}
inline std::pair<uint32_t, uint32_t> lookup3(InputIt first, InputIt last, const std::pair<uint32_t, uint32_t> &init = { 0, 0 })
{
auto pc = init.first;
auto pb = init.second;
hashlittle2(
&*first,
sizeof(typename std::iterator_traits<InputIt>::value_type) * (last - first),
&pc, &pb);
return std::make_pair(pc, pb);
}
inline std::pair<uint32_t, uint32_t> lookup3(const Container &container, const std::pair<uint32_t, uint32_t> &init = { 0, 0 })
{
auto pc = init.first;
auto pb = init.second;
hashlittle2(
&*std::begin(container),
sizeof(typename Container::value_type) * (std::end(container) - std::begin(container)),
&pc, &pb);
return std::make_pair(pc, pb);
}
inline std::pair<uint32_t, uint32_t> lookup3(std::ifstream &stream, uint32_t length, const std::pair<uint32_t, uint32_t> &init = { 0, 0 })
{
auto pc = init.first;
auto pb = init.second;
std::vector<char> buffer(length);
auto pos = stream.tellg();
stream.read(buffer.data(), length);
hashlittle2(buffer.data(), length, &pc, &pb);
stream.seekg(pos);
return std::make_pair(pc, pb);
}
// Also used in CascSearchFile
PCASC_ROOT_ENTRY FindFirstRootEntry(TCascStorage * hs, const char * szFileName, size_t * PtrIndex)
{
PCASC_ROOT_ENTRY pFoundEntry = NULL;
PCASC_ROOT_ENTRY pRootEntry;
ULONGLONG FileNameHash;
uint32_t dwHashHigh = 0;
uint32_t dwHashLow = 0;
size_t StartEntry = 0;
size_t MidlEntry = 0;
size_t EndEntry = hs->nRootEntries;
// Calculate the HASH value of the normalized file name
hashlittle2(szFileName, strlen(szFileName), &dwHashHigh, &dwHashLow);
FileNameHash = ((ULONGLONG)dwHashHigh << 0x20) | dwHashLow;
// Perform binary search
while(StartEntry < EndEntry)
{
// Calculate the middle of the interval
MidlEntry = StartEntry + ((EndEntry - StartEntry) / 2);
pRootEntry = hs->ppRootEntries[MidlEntry];
// Did we find it?
if(pRootEntry->FileNameHash == FileNameHash)
{
pFoundEntry = pRootEntry;
break;
}
// Move the interval to the left or right
(FileNameHash < pRootEntry->FileNameHash) ? EndEntry = MidlEntry : StartEntry = MidlEntry + 1;
}
// Move the pointer back to the first entry with that hash
if(pFoundEntry != NULL)
{
while(MidlEntry > 0 && hs->ppRootEntries[MidlEntry - 1]->FileNameHash == FileNameHash)
{
pFoundEntry = hs->ppRootEntries[MidlEntry - 1];
MidlEntry--;
}
}
// Return what we found
if(PtrIndex != NULL)
*PtrIndex = MidlEntry;
return pFoundEntry;
}
static PLISTFILE_MAP ListMap_InsertName(PLISTFILE_MAP pListMap, const char * szFileName, size_t nLength)
{
PLISTFILE_ENTRY pListEntry;
char szFileName2[MAX_PATH+1];
size_t cbToAllocate;
size_t cbEntrySize;
uint32_t dwHashHigh = 0;
uint32_t dwHashLow = 0;
// Make sure there is enough space in the list map
cbEntrySize = sizeof(LISTFILE_ENTRY) + nLength;
cbEntrySize = ALIGN_TO_SIZE(cbEntrySize, 8);
if((pListMap->cbBuffer + cbEntrySize) > pListMap->cbBufferMax)
{
cbToAllocate = sizeof(LISTFILE_MAP) + (pListMap->cbBufferMax * 3) / 2;
pListMap = (PLISTFILE_MAP)CASC_REALLOC(BYTE, pListMap, cbToAllocate);
if(pListMap == NULL)
return NULL;
pListMap->cbBufferMax = (pListMap->cbBufferMax * 3) / 2;
}
// Get the pointer to the first entry
pListEntry = (PLISTFILE_ENTRY)((LPBYTE)(pListMap + 1) + pListMap->cbBuffer);
// Get the name hash
NormalizeFileName_UpperBkSlash(szFileName, szFileName2);
hashlittle2(szFileName2, nLength, &dwHashHigh, &dwHashLow);
// Calculate the HASH value of the normalized file name
pListEntry->FileNameHash = ((ULONGLONG)dwHashHigh << 0x20) | dwHashLow;
pListEntry->cbEntrySize = (DWORD)cbEntrySize;
memcpy(pListEntry->szFileName, szFileName, nLength);
pListEntry->szFileName[nLength] = 0;
// Move the next entry
pListMap->cbBuffer += cbEntrySize;
pListMap->nEntries++;
return pListMap;
}
ULONGLONG HashStringJenkins(const char * szFileName)
{
char * szTemp;
char szLocFileName[0x108];
char chOneChar;
size_t nLength = 0;
unsigned int primary_hash = 1;
unsigned int secondary_hash = 2;
// This is required to produce defined results
assert(szFileName != NULL);
// Normalize the file name - convert to uppercase, and convert "/" to "\\".
if(szFileName != NULL)
{
szTemp = szLocFileName;
while(*szFileName != 0)
{
chOneChar = (char)tolower(*szFileName++);
if(chOneChar == '/')
chOneChar = '\\';
*szTemp++ = chOneChar;
}
nLength = szTemp - szLocFileName;
*szTemp = 0;
}
// Thanks Quantam for finding out what the algorithm is.
// I am really getting old for reversing large chunks of assembly
// that does hashing :-)
hashlittle2(szLocFileName, nLength, &secondary_hash, &primary_hash);
// Combine those 2 together
return (ULONGLONG)primary_hash * (ULONGLONG)0x100000000ULL + (ULONGLONG)secondary_hash;
}
// Also used in CascSearchFile
PCASC_ROOT_ENTRY FindRootEntry(TCascStorage * hs, const char * szFileName, DWORD * PtrTableIndex)
{
PCASC_ROOT_ENTRY pRootEntry;
ULONGLONG FileNameHash;
DWORD TableIndex;
uint32_t dwHashHigh = 0;
uint32_t dwHashLow = 0;
// Calculate the HASH value of the normalized file name
hashlittle2(szFileName, strlen(szFileName), &dwHashHigh, &dwHashLow);
FileNameHash = ((ULONGLONG)dwHashHigh << 0x20) | dwHashLow;
// Get the first table index
TableIndex = (DWORD)(FileNameHash & (hs->RootTable.TableSize - 1));
assert(hs->RootTable.ItemCount < hs->RootTable.TableSize);
// Search the proper entry
for(;;)
{
// Does the has match?
pRootEntry = hs->RootTable.TablePtr + TableIndex;
if(pRootEntry->FileNameHash == FileNameHash)
{
if(PtrTableIndex != NULL)
PtrTableIndex[0] = TableIndex;
return pRootEntry;
}
// If the entry is free, the file is not there
if(pRootEntry->FileNameHash == 0 && pRootEntry->SumValue == 0)
return NULL;
// Move to the next entry
TableIndex = (DWORD)((TableIndex + 1) & (hs->RootTable.TableSize - 1));
}
}
void CuckooHashMap<T>::put(std::string key, T val) {
int num_iters = 0;
std::string curr_key = key;
T curr_val = val;
while (num_iters < MAX_ITERS) {
num_iters++;
uint32_t h1, h2;
h1 = 0;
h2 = 0;
hashlittle2(curr_key.c_str(), curr_key.length(), &h1, &h2);
h1 = h1 % m_num_buckets;
h2 = h2 % m_num_buckets;
h1 *= SLOTS_PER_BUCKET;
h2 *= SLOTS_PER_BUCKET;
// Look at the first bucket.
for (unsigned int i = h1; i < h1 + SLOTS_PER_BUCKET; i++) {
HashEntry* hash_entry = m_table[i];
if (hash_entry == NULL) {
HashEntry* hash_entry = new HashEntry();
hash_entry->key = curr_key;
hash_entry->val = curr_val;
m_table[i] = hash_entry;
return;
} else if (hash_entry->key == curr_key) {
hash_entry->val = curr_val;
return;
}
}
// Look at the second bucket.
for (unsigned int i = h2; i < h2 + SLOTS_PER_BUCKET; i++) {
HashEntry* hash_entry = m_table[i];
if (hash_entry == NULL) {
HashEntry* hash_entry = new HashEntry();
hash_entry->key = curr_key;
hash_entry->val = curr_val;
m_table[i] = hash_entry;
return;
} else if (hash_entry->key == curr_key) {
hash_entry->val = curr_val;
return;
}
}
int index = rand() % (2 * SLOTS_PER_BUCKET);
HashEntry* temp_hash_entry;
if (0 <= index && index < SLOTS_PER_BUCKET)
temp_hash_entry = m_table[h1 + index];
else
temp_hash_entry = m_table[h2 + index - SLOTS_PER_BUCKET];
std::string temp_key = temp_hash_entry->key;
T temp_val = temp_hash_entry->val;
temp_hash_entry->key = curr_key;
temp_hash_entry->val = curr_val;
curr_key = temp_key;
curr_val = temp_val;
}
std::cout << "Abort" << std::endl;
}
