void leerficheroSP(Dispersion<StructPost>& disp, const string& nfichero, const bool& cabecera) { string aux_str; ifstream f; f.open(nfichero.c_str()); if (f.good()) { if (cabecera && !f.eof()) { getline(f,aux_str); } while (getline(f,aux_str)) { CodigoPost cp; aux_str=cp.fromCSV(aux_str); StructPost* existente=disp.buscar(hash_djb2(aux_str.c_str())); if (existente) { existente->codigos.insertarFin(cp); } else { StructPost sp={aux_str}; sp.codigos.insertarFin(cp); disp.insertar(hash_djb2(aux_str.c_str()),sp); } } } f.close(); }
int main() { init_rs232(); enable_rs232_interrupts(); enable_rs232(); fs_init(); fio_init(); register_romfs("romfs", &_sromfs); pwd_hash=hash_djb2((const uint8_t *)&"",-1); // init pwd to /romfs/ /* Create the queue used by the serial task. Messages for write to * the RS232. */ vSemaphoreCreateBinary(serial_tx_wait_sem); /* Add for serial input * Reference: www.freertos.org/a00116.html */ serial_rx_queue = xQueueCreate(1, sizeof(char)); /* Create a task to output text read from romfs. */ xTaskCreate(command_prompt, (signed portCHAR *) "CLI", 512 /* stack size */, NULL, tskIDLE_PRIORITY + 2, NULL); #if 0 /* Create a task to record system log. */ xTaskCreate(system_logger, (signed portCHAR *) "Logger", 1024 /* stack size */, NULL, tskIDLE_PRIORITY + 1, NULL); #endif /* Start running the tasks. */ vTaskStartScheduler(); return 0; }
bool ShmPc::getIndexByKey(const string& _key, uint32_t& _index) { uint32_t start = hash_djb2(_key) % BLOCK_COUNT; uint32_t index; ShmBlock* block; // hash 冲突算法 Quadratic probing (https://en.wikipedia.org/wiki/Quadratic_probing) for (uint32_t i = 0; i < BLOCK_COUNT; i++) { index = (start + i * i) % BLOCK_COUNT; block = (ShmBlock*)(this->m_start + index * sizeof(ShmBlock)); if (block->used != 1) { // not used block->used = 1; strncpy(block->key, _key.c_str(), BLOCK_KEY_LEN); block->value = 0; _index = index; return true; } if (strncmp(block->key, _key.c_str(), BLOCK_KEY_LEN) == 0) { // found _index = index; return true; } } // full return false; }
/* note that 3rd argument must evaluate to non-NULL - returns hash value * of string_to_lookup to add_data() to allow it to skip doing a hash */ struct hashnode * node_lookup(struct hashtable *h, const char *string_to_lookup, unsigned int *rhv) { struct hashnode *r = NULL; unsigned int hashval = hash_djb2(string_to_lookup); int idx = MOD(hashval, h->maxp); struct hashnode *chain; *rhv = hashval; if (idx < h->p) idx = MOD(hashval, (2*h->maxp)); chain = h->buckets[idx]->next; while (chain != h->sentinels[idx]) { /* checking the equivalence of hash value first prevents * expensive byte-by-byte strcmp(). Seems to improve performance. */ if (chain->value == hashval && !strcmp(chain->string, string_to_lookup)) { r = chain; break; } chain = chain->next; } return r; }
/** Force Insert **/ uint32_t fasthash_force_insert(fasthash_table *fh_table, char *key, void *entry) { if (!fh_table) return 0; uint32_t index = hash_djb2((unsigned char *)key) % fh_table->size; fh_table->entries[index] = entry; return index; }
/** Lookup **/ fasthash_node *fasthash_lookup(fasthash_table *fh_table, char *key) { if (!fh_table) return 0; uint32_t index = hash_djb2((unsigned char *)key) % fh_table->size; fasthash_node *node = fh_table->entries[index]; while (node && (strcmp(key, node->key))) { node = node->next; } return node; }
int register_fs(const char * mountpoint, fs_open_t callback, void * opaque) { int i; DBGOUT("register_fs(\"%s\", %p, %p)\r\n", mountpoint, callback, opaque); for (i = 0; i < MAX_FS; i++) { if (!fss[i].cb) { fss[i].hash = hash_djb2((const uint8_t *) mountpoint, -1); fss[i].cb = callback; fss[i].opaque = opaque; return 0; } } return -1; }
void list_fs(char * buf, const char * path) { int i; *buf = (char)0x00; if (path == NULL) { for (i = 0; i < MAX_FS; i++) { if (!fss[i].cb) { return; } strcat((char *)buf, (const char *)"\r\n"); strcat((char *)buf, fss[i].fs_name); } } if (strcmp((char *)path, (const char *)"/") == 0) { for (i = 0; i < MAX_FS; i++) { if (!fss[i].cb) { return; } strcat((char *)buf, (const char *)"\r\n"); strcat((char *)buf, fss[i].fs_name); } } const char * slash; uint32_t hash; while (path[0] == '/') path++; slash = strchr(path, '/'); if (!slash) return; hash = hash_djb2((const uint8_t *) path, slash - path); path = slash + 1; for (i = 0; i < MAX_FS; i++) { if (fss[i].hash == hash) fss[i].list_cb(fss[i].opaque, buf); } }
static int romfs_open(void * opaque, const char * path, int flags, int mode) { uint32_t h = hash_djb2((const uint8_t *) path, -1); const uint8_t * romfs = (const uint8_t *) opaque; const uint8_t * file; int r = -1; file = romfs_get_file_by_hash(romfs, h, NULL); if (file) { r = fio_open(romfs_read, NULL, romfs_seek, NULL, NULL); if (r > 0) { romfs_fds[r].file = file; romfs_fds[r].cursor = 0; fio_set_opaque(r, romfs_fds + r); } } return r; }
int main(int argc, char *argv[]) { //unsigned long hash; jhash_t hash; char *str; if (argc < 2) { printf("No string input\n"); exit(0); } str = argv[1]; printf("Input: %s\n", str); hash = hash_djb2(str); printf("djb2 Hash: %lu\n", hash); hash = hash_sdbm(str); printf("sdbm Hash: %lu\n", hash); hash = hash_lose(str); printf("lose Hash: %lu\n", hash); return 0; }
static int devfs_open(void * opaque, const char * path, int flags, int mode) { uint32_t h = hash_djb2((const uint8_t *) path, -1); // DBGOUT("devfs_open(%p, \"%s\", %i, %i)\r\n", opaque, path, flags, mode); switch (h) { case stdin_hash: if (flags & (O_WRONLY | O_RDWR)) return -1; return fio_open(stdin_read, NULL, NULL, NULL, NULL); break; case stdout_hash: if (flags & O_RDONLY) return -1; return fio_open(NULL, stdout_write, NULL, NULL, NULL); break; case stderr_hash: if (flags & O_RDONLY) return -1; return fio_open(NULL, stdout_write, NULL, NULL, NULL); break; } return -1; }
/* helper function to invoke the correct hash method */ static unsigned int gen_hash(const struct proxy* px, const char* key, unsigned long len) { unsigned int hash; switch (px->lbprm.algo & BE_LB_HASH_FUNC) { case BE_LB_HFCN_DJB2: hash = hash_djb2(key, len); break; case BE_LB_HFCN_WT6: hash = hash_wt6(key, len); break; case BE_LB_HFCN_CRC32: hash = hash_crc32(key, len); break; case BE_LB_HFCN_SDBM: /* this is the default hash function */ default: hash = hash_sdbm(key, len); break; } return hash; }
static int romfs_open(void * opaque, const char * path, int flags, int mode) { uint32_t h = hash_djb2((const uint8_t *) path, -1); const uint8_t * romfs = (const uint8_t *) opaque; const uint8_t * file; int r = -1; file = romfs_get_file_by_hash(romfs, h, NULL); if (file) { r = fio_open(romfs_read, NULL, romfs_seek, NULL, NULL); if (r > 0) { uint32_t size = get_unaligned(file - 8); const uint8_t *filestart = file; while(*filestart) ++filestart; ++filestart; size -= filestart - file; romfs_fds[r].file = filestart; romfs_fds[r].cursor = 0; romfs_fds[r].size = size; fio_set_opaque(r, romfs_fds + r); } } return r; }
uint32_t fasthash_insert_impl(fasthash_table *fh_table, char *key, void *entry) { if (!fh_table) return 0; uint32_t index = hash_djb2((unsigned char *)key) % fh_table->size; fasthash_node *node = fh_table->entries[index]; if (node) { /* If we aren't allowing collisions, return out of range */ if (fh_table->opts & FH_NO_COLLISIONS) { return fh_table->size + 1; } } else if (!(fh_table->opts & FH_NO_INDEXING)) { /* Build index node if we are indexing */ index_node *in = (index_node *)malloc(sizeof(index_node)); in->index = index; in->next = fh_table->index_list; fh_table->index_list = in; } node = fasthash_node_construct(key, entry, node); //printf("Node \"%s\" = %p\n", key, entry); fh_table->entries[index] = node; return index; /* We don't need to add an additional index */ }
int fs_open(const char * path, int flags, int mode) { const char * slash; uint32_t hash; int i; // DBGOUT("fs_open(\"%s\", %i, %i)\r\n", path, flags, mode); while (path[0] == '/') path++; slash = strchr(path, '/'); if (!slash) return -2; hash = hash_djb2((const uint8_t *) path, slash - path); path = slash + 1; for (i = 0; i < MAX_FS; i++) { if (fss[i].hash == hash) return fss[i].cb(fss[i].opaque, path, flags, mode); } return -2; }
int main (int argc, char *argv[]) { paper_rec_t DedupeRecord; dd_uint64_t Unique_CRID; /* Unique CR_ID = (C_ID << 16) | CR_ID */ long victim_index = 0, cache_size, window_size, bloom_filter_size; long i, j=0, temp_index; int Initial_Flag = 0, cache_algorithm; dd_uint8_t *sha1_value=NULL; int nLen = 0; long max_counter=0; HTItem *chunk_item, *item; long byte_len, temp, offset, ver, temp1; /* to read a trace file */ unsigned long hash1, hash2; /* Heap Data structure variables */ Cache_Memory Dataitem; std::vector<Cache_Memory>::iterator itr; unsigned long writeCounter = 0; unsigned long access_counter; long file_position; FILE *fp1, *fp; size_t keySize=0,iCnt; clock_t start = clock(); time_t begin,end; time(&begin); if (argc < 5) { /* 0 1 2 3 4 */ fprintf(stderr, "usage: %s <Cache Size> <Window Size> <Cache Algorithm (0, 1, 2)> <Trace File>\n", argv[0]); fprintf(stderr, " - Cache Size: Dedupe read cache size in terms of # of data chunk (e.g. 500 chunks = 4MB (500*8KB))\n"); fprintf(stderr, " - Window Size: Future sliding window size in terms of TIMES of cache size.\n"); fprintf(stderr, " - Cache Algorithm: 0 (Belady MIN), 1 (Belady MIN with a future window), 2 (Lookahead read cache)\n"); fprintf(stderr, " - Trace File: Trace file name with path\n"); exit(1); } cache_size = atol(argv[1]); assert(cache_size > 0); /* cache size must be positive */ window_size = atol(argv[2]); assert(window_size > 0); /* window size must be positive */ cache_algorithm = atoi(argv[3]); assert((cache_algorithm == 0)||(cache_algorithm == 1)||(cache_algorithm == 2)); /* there are only three selections */ bloom_filter_size = cache_size*2; //No. of Hash functions for BF is 2 bloom_filter = (long *)malloc(sizeof(long)*bloom_filter_size); ht_cache = AllocateHashTable(SHA1_VALUE_LENGTH, 1); heap = newMinHeap((u32)cache_size); if((fp1 = fopen(argv[4], "rb")) == NULL){ //for reading data one by one DEBUG_INFO("File open error....1\n"); exit (-1); } if((fp = fopen(argv[4], "rb")) == NULL){ //for searching its future reference distance DEBUG_INFO("File open error....2\n"); exit (-1); } long c=0, d=0; u32 itemIndex; keySize = sizeof(DedupeRecord.fp_bytes); DEBUG_INFO("Record Size is: %d\n",keySize); while (1) { fread(&DedupeRecord, sizeof(struct _paper_rec_t),1, fp1); /*if(DedupeRecord.fp_bytes[0] == 0) DedupeRecord.fp_bytes[0] = '0';*/ /*for(iCnt = 0;iCnt<sizeof(DedupeRecord.fp_bytes);++iCnt) printf("%c",DedupeRecord.fp_bytes[iCnt]);*/ //DEBUG_INFO("Reading chunks : %ld\n", c++); c++; if(c%1000 == 0){ printf("Reading Chunks: %ld\n",c); } if (c % 10000 == 0) { printf("Cache hit ratio: %.3f = %lu / %lu \n", (double) (Hit_Count * 100) / (double) totalAccess , Hit_Count, totalAccess); } if(feof(fp1)) break; file_position = ftell(fp1); /* initially fill the cache. During this initialization process, we do not count the cache hit ratio. */ if (Initial_Flag == 0) { // Temporally store this current access chunk with its future reference distance in the cache chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,DedupeRecord.fp_bytes)); //Update Bloom filter counters hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; if(chunk_item) { //Cache Hit itemIndex = (u32)chunk_item->data; DEBUG_INFO("Index its updating is %ld:\n",itemIndex); heapUpdate(heap,max_counter,itemIndex,&ht_cache); } else { heapInsert(heap,DedupeRecord.fp_bytes, max_counter,&ht_cache); //Sandeep - Insert into Heap and Heapify cache_counter++; } if(cache_counter == cache_size) { DEBUG_INFO("\n#### Cache Initialization complete~!!####\n"); Initial_Flag = 1; //Sandeep - Construct Heap and Heapify //fnBuildHeap(cache_heap); #ifdef DEBUG printf("Heap Size is: %d\n",cache_heap.size()); /*PrintHashTable(ht_cache,-1,2); fnPrintHeap(cache_heap);*/ #endif } } else { /* Once the cache is full of data initially, we start to measure the cache hit ratio from now. */ totalAccess++; if((totalAccess % 100) == 0) { DEBUG_INFO("[CHECK] Current Access Number: %ld\n", totalAccess); } Unique_CRID = (DedupeRecord.cmc_id << 16) | DedupeRecord.creg_id; chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,DedupeRecord.fp_bytes)); if(chunk_item) { //Cache Hit Hit_Count++; DEBUG_INFO("Cache Hit\n"); //Update Bloom filter counters hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; //DEBUG_INFO("### Returned hash values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; itemIndex = (ulong)chunk_item->data; DEBUG_INFO("Index its updating is %ld:\n",itemIndex); assert(itemIndex>=0 && itemIndex<=cache_size); heapUpdate(heap,max_counter,itemIndex,&ht_cache); //Sandeep - Update heap counter val for this chunk with max_counter //fnUpdateHeap(cache_heap, Read_Cache[(ulong)chunk_item->data],max_counter); } else { heapPopMin(heap,&sha1_value,&access_counter,&ht_cache); if(!sha1_value) ERROR("SHA1 Value in main is NULL\n"); /*for(iCnt = 0;iCnt<sizeof(DedupeRecord.fp_bytes);++iCnt) printf("%c",sha1_value[iCnt]);*/ //Update Bloom filter counters hash1 = hash_djb2(sha1_value,sizeof(sha1_value))%bloom_filter_size; hash2 = hash_sdbm(sha1_value,sizeof(sha1_value))%bloom_filter_size; //DEBUG_INFO("### In Main before decrement %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); //Decrement BF counters bloom_filter[hash1]--; bloom_filter[hash2]--; free(sha1_value); //GP - Increment the BF counters for this chunk hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; //DEBUG_INFO("### Returned hash values are in main cache_miss %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; heapInsert(heap,DedupeRecord.fp_bytes,max_counter,&ht_cache); if(cache_algorithm == LOOKAHEAD){ /* Check if any other chunks in the current CR will appear within the future window. If we found one, we add such chunk(s) in the cache. */ Check_Unique_CRID(fp, Unique_CRID, file_position, 0, cache_size, window_size*cache_size, bloom_filter_size); } } } //else } //while printf("\n###################################################################\n"); printf("Cache hit ratio: %.3f = %lu / %lu \n", (double) (Hit_Count * 100) / (double) totalAccess , Hit_Count, totalAccess); printf("Cache size: %ld, window size: %ld\n", cache_size, window_size*cache_size); printf("Dedupe trace: %s\n", argv[4]); printf("###################################################################\n"); fclose(fp1); fclose(fp); FreeHashTable(ht_cache); deleteMinHeap(heap); time(&end); printf("###################################################################\n"); printf("Total time taken is %f \n",((double)clock()-start)/CLOCKS_PER_SEC); printf("###################################################################\n"); return 0; }
void Check_Unique_CRID(FILE *pFile, dd_uint64_t Current_Unique_CRID, long position, long distance, long cache_size, long window_size, long bloom_filter_size) { long i, victim_index; dd_uint64_t Temp_Unique_CRID; paper_rec_t Temp_DedupeTrace; long window_counter=0; long max_counter=0, temp_index=0; dd_uint8_t *sha1_value=NULL; HTItem *chunk_item, *item; unsigned long hash1, hash2; unsigned long access_counter; size_t keySize = sizeof(Temp_DedupeTrace.fp_bytes),iCnt; bool flag=true; std::list<paper_rec_t>::iterator itr; /* Heap Data structure variables */ /* Check the size of sliding window vector if its empty * the check unique crid has been called first time * initialize the vector with size of sliding window */ if(SlidingWindow.size() == 0) { fseek(pFile, position, SEEK_SET); while (1) { fread(&Temp_DedupeTrace, sizeof(struct _paper_rec_t),1, pFile); /*if(Temp_DedupeTrace.fp_bytes[0] == 0) Temp_DedupeTrace.fp_bytes[0] = '0';*/ if(feof(pFile)) break; SlidingWindow.push_back(Temp_DedupeTrace); if(SlidingWindow.size() >= window_size) { break; } } } else if(SlidingWindow.size() == window_size){ /* Remove one old record and insert the latest record */ SlidingWindow.pop_front(); fseek(pFile, position + window_size, SEEK_SET); fread(&Temp_DedupeTrace, sizeof(struct _paper_rec_t),1, pFile); SlidingWindow.push_back(Temp_DedupeTrace); } for(itr = SlidingWindow.begin();itr!=SlidingWindow.end();itr++){ Temp_Unique_CRID = ((*itr).cmc_id << 16) | (*itr).creg_id; /* if any data chunk in current CR appear within the future window */ if(Temp_Unique_CRID == Current_Unique_CRID) { DEBUG_INFO("[Found~!!] A chunk in current access CR will appeare within a future window.\n"); chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,(*itr).fp_bytes)); if(chunk_item) { //Cache Hit DEBUG_INFO("Cache Hit - New\n"); //Update Bloom filter counters hash1 = hash_djb2((*itr).fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm((*itr).fp_bytes,keySize)%bloom_filter_size; /* DEBUG_INFO("### Returned hash values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);*/ max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; temp_index = (u32)chunk_item->data; /* DEBUG_INFO("Index its updating is %ld:\n",temp_index);*/ heapUpdate(heap, max_counter, temp_index,&ht_cache); } else { //Sandeep - Choose victim from heap and strcpy Sha1 Value of the victim chunk to "sha1_value" variable /*sha1_value = fnDeleteItemHeap(cache_heap);*/ //GP - Increment the BF counters for this chunk hash1 = hash_djb2((*itr).fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm((*itr).fp_bytes,keySize)%bloom_filter_size; //DEBUG_INFO("### Returned hash values are before insert cache miss %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; flag = checkMin(heap,max_counter); /* If Min on heap is less than the new chuck then only replace with new * else just skip */ if(flag){ heapPopMin(heap,&sha1_value,&access_counter,&ht_cache); if(!sha1_value) ERROR("SHA1 Value in Check Unique CR is NULL\n"); /*for(iCnt = 0;iCnt<sizeof((*itr).fp_bytes);++iCnt) printf("%c",sha1_value[iCnt]);*/ //Update Bloom filter counters hash1 = hash_djb2(sha1_value,sizeof(sha1_value))%bloom_filter_size; hash2 = hash_sdbm(sha1_value,sizeof(sha1_value))%bloom_filter_size; //DEBUG_INFO("### Before Decrementing values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); //Decrement BF counters bloom_filter[hash1]--; bloom_filter[hash2]--; free(sha1_value); //Sandeep - Insert chunk into Heap with max_counter heapInsert(heap,(*itr).fp_bytes,max_counter,&ht_cache); } } } } }
operator size_t() const { return hash_djb2((const unsigned char *)c_str); }
/** * key_index - gives the index of a key * @key: the key * @size: size of the array of the hash table * Return: key index */ unsigned long int key_index(const unsigned char *key, unsigned long int size) { return (hash_djb2(key) % size); }
static unsigned int hashfromkey(void *ky) { struct key *k = ky; return hash_djb2(k->str); }