void leerficheroSP(Dispersion<StructPost>& disp, const string& nfichero, const bool& cabecera) {
string aux_str;
ifstream f;
f.open(nfichero.c_str());
if (f.good()) {
if (cabecera && !f.eof()) {
getline(f,aux_str);
}
while (getline(f,aux_str)) {
CodigoPost cp;
aux_str=cp.fromCSV(aux_str);
StructPost* existente=disp.buscar(hash_djb2(aux_str.c_str()));
if (existente) {
existente->codigos.insertarFin(cp);
} else {
StructPost sp={aux_str};
sp.codigos.insertarFin(cp);
disp.insertar(hash_djb2(aux_str.c_str()),sp);
}
}
}
f.close();
}
int main()
{
init_rs232();
enable_rs232_interrupts();
enable_rs232();
fs_init();
fio_init();
register_romfs("romfs", &_sromfs);
pwd_hash=hash_djb2((const uint8_t *)&"",-1); // init pwd to /romfs/
/* Create the queue used by the serial task. Messages for write to
* the RS232. */
vSemaphoreCreateBinary(serial_tx_wait_sem);
/* Add for serial input
* Reference: www.freertos.org/a00116.html */
serial_rx_queue = xQueueCreate(1, sizeof(char));
/* Create a task to output text read from romfs. */
xTaskCreate(command_prompt,
(signed portCHAR *) "CLI",
512 /* stack size */, NULL, tskIDLE_PRIORITY + 2, NULL);
#if 0
/* Create a task to record system log. */
xTaskCreate(system_logger,
(signed portCHAR *) "Logger",
1024 /* stack size */, NULL, tskIDLE_PRIORITY + 1, NULL);
#endif
/* Start running the tasks. */
vTaskStartScheduler();
return 0;
}
bool ShmPc::getIndexByKey(const string& _key, uint32_t& _index) {
uint32_t start = hash_djb2(_key) % BLOCK_COUNT;
uint32_t index;
ShmBlock* block;
// hash 冲突算法 Quadratic probing (https://en.wikipedia.org/wiki/Quadratic_probing)
for (uint32_t i = 0; i < BLOCK_COUNT; i++) {
index = (start + i * i) % BLOCK_COUNT;
block = (ShmBlock*)(this->m_start + index * sizeof(ShmBlock));
if (block->used != 1) {
// not used
block->used = 1;
strncpy(block->key, _key.c_str(), BLOCK_KEY_LEN);
block->value = 0;
_index = index;
return true;
}
if (strncmp(block->key, _key.c_str(), BLOCK_KEY_LEN) == 0) {
// found
_index = index;
return true;
}
}
// full
return false;
}
/* note that 3rd argument must evaluate to non-NULL - returns hash value
* of string_to_lookup to add_data() to allow it to skip doing a hash
*/
struct hashnode *
node_lookup(struct hashtable *h, const char *string_to_lookup, unsigned int *rhv)
{
struct hashnode *r = NULL;
unsigned int hashval = hash_djb2(string_to_lookup);
int idx = MOD(hashval, h->maxp);
struct hashnode *chain;
*rhv = hashval;
if (idx < h->p)
idx = MOD(hashval, (2*h->maxp));
chain = h->buckets[idx]->next;
while (chain != h->sentinels[idx])
{
/* checking the equivalence of hash value first prevents
* expensive byte-by-byte strcmp(). Seems to improve performance. */
if (chain->value == hashval &&
!strcmp(chain->string, string_to_lookup))
{
r = chain;
break;
}
chain = chain->next;
}
return r;
}
/** Force Insert **/
uint32_t fasthash_force_insert(fasthash_table *fh_table, char *key, void *entry) {
if (!fh_table) return 0;
uint32_t index = hash_djb2((unsigned char *)key) % fh_table->size;
fh_table->entries[index] = entry;
return index;
}
/** Lookup **/
fasthash_node *fasthash_lookup(fasthash_table *fh_table, char *key) {
if (!fh_table) return 0;
uint32_t index = hash_djb2((unsigned char *)key) % fh_table->size;
fasthash_node *node = fh_table->entries[index];
while (node && (strcmp(key, node->key))) {
node = node->next;
}
return node;
}
int register_fs(const char * mountpoint, fs_open_t callback, void * opaque) {
int i;
DBGOUT("register_fs(\"%s\", %p, %p)\r\n", mountpoint, callback, opaque);
for (i = 0; i < MAX_FS; i++) {
if (!fss[i].cb) {
fss[i].hash = hash_djb2((const uint8_t *) mountpoint, -1);
fss[i].cb = callback;
fss[i].opaque = opaque;
return 0;
}
}
return -1;
}
void list_fs(char * buf, const char * path) {
int i;
*buf = (char)0x00;
if (path == NULL) {
for (i = 0; i < MAX_FS; i++) {
if (!fss[i].cb) {
return;
}
strcat((char *)buf, (const char *)"\r\n");
strcat((char *)buf, fss[i].fs_name);
}
}
if (strcmp((char *)path, (const char *)"/") == 0) {
for (i = 0; i < MAX_FS; i++) {
if (!fss[i].cb) {
return;
}
strcat((char *)buf, (const char *)"\r\n");
strcat((char *)buf, fss[i].fs_name);
}
}
const char * slash;
uint32_t hash;
while (path[0] == '/')
path++;
slash = strchr(path, '/');
if (!slash)
return;
hash = hash_djb2((const uint8_t *) path, slash - path);
path = slash + 1;
for (i = 0; i < MAX_FS; i++) {
if (fss[i].hash == hash)
fss[i].list_cb(fss[i].opaque, buf);
}
}
static int romfs_open(void * opaque, const char * path, int flags, int mode) {
uint32_t h = hash_djb2((const uint8_t *) path, -1);
const uint8_t * romfs = (const uint8_t *) opaque;
const uint8_t * file;
int r = -1;
file = romfs_get_file_by_hash(romfs, h, NULL);
if (file) {
r = fio_open(romfs_read, NULL, romfs_seek, NULL, NULL);
if (r > 0) {
romfs_fds[r].file = file;
romfs_fds[r].cursor = 0;
fio_set_opaque(r, romfs_fds + r);
}
}
return r;
}
int main(int argc, char *argv[]) {
//unsigned long hash;
jhash_t hash;
char *str;
if (argc < 2) {
printf("No string input\n");
exit(0);
}
str = argv[1];
printf("Input: %s\n", str);
hash = hash_djb2(str);
printf("djb2 Hash: %lu\n", hash);
hash = hash_sdbm(str);
printf("sdbm Hash: %lu\n", hash);
hash = hash_lose(str);
printf("lose Hash: %lu\n", hash);
return 0;
}
static int devfs_open(void * opaque, const char * path, int flags, int mode) {
uint32_t h = hash_djb2((const uint8_t *) path, -1);
// DBGOUT("devfs_open(%p, \"%s\", %i, %i)\r\n", opaque, path, flags, mode);
switch (h) {
case stdin_hash:
if (flags & (O_WRONLY | O_RDWR))
return -1;
return fio_open(stdin_read, NULL, NULL, NULL, NULL);
break;
case stdout_hash:
if (flags & O_RDONLY)
return -1;
return fio_open(NULL, stdout_write, NULL, NULL, NULL);
break;
case stderr_hash:
if (flags & O_RDONLY)
return -1;
return fio_open(NULL, stdout_write, NULL, NULL, NULL);
break;
}
return -1;
}
/* helper function to invoke the correct hash method */
static unsigned int gen_hash(const struct proxy* px, const char* key, unsigned long len)
{
unsigned int hash;
switch (px->lbprm.algo & BE_LB_HASH_FUNC) {
case BE_LB_HFCN_DJB2:
hash = hash_djb2(key, len);
break;
case BE_LB_HFCN_WT6:
hash = hash_wt6(key, len);
break;
case BE_LB_HFCN_CRC32:
hash = hash_crc32(key, len);
break;
case BE_LB_HFCN_SDBM:
/* this is the default hash function */
default:
hash = hash_sdbm(key, len);
break;
}
return hash;
}
static int romfs_open(void * opaque, const char * path, int flags, int mode) {
uint32_t h = hash_djb2((const uint8_t *) path, -1);
const uint8_t * romfs = (const uint8_t *) opaque;
const uint8_t * file;
int r = -1;
file = romfs_get_file_by_hash(romfs, h, NULL);
if (file) {
r = fio_open(romfs_read, NULL, romfs_seek, NULL, NULL);
if (r > 0) {
uint32_t size = get_unaligned(file - 8);
const uint8_t *filestart = file;
while(*filestart) ++filestart;
++filestart;
size -= filestart - file;
romfs_fds[r].file = filestart;
romfs_fds[r].cursor = 0;
romfs_fds[r].size = size;
fio_set_opaque(r, romfs_fds + r);
}
}
return r;
}
uint32_t fasthash_insert_impl(fasthash_table *fh_table, char *key, void *entry) {
if (!fh_table) return 0;
uint32_t index = hash_djb2((unsigned char *)key) % fh_table->size;
fasthash_node *node = fh_table->entries[index];
if (node) {
/* If we aren't allowing collisions, return out of range */
if (fh_table->opts & FH_NO_COLLISIONS) {
return fh_table->size + 1;
}
} else if (!(fh_table->opts & FH_NO_INDEXING)) {
/* Build index node if we are indexing */
index_node *in = (index_node *)malloc(sizeof(index_node));
in->index = index;
in->next = fh_table->index_list;
fh_table->index_list = in;
}
node = fasthash_node_construct(key, entry, node);
//printf("Node \"%s\" = %p\n", key, entry);
fh_table->entries[index] = node;
return index; /* We don't need to add an additional index */
}
int fs_open(const char * path, int flags, int mode) {
const char * slash;
uint32_t hash;
int i;
// DBGOUT("fs_open(\"%s\", %i, %i)\r\n", path, flags, mode);
while (path[0] == '/')
path++;
slash = strchr(path, '/');
if (!slash)
return -2;
hash = hash_djb2((const uint8_t *) path, slash - path);
path = slash + 1;
for (i = 0; i < MAX_FS; i++) {
if (fss[i].hash == hash)
return fss[i].cb(fss[i].opaque, path, flags, mode);
}
return -2;
}
int main (int argc, char *argv[])
{
paper_rec_t DedupeRecord;
dd_uint64_t Unique_CRID; /* Unique CR_ID = (C_ID << 16) | CR_ID */
long victim_index = 0, cache_size, window_size, bloom_filter_size;
long i, j=0, temp_index;
int Initial_Flag = 0, cache_algorithm;
dd_uint8_t *sha1_value=NULL;
int nLen = 0;
long max_counter=0;
HTItem *chunk_item, *item;
long byte_len, temp, offset, ver, temp1; /* to read a trace file */
unsigned long hash1, hash2;
/* Heap Data structure variables */
Cache_Memory Dataitem;
std::vector<Cache_Memory>::iterator itr;
unsigned long writeCounter = 0;
unsigned long access_counter;
long file_position;
FILE *fp1, *fp;
size_t keySize=0,iCnt;
clock_t start = clock();
time_t begin,end;
time(&begin);
if (argc < 5) {
/* 0 1 2 3 4 */
fprintf(stderr, "usage: %s <Cache Size> <Window Size> <Cache Algorithm (0, 1, 2)> <Trace File>\n", argv[0]);
fprintf(stderr, " - Cache Size: Dedupe read cache size in terms of # of data chunk (e.g. 500 chunks = 4MB (500*8KB))\n");
fprintf(stderr, " - Window Size: Future sliding window size in terms of TIMES of cache size.\n");
fprintf(stderr, " - Cache Algorithm: 0 (Belady MIN), 1 (Belady MIN with a future window), 2 (Lookahead read cache)\n");
fprintf(stderr, " - Trace File: Trace file name with path\n");
exit(1);
}
cache_size = atol(argv[1]);
assert(cache_size > 0); /* cache size must be positive */
window_size = atol(argv[2]);
assert(window_size > 0); /* window size must be positive */
cache_algorithm = atoi(argv[3]);
assert((cache_algorithm == 0)||(cache_algorithm == 1)||(cache_algorithm == 2)); /* there are only three selections */
bloom_filter_size = cache_size*2; //No. of Hash functions for BF is 2
bloom_filter = (long *)malloc(sizeof(long)*bloom_filter_size);
ht_cache = AllocateHashTable(SHA1_VALUE_LENGTH, 1);
heap = newMinHeap((u32)cache_size);
if((fp1 = fopen(argv[4], "rb")) == NULL){ //for reading data one by one
DEBUG_INFO("File open error....1\n");
exit (-1);
}
if((fp = fopen(argv[4], "rb")) == NULL){ //for searching its future reference distance
DEBUG_INFO("File open error....2\n");
exit (-1);
}
long c=0, d=0;
u32 itemIndex;
keySize = sizeof(DedupeRecord.fp_bytes);
DEBUG_INFO("Record Size is: %d\n",keySize);
while (1)
{
fread(&DedupeRecord, sizeof(struct _paper_rec_t),1, fp1);
/*if(DedupeRecord.fp_bytes[0] == 0)
DedupeRecord.fp_bytes[0] = '0';*/
/*for(iCnt = 0;iCnt<sizeof(DedupeRecord.fp_bytes);++iCnt)
printf("%c",DedupeRecord.fp_bytes[iCnt]);*/
//DEBUG_INFO("Reading chunks : %ld\n", c++);
c++;
if(c%1000 == 0){
printf("Reading Chunks: %ld\n",c);
}
if (c % 10000 == 0) {
printf("Cache hit ratio: %.3f = %lu / %lu \n",
(double) (Hit_Count * 100) / (double) totalAccess ,
Hit_Count,
totalAccess);
}
if(feof(fp1))
break;
file_position = ftell(fp1);
/* initially fill the cache. During this initialization process, we do not count the cache hit ratio. */
if (Initial_Flag == 0) {
// Temporally store this current access chunk with its future reference distance in the cache
chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,DedupeRecord.fp_bytes));
//Update Bloom filter counters
hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
if(chunk_item) { //Cache Hit
itemIndex = (u32)chunk_item->data;
DEBUG_INFO("Index its updating is %ld:\n",itemIndex);
heapUpdate(heap,max_counter,itemIndex,&ht_cache);
}
else {
heapInsert(heap,DedupeRecord.fp_bytes, max_counter,&ht_cache);
//Sandeep - Insert into Heap and Heapify
cache_counter++;
}
if(cache_counter == cache_size) {
DEBUG_INFO("\n#### Cache Initialization complete~!!####\n");
Initial_Flag = 1;
//Sandeep - Construct Heap and Heapify
//fnBuildHeap(cache_heap);
#ifdef DEBUG
printf("Heap Size is: %d\n",cache_heap.size());
/*PrintHashTable(ht_cache,-1,2);
fnPrintHeap(cache_heap);*/
#endif
}
}
else { /* Once the cache is full of data initially, we start to measure the cache hit ratio from now. */
totalAccess++;
if((totalAccess % 100) == 0) {
DEBUG_INFO("[CHECK] Current Access Number: %ld\n", totalAccess);
}
Unique_CRID = (DedupeRecord.cmc_id << 16) | DedupeRecord.creg_id;
chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,DedupeRecord.fp_bytes));
if(chunk_item) { //Cache Hit
Hit_Count++;
DEBUG_INFO("Cache Hit\n");
//Update Bloom filter counters
hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
//DEBUG_INFO("### Returned hash values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
itemIndex = (ulong)chunk_item->data;
DEBUG_INFO("Index its updating is %ld:\n",itemIndex);
assert(itemIndex>=0 && itemIndex<=cache_size);
heapUpdate(heap,max_counter,itemIndex,&ht_cache);
//Sandeep - Update heap counter val for this chunk with max_counter
//fnUpdateHeap(cache_heap, Read_Cache[(ulong)chunk_item->data],max_counter);
}
else {
heapPopMin(heap,&sha1_value,&access_counter,&ht_cache);
if(!sha1_value)
ERROR("SHA1 Value in main is NULL\n");
/*for(iCnt = 0;iCnt<sizeof(DedupeRecord.fp_bytes);++iCnt)
printf("%c",sha1_value[iCnt]);*/
//Update Bloom filter counters
hash1 = hash_djb2(sha1_value,sizeof(sha1_value))%bloom_filter_size;
hash2 = hash_sdbm(sha1_value,sizeof(sha1_value))%bloom_filter_size;
//DEBUG_INFO("### In Main before decrement %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
//Decrement BF counters
bloom_filter[hash1]--;
bloom_filter[hash2]--;
free(sha1_value);
//GP - Increment the BF counters for this chunk
hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
//DEBUG_INFO("### Returned hash values are in main cache_miss %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
heapInsert(heap,DedupeRecord.fp_bytes,max_counter,&ht_cache);
if(cache_algorithm == LOOKAHEAD){
/* Check if any other chunks in the current CR will appear within the future window.
If we found one, we add such chunk(s) in the cache. */
Check_Unique_CRID(fp, Unique_CRID, file_position, 0, cache_size, window_size*cache_size, bloom_filter_size);
}
}
} //else
} //while
printf("\n###################################################################\n");
printf("Cache hit ratio: %.3f = %lu / %lu \n", (double) (Hit_Count * 100) / (double) totalAccess , Hit_Count, totalAccess);
printf("Cache size: %ld, window size: %ld\n", cache_size, window_size*cache_size);
printf("Dedupe trace: %s\n", argv[4]);
printf("###################################################################\n");
fclose(fp1);
fclose(fp);
FreeHashTable(ht_cache);
deleteMinHeap(heap);
time(&end);
printf("###################################################################\n");
printf("Total time taken is %f \n",((double)clock()-start)/CLOCKS_PER_SEC);
printf("###################################################################\n");
return 0;
}
void Check_Unique_CRID(FILE *pFile, dd_uint64_t Current_Unique_CRID, long position, long distance, long cache_size, long window_size, long bloom_filter_size)
{
long i, victim_index;
dd_uint64_t Temp_Unique_CRID;
paper_rec_t Temp_DedupeTrace;
long window_counter=0;
long max_counter=0, temp_index=0;
dd_uint8_t *sha1_value=NULL;
HTItem *chunk_item, *item;
unsigned long hash1, hash2;
unsigned long access_counter;
size_t keySize = sizeof(Temp_DedupeTrace.fp_bytes),iCnt;
bool flag=true;
std::list<paper_rec_t>::iterator itr;
/* Heap Data structure variables */
/* Check the size of sliding window vector if its empty
* the check unique crid has been called first time
* initialize the vector with size of sliding window */
if(SlidingWindow.size() == 0)
{
fseek(pFile, position, SEEK_SET);
while (1)
{
fread(&Temp_DedupeTrace, sizeof(struct _paper_rec_t),1, pFile);
/*if(Temp_DedupeTrace.fp_bytes[0] == 0)
Temp_DedupeTrace.fp_bytes[0] = '0';*/
if(feof(pFile))
break;
SlidingWindow.push_back(Temp_DedupeTrace);
if(SlidingWindow.size() >= window_size) {
break;
}
}
}
else if(SlidingWindow.size() == window_size){
/* Remove one old record and insert the latest record */
SlidingWindow.pop_front();
fseek(pFile, position + window_size, SEEK_SET);
fread(&Temp_DedupeTrace, sizeof(struct _paper_rec_t),1, pFile);
SlidingWindow.push_back(Temp_DedupeTrace);
}
for(itr = SlidingWindow.begin();itr!=SlidingWindow.end();itr++){
Temp_Unique_CRID = ((*itr).cmc_id << 16) | (*itr).creg_id;
/* if any data chunk in current CR appear within the future window */
if(Temp_Unique_CRID == Current_Unique_CRID) {
DEBUG_INFO("[Found~!!] A chunk in current access CR will appeare within a future window.\n");
chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,(*itr).fp_bytes));
if(chunk_item) { //Cache Hit
DEBUG_INFO("Cache Hit - New\n");
//Update Bloom filter counters
hash1 = hash_djb2((*itr).fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm((*itr).fp_bytes,keySize)%bloom_filter_size;
/* DEBUG_INFO("### Returned hash values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);*/
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
temp_index = (u32)chunk_item->data;
/* DEBUG_INFO("Index its updating is %ld:\n",temp_index);*/
heapUpdate(heap, max_counter, temp_index,&ht_cache);
}
else {
//Sandeep - Choose victim from heap and strcpy Sha1 Value of the victim chunk to "sha1_value" variable
/*sha1_value = fnDeleteItemHeap(cache_heap);*/
//GP - Increment the BF counters for this chunk
hash1 = hash_djb2((*itr).fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm((*itr).fp_bytes,keySize)%bloom_filter_size;
//DEBUG_INFO("### Returned hash values are before insert cache miss %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
flag = checkMin(heap,max_counter);
/* If Min on heap is less than the new chuck then only replace with new
* else just skip */
if(flag){
heapPopMin(heap,&sha1_value,&access_counter,&ht_cache);
if(!sha1_value)
ERROR("SHA1 Value in Check Unique CR is NULL\n");
/*for(iCnt = 0;iCnt<sizeof((*itr).fp_bytes);++iCnt)
printf("%c",sha1_value[iCnt]);*/
//Update Bloom filter counters
hash1 = hash_djb2(sha1_value,sizeof(sha1_value))%bloom_filter_size;
hash2 = hash_sdbm(sha1_value,sizeof(sha1_value))%bloom_filter_size;
//DEBUG_INFO("### Before Decrementing values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
//Decrement BF counters
bloom_filter[hash1]--;
bloom_filter[hash2]--;
free(sha1_value);
//Sandeep - Insert chunk into Heap with max_counter
heapInsert(heap,(*itr).fp_bytes,max_counter,&ht_cache);
}
}
}
}
}
operator size_t() const
{ return hash_djb2((const unsigned char *)c_str); }
/**
* key_index - gives the index of a key
* @key: the key
* @size: size of the array of the hash table
* Return: key index
*/
unsigned long int key_index(const unsigned char *key, unsigned long int size)
{
return (hash_djb2(key) % size);
}
static unsigned int hashfromkey(void *ky)
{
struct key *k = ky;
return hash_djb2(k->str);
}
