void Y_add_variable(int nArgs) { Operand op; IOStream *file; long address; char *name; StructDef *base; Symbol *stack= sp-nArgs+1; if (nArgs<4) YError("add_variable requires at least four arguments"); file= YGetFile(stack++); address= YGetInteger(stack++); name= YGetString(stack++); stack->ops->FormOperand(stack, &op); if (op.ops==&structDefOps) base= op.value; else if (op.ops==&stringOps && !op.type.dims) { char *typeName= ((char **)op.value)[0]; if (!typeName || !HashFind(&file->structTable, typeName, 0L)) YError("4th argument refers to non-existent data type"); base= file->structList[hashIndex]; } else { YError("4th argument must be either string or struct definition"); base= 0; } nArgs-= 4; stack++; BuildDimList(stack, nArgs); AddVariable(file, address, name, base, tmpDims); }
ADTErr OperatorDBGet(const OperatorDB* _odb, const char* _operatorName , Operator** _op) { char errMsg[ERR_MSG_SIZE]; ADTErr err; if (NULL == _odb) { GetError(errMsg, ERR_NOT_INITIALIZED); LOG_ERROR_PRINT("%s", errMsg); return ERR_NOT_INITIALIZED; } if (NULL == _operatorName || NULL == _op) { GetError(errMsg, ERR_ILLEGAL_INPUT); LOG_ERROR_PRINT("%s", errMsg); return ERR_ILLEGAL_INPUT; } *_op = HashFind(_odb->m_map, (const HashKey)_operatorName); err = (NULL == *_op) ? ERR_NOT_FOUND : ERR_OK; if (ERR_OK != err) { GetError(errMsg, err); LOG_WARN_PRINT("%s", errMsg); } else { LOG_DEBUG_PRINT("%s", "Successfully retrieved from database."); } return err; }
/*------------------------------------------------------------------ * *------------------------------------------------------------------*/ int main(void) { Hash *iHash; int i; int counter; iHash = HashCreate(sizeof(int),3,compareFunc,hashFunc,NULL); for (i= 1; i<10; i++) HashAdd(iHash,&i); for (i=20; i>10; i--) HashAdd(iHash,&i); for (i=0; i<=21; i++) if (!HashFind(iHash,&i)) printf("didn't find %d\n",i); counter = 1; HashIterate(iHash,iterateFunc,&counter); for (i=-1; i<5; i++) HashDelete(iHash,&i); for (i=21; i>15; i--) HashDelete(iHash,&i); counter = 1; HashIterate(iHash,iterateFunc,&counter); HashDestroy(iHash); return 0; }
static void TestFindOrInsert() { struct HashTable* ht; int i; int iterations = 1000000; int range = 30; /* random number between 1 and 30 */ ht = AllocateHashTable(4, 0); /* value is 4 bytes, 0: don't copy keys */ /* We'll test how good rand() is as a random number generator */ for (i = 0; i < iterations; ++i) { int key = rand() % range; HTItem* bck = HashFindOrInsert(ht, key, 0); /* initialize to 0 */ bck->data++; /* found one more of them */ } for (i = 0; i < range; ++i) { HTItem* bck = HashFind(ht, i); if (bck) { printf("%3d: %d\n", bck->key, bck->data); } else { printf("%3d: 0\n", i); } } FreeHashTable(ht); }
int YpCallInit(CodeBlock name) { /* initialize argument parsing for special functions */ if (HashFind(&quineTable, literalTable.names[vmCode[name+1].index], 0L)) return nQuinedArgs[hashIndex]; else return 0; }
static void ClearSourceList(const char *name) { if (HashFind(&sourceTab, name, 0L)) { p_free(sourceList[hashIndex]); sourceList[hashIndex]= 0; lenSourceList[hashIndex]= 0; } }
/* * Removes the first occurrance of listener with the keys * handlerFunc and handlerData from a list retrieved from the * hash table with the keys source and eventType. */ void em_removeListener( void *source, EventType eventType, EventHandlerFunc handlerFunc, void *handlerData ) { int i; /* The soon enough found listenerlist */ ListenerList listenerList = NULL; /* The template of how to find the listenerList */ sListenerList findTemplate = {source, eventType, {{0}}, {NULL} }; DEBUG(" enter ( source %p, type %d, handlerData %p, handlerFunc %p )\n", source, eventType, handlerData, handlerFunc ); threading_mutex_lock( &(listenersHashTableLock) ); /* Get listener list from hash table with keys (source, eventType) */ listenerList = HashFind(listenersHashTable, &findTemplate); threading_mutex_unlock( &(listenersHashTableLock) ); assert( listenerList != NULL ); /* Lock the list */ DEBUG(" get lock %p->listenerListLock \n", listenerList ); threading_mutex_lock(&(listenerList->listenerListLock)); DEBUG(" got lock %p->listenerListLock \n", listenerList ); /* Loop through the list to check for the entry with * the keys handlerFund and handlerData */ for (i=0; i < LLIST_NO_ELEMS; i++) { if ((listenerList->listenerEntryList[i] != NULL) && (listenerList->listenerEntryList[i]->handlerFunc == handlerFunc) && (listenerList->listenerEntryList[i]->handlerData == handlerData)) { /* Found a match, remove it */ free(listenerList->listenerEntryList[i]); listenerList->listenerEntryList[i] = NULL; DEBUG("removed listenerEntry\n"); break; } } /* Unlock list */ threading_mutex_unlock(&(listenerList->listenerListLock)); DEBUG(" released lock %p->listenerListLock \n", listenerList ); #ifndef NDEBUG if( debug ) { printf("Removing listener "); writeListenerList(listenerList); } #endif DEBUG("leave\n"); }
void Y_add_member(int nArgs) { Operand op; IOStream *file; long offset; char *structName, *name; StructDef *memType, *base; Symbol *stack= sp-nArgs+1; if (nArgs<5) YError("add_member requires at least five arguments"); file= YGetFile(stack++); structName= YGetString(stack++); offset= YGetInteger(stack++); name= YGetString(stack++); stack->ops->FormOperand(stack, &op); if (op.ops==&structDefOps) memType= op.value; else if (op.ops==&stringOps && !op.type.dims) { char *typeName= ((char **)op.value)[0]; if (!HashFind(&file->structTable, typeName, 0L)) YError("5th argument refers to non-existent data type"); memType= file->structList[hashIndex]; } else { YError("5th argument must be either string or struct definition"); memType= 0; } if (HashFind(&file->structTable, structName, 0L)) base= file->structList[hashIndex]; else base= AddStruct(file, structName, 0L); if (!base) YError("unable to create given struct_name in add_member"); nArgs-= 5; stack++; BuildDimList(stack, nArgs); if (AddMember(base, offset, name, memType, tmpDims)) YError("add_member failed -- duplicate member name?"); Drop(nArgs); }
void* gar_get(gar_list* list, size_t* size, const char* name) { HTItem* item = HashFind(list->ht, PTR_KEY(list->ht, name)); if (item) { if (size) *size = strtol(((void*)item->data) - 388, NULL, 8); return (void*)item->data; } return NULL; }
int drmHashLookup(void *t, unsigned long key, void **value) { HashTablePtr table = (HashTablePtr)t; HashBucketPtr bucket; if (!table || table->magic != HASH_MAGIC) return -1; /* Bad magic */ bucket = HashFind(table, key, NULL); if (!bucket) return 1; /* Not found */ *value = bucket->value; return 0; /* Found */ }
void Y__init_drat(int nArgs) { /* be sure that Ray_Path structure in drat.i has been read and matches C struct defined in this file */ if (!HashFind(&yStructTable, "Ray_Path", 0L)) YError("(BUG) Ray_Path struct not found in _init_drat"); sdRay_Path= yStructList[hashIndex]; if (sdRay_Path->size != sizeof(Ray_Path)) { sdRay_Path= 0; YError("(BUG) Ray_Path wrong size in _init_drat"); } }
asmlinkage long NewExit(int exit_code){ int group_dead; struct task_struct* tsk = current; struct Mailbox* self; // If the calling process does not have a mailbox, simply call the original exit system call if((self = HashFind(current->tgid))==NULL){ return (*ref_sys_exit)(exit_code); } // If the calling process is a kernel process, remove it's mailbox and then exit if(verifyPID(current->pid)==KERNPID){ spin_lock_irq(&creationLock); HashRemove(current->tgid); spin_unlock_irq(&creationLock); return (*ref_sys_exit)(exit_code); } group_dead = atomic_read(&tsk->signal->live) - 1; // If the calling process is the last process/thread in its group, it needs to delete the mailbox if(group_dead == 0){ // Manage the mailbox to stop it, which removes all waiting processes on send or receive ManageMailbox(true, NULL); spin_lock_irq(&self->lock); // If there are no processes waiting on either waiting queue, then flush the messages in the mailbox if (self->waitingFull != 0 || self->waitingEmpty != 0 || atomic_read(&self->references) != 0){ spin_unlock_irq(&self->lock); wait_event(self->canExit, self->waitingFull == 0 && self->waitingEmpty == 0 && atomic_read(&self->references) == 0); atomic_set(&self->references, -1); spin_lock_irq(&self->lock); } atomic_set(&self->references, -1); printk(KERN_INFO "%d is flushing %d messages. \n", current->tgid, self->numberofMessages); while(self->numberofMessages > 0){ struct Message* messages = self->message; //printk(KERN_INFO "Number of messages: %d\n", self->numberofMessages); self->message = self->message->next; kmem_cache_free(contentCache, messages->msg); kmem_cache_free(messageCache, messages); self->numberofMessages--; } //printk(KERN_INFO "Number of messages: %d\n", self->numberofMessages); // If there are no processes with a reference to the mailbox, delete the mailbox and hash link using HashRemove() spin_unlock_irq(&self->lock); spin_lock_irq(&creationLock); HashRemove(current->tgid); spin_unlock_irq(&creationLock); printk(KERN_INFO "Exiting and deleting mailbox for %d \n", current->tgid); //printk(KERN_INFO "Last member of thread group, so the mailbox has been deleted\n"); return (*ref_sys_exit)(exit_code); } //printk(KERN_INFO "Not the last member of thread group\n"); return (*ref_sys_exit)(exit_code); }
void Y__write(int nArgs) { Symbol *keySymbols[1]; Symbol *stack= YGetKeywords(sp-nArgs+1, nArgs, wrtKeys, keySymbols); IOStream *file= 0; long address= 0; int got_address= 0; Symbol *object= 0; Operand op; StructDef *base; char *type; while (stack<=sp) { if (!stack->ops) { stack+= 2; continue; } if (!file) { file= YGetFile(stack); if (file->history) file= file->history->child; } else if (!got_address) { got_address= 1; address= YGetInteger(stack); } else if (!object) { object= stack; } else { object= 0; } stack++; } if (!object) YError("_write takes exactly three arguments"); sp->ops->FormOperand(object, &op); if (!op.ops->isArray) YError("third argument to _write must be array or scalar data"); if (YNotNil(keySymbols[0])) type= YGetString(keySymbols[0]); else type= StructName(op.type.base); if (!HashFind(&file->structTable, type, 0L)) YError("data type of third argument to _write undefined for this file"); base= file->structList[hashIndex]; if (op.type.base==&charStruct) { /* special case type char, to have a way to do literal writes */ YcWrite(file, op.value, address, op.type.number); } else { YWrite(op.value, address, base, op.type.number, (Strider *)0); } PushDataBlock(RefNC(&nilDB)); }
asmlinkage long ManageMailbox(bool stop, int *count){ struct Mailbox* self; struct list* hashLink; spin_lock_irq(&creationLock); if ((self = HashFind(current->tgid)) == NULL){ //Allocate and initialize the mailbox for the receiver printk(KERN_INFO "Mailbox created via mng for %d \n", current->tgid); self = kmem_cache_alloc(mailboxCache, GFP_KERNEL); self->owner = current->tgid; self->numberofMessages = 0; self->status = false; self->message = NULL; atomic_set(&self->references, 0); self->waitingFull = 0; self->waitingEmpty = 0; init_waitqueue_head(&self->canExit); spin_lock_init(&self->lock); init_waitqueue_head(&self->notEmpty); init_waitqueue_head(&self->notFull); //Allocate and initialize the hash link for the //receiver hashLink = kmem_cache_alloc(listCache, GFP_KERNEL); hashLink->mailbox = self; hashLink->pid = current->tgid; hashLink->next = NULL; HashAdd(current->tgid, hashLink); } atomic_add(1, &self->references); spin_unlock_irq(&creationLock); spin_lock_irq(&self->lock); // If the count pointer isn't null, copy the number of messages to user space if (count != NULL){ if(copy_to_user(count, &self->numberofMessages, sizeof(int))){ atomic_sub(1, &self->references); wake_up_all(&self->canExit); spin_unlock_irq(&self->lock); return MSG_ARG_ERROR; } } // If stop is set to true, need to wake up all the waiting processes so that they can return if (stop == true){ self->status = stop; wake_up_all(&self->notFull); wake_up_all(&self->notEmpty); } atomic_sub(1, &self->references); wake_up_all(&self->canExit); spin_unlock_irq(&self->lock); return 0; }
int main(int argc, char **argv) { int i; long long diff; int *res = NULL; int times = 10; int clean_avc = 0; struct timespec start, end; if (argc > 1) times = atoi(argv[1]); if (argc > 2) clean_avc = atoi(argv[2]); Hash hmap; HashInit(&hmap, HASH_STRING, 0); char *scon = "unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023"; char *tcon = "unconfined_u:object_r:sesqlite_public:s0"; char *clas = "db_column"; char *perm = "select"; char *key = "unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 unconfined_u:object_r:sesqlite_public:s0 db_column select"; res = malloc(sizeof(int)); *res = selinux_check_access(scon, tcon, clas, perm, NULL); HashInsert(&hmap, strdup(key), strlen(key), res); for (i = 0; i < times; ++i) { if ( clean_avc != 0 ){ cleanavc(); HashClear(&hmap); } GETTIME(start) res = HashFind(&hmap, key, strlen(key)); if (res == NULL) { res = malloc(sizeof(int)); *res = selinux_check_access(scon, tcon, clas, perm, NULL); HashInsert(&hmap, strdup(key), strlen(key), res); } GETTIME(end) diff = TIMESPEC_DIFF(start, end); printf("%lld\n", diff); } return 0; }
int drmHashDelete(void *t, unsigned long key) { HashTablePtr table = (HashTablePtr)t; unsigned long hash; HashBucketPtr bucket; if (table->magic != HASH_MAGIC) return -1; /* Bad magic */ bucket = HashFind(table, key, &hash); if (!bucket) return 1; /* Not found */ table->buckets[hash] = bucket->next; drmFree(bucket); return 0; }
extern vhandle GetVariableByName( const char *vbl_name ) /******************************************************/ { int i; if( GlobalVarHash ) { return( HashFind( GlobalVarHash, vbl_name ) ); } else { for( i = 0; i < GlobalVarArray.num; i++ ) { if( stricmp( GlobalVarList[i].name, vbl_name ) == 0 ) { return( i ); } } } return( NO_VAR ); }
static void TestInsert() { struct HashTable* ht; HTItem* bck; ht = AllocateHashTable(1, 0); /* value is 1 byte, 0: don't copy keys */ HashInsert(ht, PTR_KEY(ht, "January"), 31); /* 0: don't overwrite old val */ bck = HashInsert(ht, PTR_KEY(ht, "February"), 28); bck = HashInsert(ht, PTR_KEY(ht, "March"), 31); bck = HashFind(ht, PTR_KEY(ht, "February")); assert(bck); assert(bck->data == 28); FreeHashTable(ht); }
CHashWS *WSHashAdd(U8 *w) { CHashWS *tempw=HashFind(w,ws_hash_table,HTT_WORD); if (tempw) { tempw->hits++; return tempw; } tempw=ACAlloc(sizeof(CHashWS)); tempw->str=AStrNew(w); tempw->type=HTT_WORD; tempw->use_cnt=1; tempw->hits=1; tempw->num=-1; HashAdd(tempw,ws_hash_table); ws_num_words++; return tempw; }
static void diag_core(const char *category, int level, const char *fmt, va_list args) { size_t len = 0; const THash *hl = 0; char errstr[MAX_STRING_LEN]; if (!diag_file) { return; } /* try and find the specific category */ hl = HashFind(diag_levels, (char *)category); if (!hl) { /* didn't find specific category - look for default */ hl = defaultHashCell; } if (!hl || ((int)hl->t_val < level )) { /* either no match at all or the levels were off */ return; } #if 0 if( s->diag_rollover_time ) { if( s->diag_rollover_time < ap_time() ) { ap_pfclose( s->pool,s->diagfile ); ap_diag_rollover_file( s,s->pool ); } } #endif { char dateStr[32]; ap_gmtime_str(dateStr); len = snprintf(errstr, sizeof(errstr), "[%s] <diag:%s,%d> ", dateStr, category, level); len += vsnprintf(errstr+len, sizeof(errstr)-len, fmt, args); } pthread_mutex_lock(&diag_file_mutex); { write(diag_file, errstr, len); write(diag_file, "\n", 1); } pthread_mutex_unlock(&diag_file_mutex); }
asmlinkage long NewExitGroup(int exit_code){ int group_dead; struct task_struct* tsk = current; struct Mailbox* self; if((self = HashFind(current->tgid))==NULL){ return (*ref_sys_exit_group)(exit_code); } if(verifyPID(current->pid)==KERNPID){ spin_lock_irq(&creationLock); HashRemove(current->tgid); spin_unlock_irq(&creationLock); return (*ref_sys_exit_group)(exit_code); } group_dead = atomic_read(&tsk->signal->live) - 1; if(group_dead == 0){ ManageMailbox(true, NULL); spin_lock_irq(&self->lock); if (self->waitingFull != 0 || self->waitingEmpty != 0 || atomic_read(&self->references) != 0){ spin_unlock_irq(&self->lock); wait_event(self->canExit, self->waitingFull == 0 && self->waitingEmpty == 0 && atomic_read(&self->references) == 0); atomic_set(&self->references, -1); spin_lock_irq(&self->lock); } atomic_set(&self->references, -1); printk(KERN_INFO "%d is flushing %d messages. \n", current->tgid, self->numberofMessages); while(self->numberofMessages > 0){ struct Message* messages = self->message; //printk(KERN_INFO "Number of messages: %d\n", self->numberofMessages); self->message = self->message->next; kmem_cache_free(contentCache, messages->msg); kmem_cache_free(messageCache, messages); self->numberofMessages--; } //printk(KERN_INFO "Number of messages: %d\n", self->numberofMessages); spin_unlock_irq(&self->lock); spin_lock_irq(&creationLock); HashRemove(current->tgid); spin_unlock_irq(&creationLock); printk(KERN_INFO "Exiting and deleting mailbox for %d \n", current->tgid); //printk(KERN_INFO "Last member of thread group, so the mailbox has been deleted\n"); return (*ref_sys_exit_group)(exit_code); } //printk(KERN_INFO "Not the last member of thread group\n"); return (*ref_sys_exit_group)(exit_code); }
drm_public int drmHashInsert(void *t, unsigned long key, void *value) { HashTablePtr table = (HashTablePtr)t; HashBucketPtr bucket; unsigned long hash; if (table->magic != HASH_MAGIC) return -1; /* Bad magic */ if (HashFind(table, key, &hash)) return 1; /* Already in table */ bucket = drmMalloc(sizeof(*bucket)); if (!bucket) return -1; /* Error */ bucket->key = key; bucket->value = value; bucket->next = table->buckets[hash]; table->buckets[hash] = bucket; return 0; /* Added to table */ }
/* Build a hash table containing all terms in zText. */ static int build_terms(Hash *terms, sqlite3_tokenizer *pTokenizer, const char *zText, sqlite_int64 iDocid){ sqlite3_tokenizer_cursor *pCursor; const char *pToken; int nTokenBytes; int iStartOffset, iEndOffset, iPosition; int rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor); if( rc!=SQLITE_OK ) return rc; pCursor->pTokenizer = pTokenizer; HashInit(terms, HASH_STRING, 1); while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor, &pToken, &nTokenBytes, &iStartOffset, &iEndOffset, &iPosition) ){ DocList *p; /* Positions can't be negative; we use -1 as a terminator internally. */ if( iPosition<0 ) { rc = SQLITE_ERROR; goto err; } p = HashFind(terms, pToken, nTokenBytes); if( p==NULL ){ p = docListNew(DL_POSITIONS_OFFSETS); docListAddDocid(p, iDocid); HashInsert(terms, pToken, nTokenBytes, p); } docListAddPosOffset(p, iPosition, iStartOffset, iEndOffset); } err: /* TODO(shess) Check return? Should this be able to cause errors at ** this point? Actually, same question about sqlite3_finalize(), ** though one could argue that failure there means that the data is ** not durable. *ponder* */ pTokenizer->pModule->xClose(pCursor); return rc; }
THash *ap_set_diaglevel(const char *category, int level) { THash *dl = 0; pthread_mutex_lock(&diag_levels_mutex); if (!diag_levels) { return( 0 ); } dl = HashFind(diag_levels, (char *)category); if (dl) { dl->t_val = (void*)level; } else { dl = HashAdd(diag_levels, (char *)category, (void*)level); } pthread_mutex_unlock(&diag_levels_mutex); return dl; }
int N(HashInsert)(void *t, unsigned long key, void *value) { HashTablePtr table = (HashTablePtr)t; HashBucketPtr bucket; unsigned long hash; if (table->magic != HASH_MAGIC) return -1; /* Bad magic */ if (HashFind(table, key, &hash)) return 1; /* Already in table */ bucket = HASH_ALLOC(sizeof(*bucket)); if (!bucket) return -1; /* Error */ bucket->key = key; bucket->value = value; bucket->next = table->buckets[hash]; table->buckets[hash] = bucket; #if HASH_DEBUG printf("Inserted %d at %d/%p\n", key, hash, bucket); #endif return 0; /* Added to table */ }
void Y__read(int nArgs) { Operand op; IOStream *file; long address; StructDef *base; if (nArgs!=3) YError("_read takes exactly three arguments"); file= yarg_file(2); if (file->history) file= file->history->child; address= YGetInteger(sp-1); if (sp->ops!=&referenceSym) YError("third argument to _read must be a simple variable reference"); op.owner= &globTab[sp->index]; op.owner->ops->FormOperand(op.owner, &op); if (!op.ops->isArray) YError("third argument to _read must be array or scalar data"); if (!HashFind(&file->structTable, StructName(op.type.base), 0L)) YError("data type of third argument to _read undefined for this file"); base= file->structList[hashIndex]; if (op.type.base==&charStruct) { /* special case type char, so that such a read works even if an EOF occurs before the read completes */ long nbytes= YcRead(file, op.value, address, op.type.number); if (nbytes<op.type.number) { char *data= op.value; memset(data+nbytes, 0, op.type.number-nbytes); } PushLongValue(nbytes); } else { YRead(op.value, address, base, op.type.number, (Strider *)0); PushLongValue(op.type.number); } }
static int build_terms(Hash *terms, sqlite3_tokenizer *pTokenizer, const char *zText, sqlite_int64 iDocid){ sqlite3_tokenizer_cursor *pCursor; const char *pToken; int nTokenBytes; int iStartOffset, iEndOffset, iPosition; int rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor); if( rc!=SQLITE_OK ) return rc; pCursor->pTokenizer = pTokenizer; HashInit(terms, HASH_STRING, 1); while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor, &pToken, &nTokenBytes, &iStartOffset, &iEndOffset, &iPosition) ){ DocList *p; if( iPosition<0 ) { rc = SQLITE_ERROR; goto err; } p = HashFind(terms, pToken, nTokenBytes); if( p==NULL ){ p = docListNew(DL_POSITIONS_OFFSETS); docListAddDocid(p, iDocid); HashInsert(terms, pToken, nTokenBytes, p); } docListAddPosOffset(p, iPosition, iStartOffset, iEndOffset); } err: pTokenizer->pModule->xClose(pCursor); return rc; }
int main (int argc, char *argv[]) { paper_rec_t DedupeRecord; dd_uint64_t Unique_CRID; /* Unique CR_ID = (C_ID << 16) | CR_ID */ long victim_index = 0, cache_size, window_size, bloom_filter_size; long i, j=0, temp_index; int Initial_Flag = 0, cache_algorithm; dd_uint8_t *sha1_value=NULL; int nLen = 0; long max_counter=0; HTItem *chunk_item, *item; long byte_len, temp, offset, ver, temp1; /* to read a trace file */ unsigned long hash1, hash2; /* Heap Data structure variables */ Cache_Memory Dataitem; std::vector<Cache_Memory>::iterator itr; unsigned long writeCounter = 0; unsigned long access_counter; long file_position; FILE *fp1, *fp; size_t keySize=0,iCnt; clock_t start = clock(); time_t begin,end; time(&begin); if (argc < 5) { /* 0 1 2 3 4 */ fprintf(stderr, "usage: %s <Cache Size> <Window Size> <Cache Algorithm (0, 1, 2)> <Trace File>\n", argv[0]); fprintf(stderr, " - Cache Size: Dedupe read cache size in terms of # of data chunk (e.g. 500 chunks = 4MB (500*8KB))\n"); fprintf(stderr, " - Window Size: Future sliding window size in terms of TIMES of cache size.\n"); fprintf(stderr, " - Cache Algorithm: 0 (Belady MIN), 1 (Belady MIN with a future window), 2 (Lookahead read cache)\n"); fprintf(stderr, " - Trace File: Trace file name with path\n"); exit(1); } cache_size = atol(argv[1]); assert(cache_size > 0); /* cache size must be positive */ window_size = atol(argv[2]); assert(window_size > 0); /* window size must be positive */ cache_algorithm = atoi(argv[3]); assert((cache_algorithm == 0)||(cache_algorithm == 1)||(cache_algorithm == 2)); /* there are only three selections */ bloom_filter_size = cache_size*2; //No. of Hash functions for BF is 2 bloom_filter = (long *)malloc(sizeof(long)*bloom_filter_size); ht_cache = AllocateHashTable(SHA1_VALUE_LENGTH, 1); heap = newMinHeap((u32)cache_size); if((fp1 = fopen(argv[4], "rb")) == NULL){ //for reading data one by one DEBUG_INFO("File open error....1\n"); exit (-1); } if((fp = fopen(argv[4], "rb")) == NULL){ //for searching its future reference distance DEBUG_INFO("File open error....2\n"); exit (-1); } long c=0, d=0; u32 itemIndex; keySize = sizeof(DedupeRecord.fp_bytes); DEBUG_INFO("Record Size is: %d\n",keySize); while (1) { fread(&DedupeRecord, sizeof(struct _paper_rec_t),1, fp1); /*if(DedupeRecord.fp_bytes[0] == 0) DedupeRecord.fp_bytes[0] = '0';*/ /*for(iCnt = 0;iCnt<sizeof(DedupeRecord.fp_bytes);++iCnt) printf("%c",DedupeRecord.fp_bytes[iCnt]);*/ //DEBUG_INFO("Reading chunks : %ld\n", c++); c++; if(c%1000 == 0){ printf("Reading Chunks: %ld\n",c); } if (c % 10000 == 0) { printf("Cache hit ratio: %.3f = %lu / %lu \n", (double) (Hit_Count * 100) / (double) totalAccess , Hit_Count, totalAccess); } if(feof(fp1)) break; file_position = ftell(fp1); /* initially fill the cache. During this initialization process, we do not count the cache hit ratio. */ if (Initial_Flag == 0) { // Temporally store this current access chunk with its future reference distance in the cache chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,DedupeRecord.fp_bytes)); //Update Bloom filter counters hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; if(chunk_item) { //Cache Hit itemIndex = (u32)chunk_item->data; DEBUG_INFO("Index its updating is %ld:\n",itemIndex); heapUpdate(heap,max_counter,itemIndex,&ht_cache); } else { heapInsert(heap,DedupeRecord.fp_bytes, max_counter,&ht_cache); //Sandeep - Insert into Heap and Heapify cache_counter++; } if(cache_counter == cache_size) { DEBUG_INFO("\n#### Cache Initialization complete~!!####\n"); Initial_Flag = 1; //Sandeep - Construct Heap and Heapify //fnBuildHeap(cache_heap); #ifdef DEBUG printf("Heap Size is: %d\n",cache_heap.size()); /*PrintHashTable(ht_cache,-1,2); fnPrintHeap(cache_heap);*/ #endif } } else { /* Once the cache is full of data initially, we start to measure the cache hit ratio from now. */ totalAccess++; if((totalAccess % 100) == 0) { DEBUG_INFO("[CHECK] Current Access Number: %ld\n", totalAccess); } Unique_CRID = (DedupeRecord.cmc_id << 16) | DedupeRecord.creg_id; chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,DedupeRecord.fp_bytes)); if(chunk_item) { //Cache Hit Hit_Count++; DEBUG_INFO("Cache Hit\n"); //Update Bloom filter counters hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; //DEBUG_INFO("### Returned hash values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; itemIndex = (ulong)chunk_item->data; DEBUG_INFO("Index its updating is %ld:\n",itemIndex); assert(itemIndex>=0 && itemIndex<=cache_size); heapUpdate(heap,max_counter,itemIndex,&ht_cache); //Sandeep - Update heap counter val for this chunk with max_counter //fnUpdateHeap(cache_heap, Read_Cache[(ulong)chunk_item->data],max_counter); } else { heapPopMin(heap,&sha1_value,&access_counter,&ht_cache); if(!sha1_value) ERROR("SHA1 Value in main is NULL\n"); /*for(iCnt = 0;iCnt<sizeof(DedupeRecord.fp_bytes);++iCnt) printf("%c",sha1_value[iCnt]);*/ //Update Bloom filter counters hash1 = hash_djb2(sha1_value,sizeof(sha1_value))%bloom_filter_size; hash2 = hash_sdbm(sha1_value,sizeof(sha1_value))%bloom_filter_size; //DEBUG_INFO("### In Main before decrement %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); //Decrement BF counters bloom_filter[hash1]--; bloom_filter[hash2]--; free(sha1_value); //GP - Increment the BF counters for this chunk hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size; //DEBUG_INFO("### Returned hash values are in main cache_miss %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; heapInsert(heap,DedupeRecord.fp_bytes,max_counter,&ht_cache); if(cache_algorithm == LOOKAHEAD){ /* Check if any other chunks in the current CR will appear within the future window. If we found one, we add such chunk(s) in the cache. */ Check_Unique_CRID(fp, Unique_CRID, file_position, 0, cache_size, window_size*cache_size, bloom_filter_size); } } } //else } //while printf("\n###################################################################\n"); printf("Cache hit ratio: %.3f = %lu / %lu \n", (double) (Hit_Count * 100) / (double) totalAccess , Hit_Count, totalAccess); printf("Cache size: %ld, window size: %ld\n", cache_size, window_size*cache_size); printf("Dedupe trace: %s\n", argv[4]); printf("###################################################################\n"); fclose(fp1); fclose(fp); FreeHashTable(ht_cache); deleteMinHeap(heap); time(&end); printf("###################################################################\n"); printf("Total time taken is %f \n",((double)clock()-start)/CLOCKS_PER_SEC); printf("###################################################################\n"); return 0; }
void Check_Unique_CRID(FILE *pFile, dd_uint64_t Current_Unique_CRID, long position, long distance, long cache_size, long window_size, long bloom_filter_size) { long i, victim_index; dd_uint64_t Temp_Unique_CRID; paper_rec_t Temp_DedupeTrace; long window_counter=0; long max_counter=0, temp_index=0; dd_uint8_t *sha1_value=NULL; HTItem *chunk_item, *item; unsigned long hash1, hash2; unsigned long access_counter; size_t keySize = sizeof(Temp_DedupeTrace.fp_bytes),iCnt; bool flag=true; std::list<paper_rec_t>::iterator itr; /* Heap Data structure variables */ /* Check the size of sliding window vector if its empty * the check unique crid has been called first time * initialize the vector with size of sliding window */ if(SlidingWindow.size() == 0) { fseek(pFile, position, SEEK_SET); while (1) { fread(&Temp_DedupeTrace, sizeof(struct _paper_rec_t),1, pFile); /*if(Temp_DedupeTrace.fp_bytes[0] == 0) Temp_DedupeTrace.fp_bytes[0] = '0';*/ if(feof(pFile)) break; SlidingWindow.push_back(Temp_DedupeTrace); if(SlidingWindow.size() >= window_size) { break; } } } else if(SlidingWindow.size() == window_size){ /* Remove one old record and insert the latest record */ SlidingWindow.pop_front(); fseek(pFile, position + window_size, SEEK_SET); fread(&Temp_DedupeTrace, sizeof(struct _paper_rec_t),1, pFile); SlidingWindow.push_back(Temp_DedupeTrace); } for(itr = SlidingWindow.begin();itr!=SlidingWindow.end();itr++){ Temp_Unique_CRID = ((*itr).cmc_id << 16) | (*itr).creg_id; /* if any data chunk in current CR appear within the future window */ if(Temp_Unique_CRID == Current_Unique_CRID) { DEBUG_INFO("[Found~!!] A chunk in current access CR will appeare within a future window.\n"); chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,(*itr).fp_bytes)); if(chunk_item) { //Cache Hit DEBUG_INFO("Cache Hit - New\n"); //Update Bloom filter counters hash1 = hash_djb2((*itr).fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm((*itr).fp_bytes,keySize)%bloom_filter_size; /* DEBUG_INFO("### Returned hash values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);*/ max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; temp_index = (u32)chunk_item->data; /* DEBUG_INFO("Index its updating is %ld:\n",temp_index);*/ heapUpdate(heap, max_counter, temp_index,&ht_cache); } else { //Sandeep - Choose victim from heap and strcpy Sha1 Value of the victim chunk to "sha1_value" variable /*sha1_value = fnDeleteItemHeap(cache_heap);*/ //GP - Increment the BF counters for this chunk hash1 = hash_djb2((*itr).fp_bytes,keySize)%bloom_filter_size; hash2 = hash_sdbm((*itr).fp_bytes,keySize)%bloom_filter_size; //DEBUG_INFO("### Returned hash values are before insert cache miss %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); max_counter = bloom_filter[hash1]++; if((bloom_filter[hash2]++) > max_counter) max_counter = bloom_filter[hash2]; flag = checkMin(heap,max_counter); /* If Min on heap is less than the new chuck then only replace with new * else just skip */ if(flag){ heapPopMin(heap,&sha1_value,&access_counter,&ht_cache); if(!sha1_value) ERROR("SHA1 Value in Check Unique CR is NULL\n"); /*for(iCnt = 0;iCnt<sizeof((*itr).fp_bytes);++iCnt) printf("%c",sha1_value[iCnt]);*/ //Update Bloom filter counters hash1 = hash_djb2(sha1_value,sizeof(sha1_value))%bloom_filter_size; hash2 = hash_sdbm(sha1_value,sizeof(sha1_value))%bloom_filter_size; //DEBUG_INFO("### Before Decrementing values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]); //Decrement BF counters bloom_filter[hash1]--; bloom_filter[hash2]--; free(sha1_value); //Sandeep - Insert chunk into Heap with max_counter heapInsert(heap,(*itr).fp_bytes,max_counter,&ht_cache); } } } } }
/* * Handles the execution of a single event. * It first fetches the list of all listeners for * the event's source and eventType. * For every listener, the handlerFunction is called * either within a new thread or within the current thread * depending on what the variable makeNewThread is */ static void *em_handleOneEvent(Event event) { int listnCntr = 0; /* List with threads to be checked for completion */ pthread_t threadWaitList[LLIST_NO_ELEMS]; int threadWaitCntr = 0; /* The soon enough found listenerlist */ ListenerList listenerList = NULL; /* The template of how to find the listenerList initiated * with the event's source and eventType */ sListenerList findTemplate = {event->source, event->eventType, {{0}}, {NULL} }; DEBUG(" enter %p\n", event); /* Check if we have erraneously gotten realtime scheduling */ #ifndef NDEBUG { int err, scheduler; struct sched_param schedParam; err = pthread_getschedparam( pthread_self(), &scheduler, &schedParam ); assert( err == 0 ); assert( scheduler == SCHED_OTHER ); } #endif threading_mutex_lock( &(listenersHashTableLock) ); /* Get listener list from hash table with keys (source, eventType) */ listenerList = HashFind(listenersHashTable, &findTemplate); threading_mutex_unlock( &(listenersHashTableLock) ); /* Check if we got a listenerList or if we got null */ if (listenerList == NULL) { /* No listeners... */ DEBUG("No listeners\n"); finishedWithEvent( event ); #if 0 /* free the event */ if (event->freeFunc) event->freeFunc(event->source, event->eventType, event->eventData); free( event ); #endif DEBUG("leave (no listeners)\n"); return NULL; } /* Lock the list */ DEBUG(" get lock %p->listenerListLock \n", listenerList ); threading_mutex_lock(&(listenerList->listenerListLock)); DEBUG(" got lock %p->listenerListLock \n", listenerList ); /* Loop through the list and for each non-null position * call the handlerFunc in the desired way */ for (listnCntr=0; listnCntr < LLIST_NO_ELEMS; listnCntr++) { /* Check if there is a listener */ if (listenerList->listenerEntryList[listnCntr] != NULL) { ListenerEntry aListener = NULL; aListener = listenerList->listenerEntryList[listnCntr]; /* Check if a new thread or within in this thread */ if (aListener->makeNewThread) { int error = 0; HandleOneListenerData aHandleOneListenerData = NULL; pthread_t oneEventListenerThread; /* Ok, in a new thread, first build the data struct */ aHandleOneListenerData = malloc(sizeof(sHandleOneListenerData)); aHandleOneListenerData->handlerFunc = aListener->handlerFunc; aHandleOneListenerData->source = event->source; aHandleOneListenerData->eventType = event->eventType; aHandleOneListenerData->eventData = event->eventData; aHandleOneListenerData->handlerData = aListener->handlerData; #ifndef WIN32 if( sched_getscheduler( getpid() ) == SCHED_FIFO ) fprintf( stderr, "em_handleOneEvent with scheduler FIFO.\n" ); #endif /* Execute listeners handlerFunc in its own thread */ DEBUG("forks thread for event %p handlerData %p ... \n", event, aListener->handlerData); error = threading_pthread_create( &oneEventListenerThread, NULL, (thread_start_routine) em_handleOneListenerHandlerFunc, aHandleOneListenerData); DEBUG(" ... thread created %ld\n", oneEventListenerThread ); if( error == 0 ) { /* Add thread to wait list */ assert( threadWaitCntr < LLIST_NO_ELEMS ); threadWaitList[threadWaitCntr] = oneEventListenerThread; threadWaitCntr++; } else fprintf( stderr, "ERROR: error %d starting event thread.\n", error ); } else { /* This is a dangerous point. If the handlerFunc blocks for a long time, we still have the listenerList locked, and will prevent other events like this to be handled. Any handlers which can block should start new threads! */ DEBUG("forks NO thread for event %p handlerData %p\n", event, aListener->handlerData); aListener->handlerFunc(event->source, event->eventType, event->eventData, aListener->handlerData); DEBUG("handlerFunc returned-1. event %p handlerData %p\n", event, aListener->handlerData); } } } /* Unlock the list */ threading_mutex_unlock(&(listenerList->listenerListLock)); DEBUG(" released lock %p->listenerListLock \n", listenerList ); /* Wait for all threads in wait list */ DEBUG(" waits for joining threads for event %p\n", event ); for (listnCntr = 0; listnCntr < threadWaitCntr; listnCntr++) { /* For every pthread created, do a join */ pthread_join(threadWaitList[listnCntr], NULL); } DEBUG(" done joining threads for event %p\n", event ); finishedWithEvent( event ); #if 0 /* Check if freeFunc should be called. */ if (event->freeFunc) { event->freeFunc(event->source, event->eventType, event->eventData); } /* Free the event itself */ free(event); #endif DEBUG("leave\n"); return NULL; }