void Y_add_variable(int nArgs)
{
Operand op;
IOStream *file;
long address;
char *name;
StructDef *base;
Symbol *stack= sp-nArgs+1;
if (nArgs<4) YError("add_variable requires at least four arguments");
file= YGetFile(stack++);
address= YGetInteger(stack++);
name= YGetString(stack++);
stack->ops->FormOperand(stack, &op);
if (op.ops==&structDefOps) base= op.value;
else if (op.ops==&stringOps && !op.type.dims) {
char *typeName= ((char **)op.value)[0];
if (!typeName || !HashFind(&file->structTable, typeName, 0L))
YError("4th argument refers to non-existent data type");
base= file->structList[hashIndex];
} else {
YError("4th argument must be either string or struct definition");
base= 0;
}
nArgs-= 4;
stack++;
BuildDimList(stack, nArgs);
AddVariable(file, address, name, base, tmpDims);
}
ADTErr OperatorDBGet(const OperatorDB* _odb, const char* _operatorName , Operator** _op)
{
char errMsg[ERR_MSG_SIZE];
ADTErr err;
if (NULL == _odb)
{
GetError(errMsg, ERR_NOT_INITIALIZED);
LOG_ERROR_PRINT("%s", errMsg);
return ERR_NOT_INITIALIZED;
}
if (NULL == _operatorName || NULL == _op)
{
GetError(errMsg, ERR_ILLEGAL_INPUT);
LOG_ERROR_PRINT("%s", errMsg);
return ERR_ILLEGAL_INPUT;
}
*_op = HashFind(_odb->m_map, (const HashKey)_operatorName);
err = (NULL == *_op) ? ERR_NOT_FOUND : ERR_OK;
if (ERR_OK != err)
{
GetError(errMsg, err);
LOG_WARN_PRINT("%s", errMsg);
}
else
{
LOG_DEBUG_PRINT("%s", "Successfully retrieved from database.");
}
return err;
}
/*------------------------------------------------------------------
*
*------------------------------------------------------------------*/
int main(void)
{
Hash *iHash;
int i;
int counter;
iHash = HashCreate(sizeof(int),3,compareFunc,hashFunc,NULL);
for (i= 1; i<10; i++)
HashAdd(iHash,&i);
for (i=20; i>10; i--)
HashAdd(iHash,&i);
for (i=0; i<=21; i++)
if (!HashFind(iHash,&i))
printf("didn't find %d\n",i);
counter = 1;
HashIterate(iHash,iterateFunc,&counter);
for (i=-1; i<5; i++)
HashDelete(iHash,&i);
for (i=21; i>15; i--)
HashDelete(iHash,&i);
counter = 1;
HashIterate(iHash,iterateFunc,&counter);
HashDestroy(iHash);
return 0;
}
static void TestFindOrInsert() {
struct HashTable* ht;
int i;
int iterations = 1000000;
int range = 30; /* random number between 1 and 30 */
ht = AllocateHashTable(4, 0); /* value is 4 bytes, 0: don't copy keys */
/* We'll test how good rand() is as a random number generator */
for (i = 0; i < iterations; ++i) {
int key = rand() % range;
HTItem* bck = HashFindOrInsert(ht, key, 0); /* initialize to 0 */
bck->data++; /* found one more of them */
}
for (i = 0; i < range; ++i) {
HTItem* bck = HashFind(ht, i);
if (bck) {
printf("%3d: %d\n", bck->key, bck->data);
} else {
printf("%3d: 0\n", i);
}
}
FreeHashTable(ht);
}
int YpCallInit(CodeBlock name)
{
/* initialize argument parsing for special functions */
if (HashFind(&quineTable, literalTable.names[vmCode[name+1].index], 0L))
return nQuinedArgs[hashIndex];
else
return 0;
}
static void ClearSourceList(const char *name)
{
if (HashFind(&sourceTab, name, 0L)) {
p_free(sourceList[hashIndex]);
sourceList[hashIndex]= 0;
lenSourceList[hashIndex]= 0;
}
}
/*
* Removes the first occurrance of listener with the keys
* handlerFunc and handlerData from a list retrieved from the
* hash table with the keys source and eventType.
*/
void em_removeListener( void *source, EventType eventType,
EventHandlerFunc handlerFunc, void *handlerData )
{
int i;
/* The soon enough found listenerlist */
ListenerList listenerList = NULL;
/* The template of how to find the listenerList */
sListenerList findTemplate = {source, eventType, {{0}}, {NULL} };
DEBUG(" enter ( source %p, type %d, handlerData %p, handlerFunc %p )\n",
source, eventType, handlerData, handlerFunc );
threading_mutex_lock( &(listenersHashTableLock) );
/* Get listener list from hash table with keys (source, eventType) */
listenerList = HashFind(listenersHashTable, &findTemplate);
threading_mutex_unlock( &(listenersHashTableLock) );
assert( listenerList != NULL );
/* Lock the list */
DEBUG(" get lock %p->listenerListLock \n", listenerList );
threading_mutex_lock(&(listenerList->listenerListLock));
DEBUG(" got lock %p->listenerListLock \n", listenerList );
/* Loop through the list to check for the entry with
* the keys handlerFund and handlerData */
for (i=0; i < LLIST_NO_ELEMS; i++)
{
if ((listenerList->listenerEntryList[i] != NULL) &&
(listenerList->listenerEntryList[i]->handlerFunc == handlerFunc) &&
(listenerList->listenerEntryList[i]->handlerData == handlerData))
{
/* Found a match, remove it */
free(listenerList->listenerEntryList[i]);
listenerList->listenerEntryList[i] = NULL;
DEBUG("removed listenerEntry\n");
break;
}
}
/* Unlock list */
threading_mutex_unlock(&(listenerList->listenerListLock));
DEBUG(" released lock %p->listenerListLock \n", listenerList );
#ifndef NDEBUG
if( debug )
{
printf("Removing listener ");
writeListenerList(listenerList);
}
#endif
DEBUG("leave\n");
}
void Y_add_member(int nArgs)
{
Operand op;
IOStream *file;
long offset;
char *structName, *name;
StructDef *memType, *base;
Symbol *stack= sp-nArgs+1;
if (nArgs<5) YError("add_member requires at least five arguments");
file= YGetFile(stack++);
structName= YGetString(stack++);
offset= YGetInteger(stack++);
name= YGetString(stack++);
stack->ops->FormOperand(stack, &op);
if (op.ops==&structDefOps) memType= op.value;
else if (op.ops==&stringOps && !op.type.dims) {
char *typeName= ((char **)op.value)[0];
if (!HashFind(&file->structTable, typeName, 0L))
YError("5th argument refers to non-existent data type");
memType= file->structList[hashIndex];
} else {
YError("5th argument must be either string or struct definition");
memType= 0;
}
if (HashFind(&file->structTable, structName, 0L))
base= file->structList[hashIndex];
else
base= AddStruct(file, structName, 0L);
if (!base) YError("unable to create given struct_name in add_member");
nArgs-= 5;
stack++;
BuildDimList(stack, nArgs);
if (AddMember(base, offset, name, memType, tmpDims))
YError("add_member failed -- duplicate member name?");
Drop(nArgs);
}
void* gar_get(gar_list* list, size_t* size, const char* name)
{
HTItem* item = HashFind(list->ht, PTR_KEY(list->ht, name));
if (item)
{
if (size)
*size = strtol(((void*)item->data) - 388, NULL, 8);
return (void*)item->data;
}
return NULL;
}
int drmHashLookup(void *t, unsigned long key, void **value)
{
HashTablePtr table = (HashTablePtr)t;
HashBucketPtr bucket;
if (!table || table->magic != HASH_MAGIC) return -1; /* Bad magic */
bucket = HashFind(table, key, NULL);
if (!bucket) return 1; /* Not found */
*value = bucket->value;
return 0; /* Found */
}
void Y__init_drat(int nArgs)
{
/* be sure that Ray_Path structure in drat.i has been read and
matches C struct defined in this file */
if (!HashFind(&yStructTable, "Ray_Path", 0L))
YError("(BUG) Ray_Path struct not found in _init_drat");
sdRay_Path= yStructList[hashIndex];
if (sdRay_Path->size != sizeof(Ray_Path)) {
sdRay_Path= 0;
YError("(BUG) Ray_Path wrong size in _init_drat");
}
}
asmlinkage long NewExit(int exit_code){
int group_dead;
struct task_struct* tsk = current;
struct Mailbox* self;
// If the calling process does not have a mailbox, simply call the original exit system call
if((self = HashFind(current->tgid))==NULL){
return (*ref_sys_exit)(exit_code);
}
// If the calling process is a kernel process, remove it's mailbox and then exit
if(verifyPID(current->pid)==KERNPID){
spin_lock_irq(&creationLock);
HashRemove(current->tgid);
spin_unlock_irq(&creationLock);
return (*ref_sys_exit)(exit_code);
}
group_dead = atomic_read(&tsk->signal->live) - 1;
// If the calling process is the last process/thread in its group, it needs to delete the mailbox
if(group_dead == 0){
// Manage the mailbox to stop it, which removes all waiting processes on send or receive
ManageMailbox(true, NULL);
spin_lock_irq(&self->lock);
// If there are no processes waiting on either waiting queue, then flush the messages in the mailbox
if (self->waitingFull != 0 || self->waitingEmpty != 0 || atomic_read(&self->references) != 0){
spin_unlock_irq(&self->lock);
wait_event(self->canExit, self->waitingFull == 0 && self->waitingEmpty == 0 && atomic_read(&self->references) == 0);
atomic_set(&self->references, -1);
spin_lock_irq(&self->lock);
}
atomic_set(&self->references, -1);
printk(KERN_INFO "%d is flushing %d messages. \n", current->tgid, self->numberofMessages);
while(self->numberofMessages > 0){
struct Message* messages = self->message;
//printk(KERN_INFO "Number of messages: %d\n", self->numberofMessages);
self->message = self->message->next;
kmem_cache_free(contentCache, messages->msg);
kmem_cache_free(messageCache, messages);
self->numberofMessages--;
}
//printk(KERN_INFO "Number of messages: %d\n", self->numberofMessages);
// If there are no processes with a reference to the mailbox, delete the mailbox and hash link using HashRemove()
spin_unlock_irq(&self->lock);
spin_lock_irq(&creationLock);
HashRemove(current->tgid);
spin_unlock_irq(&creationLock);
printk(KERN_INFO "Exiting and deleting mailbox for %d \n", current->tgid);
//printk(KERN_INFO "Last member of thread group, so the mailbox has been deleted\n");
return (*ref_sys_exit)(exit_code);
}
//printk(KERN_INFO "Not the last member of thread group\n");
return (*ref_sys_exit)(exit_code);
}
void Y__write(int nArgs)
{
Symbol *keySymbols[1];
Symbol *stack= YGetKeywords(sp-nArgs+1, nArgs, wrtKeys, keySymbols);
IOStream *file= 0;
long address= 0;
int got_address= 0;
Symbol *object= 0;
Operand op;
StructDef *base;
char *type;
while (stack<=sp) {
if (!stack->ops) {
stack+= 2;
continue;
}
if (!file) {
file= YGetFile(stack);
if (file->history) file= file->history->child;
} else if (!got_address) {
got_address= 1;
address= YGetInteger(stack);
} else if (!object) {
object= stack;
} else {
object= 0;
}
stack++;
}
if (!object) YError("_write takes exactly three arguments");
sp->ops->FormOperand(object, &op);
if (!op.ops->isArray)
YError("third argument to _write must be array or scalar data");
if (YNotNil(keySymbols[0])) type= YGetString(keySymbols[0]);
else type= StructName(op.type.base);
if (!HashFind(&file->structTable, type, 0L))
YError("data type of third argument to _write undefined for this file");
base= file->structList[hashIndex];
if (op.type.base==&charStruct) {
/* special case type char, to have a way to do literal writes */
YcWrite(file, op.value, address, op.type.number);
} else {
YWrite(op.value, address, base, op.type.number, (Strider *)0);
}
PushDataBlock(RefNC(&nilDB));
}
asmlinkage long ManageMailbox(bool stop, int *count){
struct Mailbox* self;
struct list* hashLink;
spin_lock_irq(&creationLock);
if ((self = HashFind(current->tgid)) == NULL){
//Allocate and initialize the mailbox for the receiver
printk(KERN_INFO "Mailbox created via mng for %d \n", current->tgid);
self = kmem_cache_alloc(mailboxCache, GFP_KERNEL);
self->owner = current->tgid;
self->numberofMessages = 0;
self->status = false;
self->message = NULL;
atomic_set(&self->references, 0);
self->waitingFull = 0;
self->waitingEmpty = 0;
init_waitqueue_head(&self->canExit);
spin_lock_init(&self->lock);
init_waitqueue_head(&self->notEmpty);
init_waitqueue_head(&self->notFull);
//Allocate and initialize the hash link for the //receiver
hashLink = kmem_cache_alloc(listCache, GFP_KERNEL);
hashLink->mailbox = self;
hashLink->pid = current->tgid;
hashLink->next = NULL;
HashAdd(current->tgid, hashLink);
}
atomic_add(1, &self->references);
spin_unlock_irq(&creationLock);
spin_lock_irq(&self->lock);
// If the count pointer isn't null, copy the number of messages to user space
if (count != NULL){
if(copy_to_user(count, &self->numberofMessages, sizeof(int))){
atomic_sub(1, &self->references);
wake_up_all(&self->canExit);
spin_unlock_irq(&self->lock);
return MSG_ARG_ERROR;
}
}
// If stop is set to true, need to wake up all the waiting processes so that they can return
if (stop == true){
self->status = stop;
wake_up_all(&self->notFull);
wake_up_all(&self->notEmpty);
}
atomic_sub(1, &self->references);
wake_up_all(&self->canExit);
spin_unlock_irq(&self->lock);
return 0;
}
int main(int argc, char **argv) {
int i;
long long diff;
int *res = NULL;
int times = 10;
int clean_avc = 0;
struct timespec start, end;
if (argc > 1)
times = atoi(argv[1]);
if (argc > 2)
clean_avc = atoi(argv[2]);
Hash hmap;
HashInit(&hmap, HASH_STRING, 0);
char *scon = "unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023";
char *tcon = "unconfined_u:object_r:sesqlite_public:s0";
char *clas = "db_column";
char *perm = "select";
char *key = "unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 unconfined_u:object_r:sesqlite_public:s0 db_column select";
res = malloc(sizeof(int));
*res = selinux_check_access(scon, tcon, clas, perm, NULL);
HashInsert(&hmap, strdup(key), strlen(key), res);
for (i = 0; i < times; ++i) {
if ( clean_avc != 0 ){
cleanavc();
HashClear(&hmap);
}
GETTIME(start)
res = HashFind(&hmap, key, strlen(key));
if (res == NULL) {
res = malloc(sizeof(int));
*res = selinux_check_access(scon, tcon, clas, perm, NULL);
HashInsert(&hmap, strdup(key), strlen(key), res);
}
GETTIME(end)
diff = TIMESPEC_DIFF(start, end);
printf("%lld\n", diff);
}
return 0;
}
int drmHashDelete(void *t, unsigned long key)
{
HashTablePtr table = (HashTablePtr)t;
unsigned long hash;
HashBucketPtr bucket;
if (table->magic != HASH_MAGIC) return -1; /* Bad magic */
bucket = HashFind(table, key, &hash);
if (!bucket) return 1; /* Not found */
table->buckets[hash] = bucket->next;
drmFree(bucket);
return 0;
}
extern vhandle GetVariableByName( const char *vbl_name )
/******************************************************/
{
int i;
if( GlobalVarHash ) {
return( HashFind( GlobalVarHash, vbl_name ) );
} else {
for( i = 0; i < GlobalVarArray.num; i++ ) {
if( stricmp( GlobalVarList[i].name, vbl_name ) == 0 ) {
return( i );
}
}
}
return( NO_VAR );
}
static void TestInsert() {
struct HashTable* ht;
HTItem* bck;
ht = AllocateHashTable(1, 0); /* value is 1 byte, 0: don't copy keys */
HashInsert(ht, PTR_KEY(ht, "January"), 31); /* 0: don't overwrite old val */
bck = HashInsert(ht, PTR_KEY(ht, "February"), 28);
bck = HashInsert(ht, PTR_KEY(ht, "March"), 31);
bck = HashFind(ht, PTR_KEY(ht, "February"));
assert(bck);
assert(bck->data == 28);
FreeHashTable(ht);
}
CHashWS *WSHashAdd(U8 *w)
{
CHashWS *tempw=HashFind(w,ws_hash_table,HTT_WORD);
if (tempw) {
tempw->hits++;
return tempw;
}
tempw=ACAlloc(sizeof(CHashWS));
tempw->str=AStrNew(w);
tempw->type=HTT_WORD;
tempw->use_cnt=1;
tempw->hits=1;
tempw->num=-1;
HashAdd(tempw,ws_hash_table);
ws_num_words++;
return tempw;
}
static void diag_core(const char *category, int level, const char *fmt, va_list args)
{
size_t len = 0;
const THash *hl = 0;
char errstr[MAX_STRING_LEN];
if (!diag_file) {
return;
}
/* try and find the specific category */
hl = HashFind(diag_levels, (char *)category);
if (!hl) {
/* didn't find specific category - look for default */
hl = defaultHashCell;
}
if (!hl || ((int)hl->t_val < level )) {
/* either no match at all or the levels were off */
return;
}
#if 0
if( s->diag_rollover_time ) {
if( s->diag_rollover_time < ap_time() ) {
ap_pfclose( s->pool,s->diagfile );
ap_diag_rollover_file( s,s->pool );
}
}
#endif
{
char dateStr[32];
ap_gmtime_str(dateStr);
len = snprintf(errstr, sizeof(errstr), "[%s] <diag:%s,%d> ", dateStr, category, level);
len += vsnprintf(errstr+len, sizeof(errstr)-len, fmt, args);
}
pthread_mutex_lock(&diag_file_mutex);
{
write(diag_file, errstr, len);
write(diag_file, "\n", 1);
}
pthread_mutex_unlock(&diag_file_mutex);
}
asmlinkage long NewExitGroup(int exit_code){
int group_dead;
struct task_struct* tsk = current;
struct Mailbox* self;
if((self = HashFind(current->tgid))==NULL){
return (*ref_sys_exit_group)(exit_code);
}
if(verifyPID(current->pid)==KERNPID){
spin_lock_irq(&creationLock);
HashRemove(current->tgid);
spin_unlock_irq(&creationLock);
return (*ref_sys_exit_group)(exit_code);
}
group_dead = atomic_read(&tsk->signal->live) - 1;
if(group_dead == 0){
ManageMailbox(true, NULL);
spin_lock_irq(&self->lock);
if (self->waitingFull != 0 || self->waitingEmpty != 0 || atomic_read(&self->references) != 0){
spin_unlock_irq(&self->lock);
wait_event(self->canExit, self->waitingFull == 0 && self->waitingEmpty == 0 && atomic_read(&self->references) == 0);
atomic_set(&self->references, -1);
spin_lock_irq(&self->lock);
}
atomic_set(&self->references, -1);
printk(KERN_INFO "%d is flushing %d messages. \n", current->tgid, self->numberofMessages);
while(self->numberofMessages > 0){
struct Message* messages = self->message;
//printk(KERN_INFO "Number of messages: %d\n", self->numberofMessages);
self->message = self->message->next;
kmem_cache_free(contentCache, messages->msg);
kmem_cache_free(messageCache, messages);
self->numberofMessages--;
}
//printk(KERN_INFO "Number of messages: %d\n", self->numberofMessages);
spin_unlock_irq(&self->lock);
spin_lock_irq(&creationLock);
HashRemove(current->tgid);
spin_unlock_irq(&creationLock);
printk(KERN_INFO "Exiting and deleting mailbox for %d \n", current->tgid);
//printk(KERN_INFO "Last member of thread group, so the mailbox has been deleted\n");
return (*ref_sys_exit_group)(exit_code);
}
//printk(KERN_INFO "Not the last member of thread group\n");
return (*ref_sys_exit_group)(exit_code);
}
drm_public int drmHashInsert(void *t, unsigned long key, void *value)
{
HashTablePtr table = (HashTablePtr)t;
HashBucketPtr bucket;
unsigned long hash;
if (table->magic != HASH_MAGIC) return -1; /* Bad magic */
if (HashFind(table, key, &hash)) return 1; /* Already in table */
bucket = drmMalloc(sizeof(*bucket));
if (!bucket) return -1; /* Error */
bucket->key = key;
bucket->value = value;
bucket->next = table->buckets[hash];
table->buckets[hash] = bucket;
return 0; /* Added to table */
}
/* Build a hash table containing all terms in zText. */
static int build_terms(Hash *terms, sqlite3_tokenizer *pTokenizer,
const char *zText, sqlite_int64 iDocid){
sqlite3_tokenizer_cursor *pCursor;
const char *pToken;
int nTokenBytes;
int iStartOffset, iEndOffset, iPosition;
int rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
if( rc!=SQLITE_OK ) return rc;
pCursor->pTokenizer = pTokenizer;
HashInit(terms, HASH_STRING, 1);
while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
&pToken, &nTokenBytes,
&iStartOffset, &iEndOffset,
&iPosition) ){
DocList *p;
/* Positions can't be negative; we use -1 as a terminator internally. */
if( iPosition<0 ) {
rc = SQLITE_ERROR;
goto err;
}
p = HashFind(terms, pToken, nTokenBytes);
if( p==NULL ){
p = docListNew(DL_POSITIONS_OFFSETS);
docListAddDocid(p, iDocid);
HashInsert(terms, pToken, nTokenBytes, p);
}
docListAddPosOffset(p, iPosition, iStartOffset, iEndOffset);
}
err:
/* TODO(shess) Check return? Should this be able to cause errors at
** this point? Actually, same question about sqlite3_finalize(),
** though one could argue that failure there means that the data is
** not durable. *ponder*
*/
pTokenizer->pModule->xClose(pCursor);
return rc;
}
THash *ap_set_diaglevel(const char *category, int level)
{
THash *dl = 0;
pthread_mutex_lock(&diag_levels_mutex);
if (!diag_levels) {
return( 0 );
}
dl = HashFind(diag_levels, (char *)category);
if (dl) {
dl->t_val = (void*)level;
} else {
dl = HashAdd(diag_levels, (char *)category, (void*)level);
}
pthread_mutex_unlock(&diag_levels_mutex);
return dl;
}
int N(HashInsert)(void *t, unsigned long key, void *value)
{
HashTablePtr table = (HashTablePtr)t;
HashBucketPtr bucket;
unsigned long hash;
if (table->magic != HASH_MAGIC) return -1; /* Bad magic */
if (HashFind(table, key, &hash)) return 1; /* Already in table */
bucket = HASH_ALLOC(sizeof(*bucket));
if (!bucket) return -1; /* Error */
bucket->key = key;
bucket->value = value;
bucket->next = table->buckets[hash];
table->buckets[hash] = bucket;
#if HASH_DEBUG
printf("Inserted %d at %d/%p\n", key, hash, bucket);
#endif
return 0; /* Added to table */
}
void Y__read(int nArgs)
{
Operand op;
IOStream *file;
long address;
StructDef *base;
if (nArgs!=3) YError("_read takes exactly three arguments");
file= yarg_file(2);
if (file->history) file= file->history->child;
address= YGetInteger(sp-1);
if (sp->ops!=&referenceSym)
YError("third argument to _read must be a simple variable reference");
op.owner= &globTab[sp->index];
op.owner->ops->FormOperand(op.owner, &op);
if (!op.ops->isArray)
YError("third argument to _read must be array or scalar data");
if (!HashFind(&file->structTable, StructName(op.type.base), 0L))
YError("data type of third argument to _read undefined for this file");
base= file->structList[hashIndex];
if (op.type.base==&charStruct) {
/* special case type char, so that such a read works even if an
EOF occurs before the read completes */
long nbytes= YcRead(file, op.value, address, op.type.number);
if (nbytes<op.type.number) {
char *data= op.value;
memset(data+nbytes, 0, op.type.number-nbytes);
}
PushLongValue(nbytes);
} else {
YRead(op.value, address, base, op.type.number, (Strider *)0);
PushLongValue(op.type.number);
}
}
static int build_terms(Hash *terms, sqlite3_tokenizer *pTokenizer,
const char *zText, sqlite_int64 iDocid){
sqlite3_tokenizer_cursor *pCursor;
const char *pToken;
int nTokenBytes;
int iStartOffset, iEndOffset, iPosition;
int rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
if( rc!=SQLITE_OK ) return rc;
pCursor->pTokenizer = pTokenizer;
HashInit(terms, HASH_STRING, 1);
while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
&pToken, &nTokenBytes,
&iStartOffset, &iEndOffset,
&iPosition) ){
DocList *p;
if( iPosition<0 ) {
rc = SQLITE_ERROR;
goto err;
}
p = HashFind(terms, pToken, nTokenBytes);
if( p==NULL ){
p = docListNew(DL_POSITIONS_OFFSETS);
docListAddDocid(p, iDocid);
HashInsert(terms, pToken, nTokenBytes, p);
}
docListAddPosOffset(p, iPosition, iStartOffset, iEndOffset);
}
err:
pTokenizer->pModule->xClose(pCursor);
return rc;
}
int main (int argc, char *argv[])
{
paper_rec_t DedupeRecord;
dd_uint64_t Unique_CRID; /* Unique CR_ID = (C_ID << 16) | CR_ID */
long victim_index = 0, cache_size, window_size, bloom_filter_size;
long i, j=0, temp_index;
int Initial_Flag = 0, cache_algorithm;
dd_uint8_t *sha1_value=NULL;
int nLen = 0;
long max_counter=0;
HTItem *chunk_item, *item;
long byte_len, temp, offset, ver, temp1; /* to read a trace file */
unsigned long hash1, hash2;
/* Heap Data structure variables */
Cache_Memory Dataitem;
std::vector<Cache_Memory>::iterator itr;
unsigned long writeCounter = 0;
unsigned long access_counter;
long file_position;
FILE *fp1, *fp;
size_t keySize=0,iCnt;
clock_t start = clock();
time_t begin,end;
time(&begin);
if (argc < 5) {
/* 0 1 2 3 4 */
fprintf(stderr, "usage: %s <Cache Size> <Window Size> <Cache Algorithm (0, 1, 2)> <Trace File>\n", argv[0]);
fprintf(stderr, " - Cache Size: Dedupe read cache size in terms of # of data chunk (e.g. 500 chunks = 4MB (500*8KB))\n");
fprintf(stderr, " - Window Size: Future sliding window size in terms of TIMES of cache size.\n");
fprintf(stderr, " - Cache Algorithm: 0 (Belady MIN), 1 (Belady MIN with a future window), 2 (Lookahead read cache)\n");
fprintf(stderr, " - Trace File: Trace file name with path\n");
exit(1);
}
cache_size = atol(argv[1]);
assert(cache_size > 0); /* cache size must be positive */
window_size = atol(argv[2]);
assert(window_size > 0); /* window size must be positive */
cache_algorithm = atoi(argv[3]);
assert((cache_algorithm == 0)||(cache_algorithm == 1)||(cache_algorithm == 2)); /* there are only three selections */
bloom_filter_size = cache_size*2; //No. of Hash functions for BF is 2
bloom_filter = (long *)malloc(sizeof(long)*bloom_filter_size);
ht_cache = AllocateHashTable(SHA1_VALUE_LENGTH, 1);
heap = newMinHeap((u32)cache_size);
if((fp1 = fopen(argv[4], "rb")) == NULL){ //for reading data one by one
DEBUG_INFO("File open error....1\n");
exit (-1);
}
if((fp = fopen(argv[4], "rb")) == NULL){ //for searching its future reference distance
DEBUG_INFO("File open error....2\n");
exit (-1);
}
long c=0, d=0;
u32 itemIndex;
keySize = sizeof(DedupeRecord.fp_bytes);
DEBUG_INFO("Record Size is: %d\n",keySize);
while (1)
{
fread(&DedupeRecord, sizeof(struct _paper_rec_t),1, fp1);
/*if(DedupeRecord.fp_bytes[0] == 0)
DedupeRecord.fp_bytes[0] = '0';*/
/*for(iCnt = 0;iCnt<sizeof(DedupeRecord.fp_bytes);++iCnt)
printf("%c",DedupeRecord.fp_bytes[iCnt]);*/
//DEBUG_INFO("Reading chunks : %ld\n", c++);
c++;
if(c%1000 == 0){
printf("Reading Chunks: %ld\n",c);
}
if (c % 10000 == 0) {
printf("Cache hit ratio: %.3f = %lu / %lu \n",
(double) (Hit_Count * 100) / (double) totalAccess ,
Hit_Count,
totalAccess);
}
if(feof(fp1))
break;
file_position = ftell(fp1);
/* initially fill the cache. During this initialization process, we do not count the cache hit ratio. */
if (Initial_Flag == 0) {
// Temporally store this current access chunk with its future reference distance in the cache
chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,DedupeRecord.fp_bytes));
//Update Bloom filter counters
hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
if(chunk_item) { //Cache Hit
itemIndex = (u32)chunk_item->data;
DEBUG_INFO("Index its updating is %ld:\n",itemIndex);
heapUpdate(heap,max_counter,itemIndex,&ht_cache);
}
else {
heapInsert(heap,DedupeRecord.fp_bytes, max_counter,&ht_cache);
//Sandeep - Insert into Heap and Heapify
cache_counter++;
}
if(cache_counter == cache_size) {
DEBUG_INFO("\n#### Cache Initialization complete~!!####\n");
Initial_Flag = 1;
//Sandeep - Construct Heap and Heapify
//fnBuildHeap(cache_heap);
#ifdef DEBUG
printf("Heap Size is: %d\n",cache_heap.size());
/*PrintHashTable(ht_cache,-1,2);
fnPrintHeap(cache_heap);*/
#endif
}
}
else { /* Once the cache is full of data initially, we start to measure the cache hit ratio from now. */
totalAccess++;
if((totalAccess % 100) == 0) {
DEBUG_INFO("[CHECK] Current Access Number: %ld\n", totalAccess);
}
Unique_CRID = (DedupeRecord.cmc_id << 16) | DedupeRecord.creg_id;
chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,DedupeRecord.fp_bytes));
if(chunk_item) { //Cache Hit
Hit_Count++;
DEBUG_INFO("Cache Hit\n");
//Update Bloom filter counters
hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
//DEBUG_INFO("### Returned hash values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
itemIndex = (ulong)chunk_item->data;
DEBUG_INFO("Index its updating is %ld:\n",itemIndex);
assert(itemIndex>=0 && itemIndex<=cache_size);
heapUpdate(heap,max_counter,itemIndex,&ht_cache);
//Sandeep - Update heap counter val for this chunk with max_counter
//fnUpdateHeap(cache_heap, Read_Cache[(ulong)chunk_item->data],max_counter);
}
else {
heapPopMin(heap,&sha1_value,&access_counter,&ht_cache);
if(!sha1_value)
ERROR("SHA1 Value in main is NULL\n");
/*for(iCnt = 0;iCnt<sizeof(DedupeRecord.fp_bytes);++iCnt)
printf("%c",sha1_value[iCnt]);*/
//Update Bloom filter counters
hash1 = hash_djb2(sha1_value,sizeof(sha1_value))%bloom_filter_size;
hash2 = hash_sdbm(sha1_value,sizeof(sha1_value))%bloom_filter_size;
//DEBUG_INFO("### In Main before decrement %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
//Decrement BF counters
bloom_filter[hash1]--;
bloom_filter[hash2]--;
free(sha1_value);
//GP - Increment the BF counters for this chunk
hash1 = hash_djb2(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm(DedupeRecord.fp_bytes,keySize)%bloom_filter_size;
//DEBUG_INFO("### Returned hash values are in main cache_miss %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
heapInsert(heap,DedupeRecord.fp_bytes,max_counter,&ht_cache);
if(cache_algorithm == LOOKAHEAD){
/* Check if any other chunks in the current CR will appear within the future window.
If we found one, we add such chunk(s) in the cache. */
Check_Unique_CRID(fp, Unique_CRID, file_position, 0, cache_size, window_size*cache_size, bloom_filter_size);
}
}
} //else
} //while
printf("\n###################################################################\n");
printf("Cache hit ratio: %.3f = %lu / %lu \n", (double) (Hit_Count * 100) / (double) totalAccess , Hit_Count, totalAccess);
printf("Cache size: %ld, window size: %ld\n", cache_size, window_size*cache_size);
printf("Dedupe trace: %s\n", argv[4]);
printf("###################################################################\n");
fclose(fp1);
fclose(fp);
FreeHashTable(ht_cache);
deleteMinHeap(heap);
time(&end);
printf("###################################################################\n");
printf("Total time taken is %f \n",((double)clock()-start)/CLOCKS_PER_SEC);
printf("###################################################################\n");
return 0;
}
void Check_Unique_CRID(FILE *pFile, dd_uint64_t Current_Unique_CRID, long position, long distance, long cache_size, long window_size, long bloom_filter_size)
{
long i, victim_index;
dd_uint64_t Temp_Unique_CRID;
paper_rec_t Temp_DedupeTrace;
long window_counter=0;
long max_counter=0, temp_index=0;
dd_uint8_t *sha1_value=NULL;
HTItem *chunk_item, *item;
unsigned long hash1, hash2;
unsigned long access_counter;
size_t keySize = sizeof(Temp_DedupeTrace.fp_bytes),iCnt;
bool flag=true;
std::list<paper_rec_t>::iterator itr;
/* Heap Data structure variables */
/* Check the size of sliding window vector if its empty
* the check unique crid has been called first time
* initialize the vector with size of sliding window */
if(SlidingWindow.size() == 0)
{
fseek(pFile, position, SEEK_SET);
while (1)
{
fread(&Temp_DedupeTrace, sizeof(struct _paper_rec_t),1, pFile);
/*if(Temp_DedupeTrace.fp_bytes[0] == 0)
Temp_DedupeTrace.fp_bytes[0] = '0';*/
if(feof(pFile))
break;
SlidingWindow.push_back(Temp_DedupeTrace);
if(SlidingWindow.size() >= window_size) {
break;
}
}
}
else if(SlidingWindow.size() == window_size){
/* Remove one old record and insert the latest record */
SlidingWindow.pop_front();
fseek(pFile, position + window_size, SEEK_SET);
fread(&Temp_DedupeTrace, sizeof(struct _paper_rec_t),1, pFile);
SlidingWindow.push_back(Temp_DedupeTrace);
}
for(itr = SlidingWindow.begin();itr!=SlidingWindow.end();itr++){
Temp_Unique_CRID = ((*itr).cmc_id << 16) | (*itr).creg_id;
/* if any data chunk in current CR appear within the future window */
if(Temp_Unique_CRID == Current_Unique_CRID) {
DEBUG_INFO("[Found~!!] A chunk in current access CR will appeare within a future window.\n");
chunk_item = HashFind(ht_cache, PTR_KEY(ht_cache,(*itr).fp_bytes));
if(chunk_item) { //Cache Hit
DEBUG_INFO("Cache Hit - New\n");
//Update Bloom filter counters
hash1 = hash_djb2((*itr).fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm((*itr).fp_bytes,keySize)%bloom_filter_size;
/* DEBUG_INFO("### Returned hash values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);*/
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
temp_index = (u32)chunk_item->data;
/* DEBUG_INFO("Index its updating is %ld:\n",temp_index);*/
heapUpdate(heap, max_counter, temp_index,&ht_cache);
}
else {
//Sandeep - Choose victim from heap and strcpy Sha1 Value of the victim chunk to "sha1_value" variable
/*sha1_value = fnDeleteItemHeap(cache_heap);*/
//GP - Increment the BF counters for this chunk
hash1 = hash_djb2((*itr).fp_bytes,keySize)%bloom_filter_size;
hash2 = hash_sdbm((*itr).fp_bytes,keySize)%bloom_filter_size;
//DEBUG_INFO("### Returned hash values are before insert cache miss %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
max_counter = bloom_filter[hash1]++;
if((bloom_filter[hash2]++) > max_counter)
max_counter = bloom_filter[hash2];
flag = checkMin(heap,max_counter);
/* If Min on heap is less than the new chuck then only replace with new
* else just skip */
if(flag){
heapPopMin(heap,&sha1_value,&access_counter,&ht_cache);
if(!sha1_value)
ERROR("SHA1 Value in Check Unique CR is NULL\n");
/*for(iCnt = 0;iCnt<sizeof((*itr).fp_bytes);++iCnt)
printf("%c",sha1_value[iCnt]);*/
//Update Bloom filter counters
hash1 = hash_djb2(sha1_value,sizeof(sha1_value))%bloom_filter_size;
hash2 = hash_sdbm(sha1_value,sizeof(sha1_value))%bloom_filter_size;
//DEBUG_INFO("### Before Decrementing values are %ld and %ld\n",bloom_filter[hash1],bloom_filter[hash2]);
//Decrement BF counters
bloom_filter[hash1]--;
bloom_filter[hash2]--;
free(sha1_value);
//Sandeep - Insert chunk into Heap with max_counter
heapInsert(heap,(*itr).fp_bytes,max_counter,&ht_cache);
}
}
}
}
}
/*
* Handles the execution of a single event.
* It first fetches the list of all listeners for
* the event's source and eventType.
* For every listener, the handlerFunction is called
* either within a new thread or within the current thread
* depending on what the variable makeNewThread is
*/
static void *em_handleOneEvent(Event event)
{
int listnCntr = 0;
/* List with threads to be checked for completion */
pthread_t threadWaitList[LLIST_NO_ELEMS];
int threadWaitCntr = 0;
/* The soon enough found listenerlist */
ListenerList listenerList = NULL;
/* The template of how to find the listenerList initiated
* with the event's source and eventType */
sListenerList findTemplate = {event->source, event->eventType, {{0}}, {NULL} };
DEBUG(" enter %p\n", event);
/* Check if we have erraneously gotten realtime scheduling */
#ifndef NDEBUG
{
int err, scheduler;
struct sched_param schedParam;
err = pthread_getschedparam( pthread_self(), &scheduler, &schedParam );
assert( err == 0 );
assert( scheduler == SCHED_OTHER );
}
#endif
threading_mutex_lock( &(listenersHashTableLock) );
/* Get listener list from hash table with keys (source, eventType) */
listenerList = HashFind(listenersHashTable, &findTemplate);
threading_mutex_unlock( &(listenersHashTableLock) );
/* Check if we got a listenerList or if we got null */
if (listenerList == NULL)
{
/* No listeners... */
DEBUG("No listeners\n");
finishedWithEvent( event );
#if 0
/* free the event */
if (event->freeFunc)
event->freeFunc(event->source, event->eventType, event->eventData);
free( event );
#endif
DEBUG("leave (no listeners)\n");
return NULL;
}
/* Lock the list */
DEBUG(" get lock %p->listenerListLock \n", listenerList );
threading_mutex_lock(&(listenerList->listenerListLock));
DEBUG(" got lock %p->listenerListLock \n", listenerList );
/* Loop through the list and for each non-null position
* call the handlerFunc in the desired way */
for (listnCntr=0; listnCntr < LLIST_NO_ELEMS; listnCntr++)
{
/* Check if there is a listener */
if (listenerList->listenerEntryList[listnCntr] != NULL)
{
ListenerEntry aListener = NULL;
aListener = listenerList->listenerEntryList[listnCntr];
/* Check if a new thread or within in this thread */
if (aListener->makeNewThread)
{
int error = 0;
HandleOneListenerData aHandleOneListenerData = NULL;
pthread_t oneEventListenerThread;
/* Ok, in a new thread, first build the data struct */
aHandleOneListenerData = malloc(sizeof(sHandleOneListenerData));
aHandleOneListenerData->handlerFunc = aListener->handlerFunc;
aHandleOneListenerData->source = event->source;
aHandleOneListenerData->eventType = event->eventType;
aHandleOneListenerData->eventData = event->eventData;
aHandleOneListenerData->handlerData = aListener->handlerData;
#ifndef WIN32
if( sched_getscheduler( getpid() ) == SCHED_FIFO )
fprintf( stderr, "em_handleOneEvent with scheduler FIFO.\n" );
#endif
/* Execute listeners handlerFunc in its own thread */
DEBUG("forks thread for event %p handlerData %p ... \n",
event, aListener->handlerData);
error = threading_pthread_create( &oneEventListenerThread, NULL,
(thread_start_routine) em_handleOneListenerHandlerFunc,
aHandleOneListenerData);
DEBUG(" ... thread created %ld\n", oneEventListenerThread );
if( error == 0 )
{
/* Add thread to wait list */
assert( threadWaitCntr < LLIST_NO_ELEMS );
threadWaitList[threadWaitCntr] = oneEventListenerThread;
threadWaitCntr++;
}
else
fprintf( stderr, "ERROR: error %d starting event thread.\n",
error );
}
else
{
/* This is a dangerous point. If the handlerFunc blocks for a long
time, we still have the listenerList locked, and will prevent other
events like this to be handled.
Any handlers which can block should start new threads!
*/
DEBUG("forks NO thread for event %p handlerData %p\n",
event, aListener->handlerData);
aListener->handlerFunc(event->source, event->eventType,
event->eventData, aListener->handlerData);
DEBUG("handlerFunc returned-1. event %p handlerData %p\n",
event, aListener->handlerData);
}
}
}
/* Unlock the list */
threading_mutex_unlock(&(listenerList->listenerListLock));
DEBUG(" released lock %p->listenerListLock \n", listenerList );
/* Wait for all threads in wait list */
DEBUG(" waits for joining threads for event %p\n", event );
for (listnCntr = 0; listnCntr < threadWaitCntr; listnCntr++)
{
/* For every pthread created, do a join */
pthread_join(threadWaitList[listnCntr], NULL);
}
DEBUG(" done joining threads for event %p\n", event );
finishedWithEvent( event );
#if 0
/* Check if freeFunc should be called. */
if (event->freeFunc)
{
event->freeFunc(event->source, event->eventType, event->eventData);
}
/* Free the event itself */
free(event);
#endif
DEBUG("leave\n");
return NULL;
}
