index.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/errno.h>
#include <math.h>
#include <pthread.h>
#include "index.h"

// #define DEBUG
// #define LOCK
// #define GLOBAL

void rwlock_rdlock(pthread_rwlock_t *lock) {
    if (pthread_rwlock_rdlock(lock)) {
        perror("pthread_rwlock_rdlock");
        exit(1);
    } else {
#ifdef LOCK  
        printf("- read lock acquired\n");
#endif
    }
}

void rwlock_rdunlock(pthread_rwlock_t *lock) {
    if (pthread_rwlock_unlock(lock)) {
        perror("pthread_rwlock_rdunlock");
        exit(1);
    } else {
#ifdef LOCK  
        printf("- read lock released\n");
#endif
    }
}

void rwlock_wrlock(pthread_rwlock_t *lock) {
    if (pthread_rwlock_wrlock(lock)) {
        perror("pthread_rwlock_wrlock");
        exit(1);
    } else {
#ifdef LOCK  
        printf("- write lock acquired\n");
#endif
    }
}

void rwlock_wrunlock(pthread_rwlock_t *lock) {
    if (pthread_rwlock_unlock(lock)) {
        perror("pthread_rwlock_wrunlock");
        exit(1);
    } else {
#ifdef DEBUG
        printf("- write lock released\n");
#endif
    }
}

/* Copyright (C) 2002 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */

struct hashtable;

/* Example of use:
 *
 *      struct hashtable  *h;
 *      struct some_key   *k;
 *      struct some_value *v;
 *
 *      static unsigned int         hash_from_key_fn( void *k );
 *      static int                  keys_equal_fn ( void *key1, void *key2 );
 *
 *      h = create_hashtable(16, hash_from_key_fn, keys_equal_fn);
 *      k = (struct some_key *)     malloc(sizeof(struct some_key));
 *      v = (struct some_value *)   malloc(sizeof(struct some_value));
 *
 *      (initialise k and v to suitable values)
 * 
 *      if (! hashtable_insert(h,k,v) )
 *      {     exit(-1);               }
 *
 *      if (NULL == (found = hashtable_search(h,k) ))
 *      {    printf("not found!");                  }
 *
 *      if (NULL == (found = hashtable_remove(h,k) ))
 *      {    printf("Not found\n");                 }
 *
 */

/* Macros may be used to define type-safe(r) hashtable access functions, with
 * methods specialized to take known key and value types as parameters.
 * 
 * Example:
 *
 * Insert this at the start of your file:
 *
 * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value);
 * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value);
 * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value);
 *
 * This defines the functions 'insert_some', 'search_some' and 'remove_some'.
 * These operate just like hashtable_insert etc., with the same parameters,
 * but their function signatures have 'struct some_key *' rather than
 * 'void *', and hence can generate compile time errors if your program is
 * supplying incorrect data as a key (and similarly for value).
 *
 * Note that the hash and key equality functions passed to create_hashtable
 * still take 'void *' parameters instead of 'some key *'. This shouldn't be
 * a difficult issue as they're only defined and passed once, and the other
 * functions will ensure that only valid keys are supplied to them.
 *
 * The cost for this checking is increased code size and runtime overhead
 * - if performance is important, it may be worth switching back to the
 * unsafe methods once your program has been debugged with the safe methods.
 * This just requires switching to some simple alternative defines - eg:
 * #define insert_some hashtable_insert
 *
 */

/*****************************************************************************
 * create_hashtable
   
 * @name                    create_hashtable
 * @param   minsize         minimum initial size of hashtable
 * @param   hashfunction    function for hashing keys
 * @param   key_eq_fn       function for determining key equality
 * @return                  newly created hashtable or NULL on failure
 */

struct hashtable *
create_hashtable(unsigned int minsize,
                 unsigned int (*hashfunction) (void*),
                 int (*key_eq_fn) (void*,void*));

/*****************************************************************************
 * hashtable_insert
   
 * @name        hashtable_insert
 * @param   h   the hashtable to insert into
 * @param   k   the key - hashtable claims ownership and will free on removal
 * @param   v   the value - does not claim ownership
 * @return      non-zero for successful insertion
 *
 * This function will cause the table to expand if the insertion would take
 * the ratio of entries to table size over the maximum load factor.
 *
 * This function does not check for repeated insertions with a duplicate key.
 * The value returned when using a duplicate key is undefined -- when
 * the hashtable changes size, the order of retrieval of duplicate key
 * entries is reversed.
 * If in doubt, remove before insert.
 */

int 
hashtable_insert(struct hashtable *h, void *k, void *v);

#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \
int fnname (struct hashtable *h, keytype *k, valuetype *v) \
{ \
    return hashtable_insert(h,k,v); \
}

/*****************************************************************************
 * hashtable_search
   
 * @name        hashtable_search
 * @param   h   the hashtable to search
 * @param   k   the key to search for  - does not claim ownership
 * @return      the value associated with the key, or NULL if none found
 */

void *
hashtable_search(struct hashtable *h, void *k);

#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \
valuetype * fnname (struct hashtable *h, keytype *k) \
{ \
    return (valuetype *) (hashtable_search(h,k)); \
}

/*****************************************************************************
 * hashtable_remove
   
 * @name        hashtable_remove
 * @param   h   the hashtable to remove the item from
 * @param   k   the key to search for  - does not claim ownership
 * @return      the value associated with the key, or NULL if none found
 */

void * /* returns value */
hashtable_remove(struct hashtable *h, void *k);

#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \
valuetype * fnname (struct hashtable *h, keytype *k) \
{ \
    return (valuetype *) (hashtable_remove(h,k)); \
}


/*****************************************************************************
 * hashtable_count
   
 * @name        hashtable_count
 * @param   h   the hashtable
 * @return      the number of items stored in the hashtable
 */
unsigned int
hashtable_count(struct hashtable *h);


/*****************************************************************************
 * hashtable_destroy
   
 * @name        hashtable_destroy
 * @param   h   the hashtable
 * @param       free_values     whether to call 'free' on the remaining values
 */

void
hashtable_destroy(struct hashtable *h, int free_values);

/* Copyright (C) 2002, 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */


/*****************************************************************************/
struct entry
{
    void *k, *v;
    unsigned int h;
    struct entry *next;
};


struct hashtable {
    unsigned int tablelength;
    struct entry **table;
    unsigned int entrycount;
    unsigned int loadlimit;
    unsigned int primeindex;
    unsigned int (*hashfn) (void *k);
    int (*eqfn) (void *k1, void *k2);
    pthread_rwlock_t globallock;
    pthread_rwlock_t entrycountlock;
    pthread_rwlock_t *locks;
    unsigned int num_locks;
};

/*****************************************************************************/
unsigned int
hash(struct hashtable *h, void *k);

/*****************************************************************************/
/* indexFor */
static inline unsigned int
indexFor(unsigned int tablelength, unsigned int hashvalue) {
    return (hashvalue % tablelength);
};

/* Only works if tablelength == 2^N */
/*static inline unsigned int
indexFor(unsigned int tablelength, unsigned int hashvalue)
{
    return (hashvalue & (tablelength - 1u));
}
*/

/*****************************************************************************/
#define freekey(X) free(X)
/*define freekey(X) ; */


/*****************************************************************************/
/* Copyright (C) 2004 Christopher Clark <firstname.lastname@cl.cam.ac.uk> */


/*
Credit for primes table: Aaron Krowne
 http://br.endernet.org/~akrowne/
 http://planetmath.org/encyclopedia/GoodHashTablePrimes.html
*/
static const unsigned int primes[] = {
53, 97, 193, 389,
769, 1543, 3079, 6151,
12289, 24593, 49157, 98317,
196613, 393241, 786433, 1572869,
3145739, 6291469, 12582917, 25165843,
50331653, 100663319, 201326611, 402653189,
805306457, 1610612741
};
const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]);
const float max_load_factor = 0.65;

/*****************************************************************************/
struct hashtable *
create_hashtable(unsigned int minsize,
                 unsigned int (*hashf) (void*),
                 int (*eqf) (void*,void*))
{
    struct hashtable *h;
    unsigned int pindex, size = primes[0];
    /* Check requested hashtable isn't too large */
    if (minsize > (1u << 30)) return NULL;
    /* Enforce size as prime */
    for (pindex=0; pindex < prime_table_length; pindex++) {
        if (primes[pindex] > minsize) { size = primes[pindex]; break; }
    }
    h = (struct hashtable *)malloc(sizeof(struct hashtable));
    if (NULL == h) return NULL; /*oom*/
    h->table = (struct entry **)malloc(sizeof(struct entry*) * size);
    if (NULL == h->table) { free(h); return NULL; } /*oom*/
    memset(h->table, 0, size * sizeof(struct entry *));
    h->tablelength  = size;
    h->primeindex   = pindex;
    h->entrycount   = 0;
    h->hashfn       = hashf;
    h->eqfn         = eqf;
    h->loadlimit    = (unsigned int) ceil(size * max_load_factor);
    // Allocate space for fine-grained locks
    h->locks        = (pthread_rwlock_t *) malloc(sizeof(pthread_rwlock_t) * size);
    h->num_locks    = size;

    // Initialize locks
    if (pthread_rwlock_init(&h->globallock, NULL)) {
        perror("pthread_rwlock_init");
        return NULL;
    }
    if (pthread_rwlock_init(&h->entrycountlock, NULL)) {
        perror("pthread_rwlock_init");
        return NULL;
    }
    for(int i = 0; i < size; ++i) {
        if (pthread_rwlock_init(&h->locks[i], NULL)) {
            perror("pthread_rwlock_init");
            return NULL;
        }
    }
	return h;
}

/*****************************************************************************/
unsigned int
hash(struct hashtable *h, void *k)
{
    /* Aim to protect against poor hash functions by adding logic here
     * - logic taken from java 1.4 hashtable source */
    unsigned int i = h->hashfn(k);
    i += ~(i << 9);
    i ^=  ((i >> 14) | (i << 18)); /* >>> */
    i +=  (i << 4);
    i ^=  ((i >> 10) | (i << 22)); /* >>> */
    return i;
}

/*****************************************************************************/
static int
hashtable_expand(struct hashtable *h)
{
    // Acquire global write lock for entire function
    rwlock_wrlock(&h->globallock);

    /* Double the size of the table to accomodate more entries */
    struct entry **newtable;
    struct entry *e;
    struct entry **pE;
    unsigned int newsize, i, index;
    /* Check we're not hitting max capacity */
    if (h->primeindex == (prime_table_length - 1)) {
        // Release global write lock for early return
        rwlock_wrunlock(&h->globallock);
        return 0;
    }
    newsize = primes[++(h->primeindex)];

    newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize);
    if (NULL != newtable)
    {
        memset(newtable, 0, newsize * sizeof(struct entry *));
        /* This algorithm is not 'stable'. ie. it reverses the list
         * when it transfers entries between the tables */
        for (i = 0; i < h->tablelength; i++) {
            while (NULL != (e = h->table[i])) {
                h->table[i] = e->next;
                index = indexFor(newsize,e->h);
                e->next = newtable[index];
                newtable[index] = e;
            }
        }
        free(h->table);
        h->table = newtable;
    }
    /* Plan B: realloc instead */
    else 
    {
        newtable = (struct entry **)
                   realloc(h->table, newsize * sizeof(struct entry *));
        if (NULL == newtable) {
            (h->primeindex)--;
            // Release global write lock for early return
            rwlock_wrunlock(&h->globallock);
            return 0;
        }
        h->table = newtable;
        memset(newtable[h->tablelength], 0, newsize - h->tablelength);
        for (i = 0; i < h->tablelength; i++) {
            for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) {
                index = indexFor(newsize,e->h);
                if (index == i)
                {
                    pE = &(e->next);
                }
                else
                {
                    *pE = e->next;
                    e->next = newtable[index];
                    newtable[index] = e;
                }
            }
        }
    }

#ifdef DEBUG
    printf("resizing fine-grained rwlock array to %d locks.\n", newsize);
#endif
    // Realloc more rwlocks for newly resized table
    h->locks = (pthread_rwlock_t *) realloc(h->locks, sizeof(pthread_rwlock_t) * newsize);
    for(unsigned int i = h->num_locks; i < newsize; ++i) {
        if (pthread_rwlock_init(&h->locks[i], NULL)) {
            perror("pthread_rwlock_init");
            exit(1);
        }
    }
    h->num_locks = newsize;

    h->tablelength = newsize;
    h->loadlimit   = (unsigned int) ceil(newsize * max_load_factor);

    // Release global write lock
    rwlock_wrunlock(&h->globallock);

    return -1;
}

/*****************************************************************************/
unsigned int
hashtable_count(struct hashtable *h)
{
    // Use read lock for entry count
    rwlock_rdlock(&h->entrycountlock);
    unsigned int cnt = h->entrycount;
    rwlock_rdunlock(&h->entrycountlock);
    return cnt;
}

/*****************************************************************************/
int
hashtable_insert(struct hashtable *h, void *k, void *v)
{
    /* This method allows duplicate keys - but they shouldn't be used */
    unsigned int index;
    struct entry *e;

    // Use write lock for entry count increment 
    rwlock_wrlock(&h->entrycountlock);
    if (++(h->entrycount) > h->loadlimit)
    {
        rwlock_wrunlock(&h->entrycountlock);

        /* Ignore the return value. If expand fails, we should
         * still try cramming just this value into the existing table
         * -- we may not have memory for a larger table, but one more
         * element may be ok. Next time we insert, we'll try expanding again.*/
        hashtable_expand(h);
    } else {
        rwlock_wrunlock(&h->entrycountlock);
    }

    e = (struct entry *)malloc(sizeof(struct entry));
    if (NULL == e) {
        // Use write lock for entry count decrement 
        rwlock_wrlock(&h->entrycountlock);
        --(h->entrycount);
        rwlock_wrunlock(&h->entrycountlock);
        return 0;
    } /*oom*/

    // Use global read lock for hashing/index calculations
    rwlock_rdlock(&h->globallock);
    e->h = hash(h,k);
    index = indexFor(h->tablelength,e->h);
    e->k = k;
    e->v = v;
    rwlock_rdunlock(&h->globallock);

    // Use global write lock for list insertion
    // TODO: internal lock causes problems, figure out why, using global instead
    //rwlock_wrlock(&h->locks[index]);
    rwlock_wrlock(&h->globallock);
#ifdef DEBUG 
    printf("[%.8x indexer] inserting '%s' into index[%d]...\n", pthread_self(), k, index);
#endif 
    e->next = h->table[index];
    h->table[index] = e;
    rwlock_wrunlock(&h->globallock);
    // TODO: internal lock causes problems, figure out why, using global instead
    //rwlock_wrunlock(&h->locks[index]);

    return -1;
}

/*****************************************************************************/
void * /* returns value associated with key */
hashtable_search(struct hashtable *h, void *k)
{
    struct entry *e;
    unsigned int hashvalue, index;

    // Use global read lock for hashing/indexing
    rwlock_rdlock(&h->globallock);
    hashvalue = hash(h,k);
    index = indexFor(h->tablelength,hashvalue);
    rwlock_rdunlock(&h->globallock);

    // Use local read lock for searching
    rwlock_rdlock(&h->locks[index]);
    e = h->table[index];
    while (NULL != e)
    {
        /* Check hash value to short circuit heavier comparison */
        if ((hashvalue == e->h) && (h->eqfn(k, e->k))) {
            // Release local read lock for early return (key found)
            rwlock_rdunlock(&h->locks[index]);
            return e->v;
        }
        e = e->next;
    }
    // Release local read lock for late return (key not found)
    rwlock_rdunlock(&h->locks[index]);

    return NULL;
}


/*****************************************************************************/
void * /* returns value associated with key */
hashtable_remove(struct hashtable *h, void *k)
{
    /* TODO: consider compacting the table when the load factor drops enough,
     *       or provide a 'compact' method. */

    struct entry *e;
    struct entry **pE;
    void *v;
    unsigned int hashvalue, index;

    // Use global read lock for hashing/indexing
    rwlock_rdlock(&h->globallock);
    hashvalue = hash(h,k);
    index = indexFor(h->tablelength,hash(h,k));
    rwlock_rdunlock(&h->globallock);

    // Use local write lock for removal
    rwlock_wrlock(&h->locks[index]);
    pE = &(h->table[index]);
    e = *pE;
    while (NULL != e)
    {
        /* Check hash value to short circuit heavier comparison */
        if ((hashvalue == e->h) && (h->eqfn(k, e->k)))
        {
            *pE = e->next;

            // Use write lock for entry count decrement
            rwlock_wrlock(&h->entrycountlock);
            h->entrycount--;
            rwlock_wrunlock(&h->entrycountlock);

            v = e->v;
            freekey(e->k);
            free(e);

            rwlock_wrunlock(&h->locks[index]);
            return v;
        }
        pE = &(e->next);
        e = e->next;
    }
    rwlock_wrunlock(&h->locks[index]);

    return NULL;
}

struct hashtable * global_index;

/*****************************************************************************/
/* destroy */
void
hashtable_destroy(struct hashtable *h, int free_values)
{
    unsigned int i;
    struct entry *e, *f;
    struct entry **table = h->table;
    if (free_values)
    {
        for (i = 0; i < h->tablelength; i++)
        {
            e = table[i];
            while (NULL != e)
            { f = e; e = e->next; freekey(f->k); free(f->v); free(f); }
        }
    }
    else
    {
        for (i = 0; i < h->tablelength; i++)
        {
            e = table[i];
            while (NULL != e)
            { f = e; e = e->next; freekey(f->k); free(f); }
        }
    }

    // Destroy locks
    if (pthread_rwlock_destroy(&h->globallock)) {
        perror("pthread_rwlock_destroy");
    }
    if (pthread_rwlock_destroy(&h->entrycountlock)) {
        perror("pthread_rwlock_destroy");
    }
    for(int i = 0; i < h->tablelength; ++i) {
        if (pthread_rwlock_destroy(&h->locks[i])) {
            perror("pthread_rwlock_destroy");
        }
    }
    free(h->locks);
    free(h->table);
    free(h);
}

/*
 * Copyright (c) 2002, Christopher Clark
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 * 
 * * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 * 
 * * Neither the name of the original author; nor the names of any contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 * 
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/


/*
 * Copyright (c) 2002, Christopher Clark
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 * 
 * * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 * 
 * * Neither the name of the original author; nor the names of any contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 * 
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

struct list_head {
	struct list_head *next, *prev;
};
#define LIST_POISON1  ((void *) 0x00100100)
#define LIST_POISON2  ((void *) 0x00200200)

#define LIST_HEAD_INIT(name) { &(name), &(name) }

#define LIST_HEAD(name) \
	struct list_head name = LIST_HEAD_INIT(name)

#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
/**
 * container_of - cast a member of a structure out to the containing structure
 * @ptr:	the pointer to the member.
 * @type:	the type of the container struct this is embedded in.
 * @member:	the name of the member within the struct.
 *
 */
#define container_of(ptr, type, member) ({			\
        const typeof( ((type *)0)->member ) *__mptr = (ptr);	\
        (type *)( (char *)__mptr - offsetof(type,member) );})

static inline void INIT_LIST_HEAD(struct list_head *list)
{
	list->next = list;
	list->prev = list;
}

/*
 * Insert a new entry between two known consecutive entries.
 *
 * This is only for internal list manipulation where we know
 * the prev/next entries already!
 */
static inline void __list_add(struct list_head *new,
			      struct list_head *prev,
			      struct list_head *next)
{
	next->prev = new;
	new->next = next;
	new->prev = prev;
	prev->next = new;
}

static inline void list_add(struct list_head *new, struct list_head *head)
{
	__list_add(new, head, head->next);
}

static inline void list_add_tail(struct list_head *new, struct list_head *head)
{
	__list_add(new, head->prev, head);
}

/*
 * Delete a list entry by making the prev/next entries
 * point to each other.
 *
 * This is only for internal list manipulation where we know
 * the prev/next entries already!
 */
static inline void __list_del(struct list_head * prev, struct list_head * next)
{
	next->prev = prev;
	prev->next = next;
}

/**
 * list_del - deletes entry from list.
 * @entry: the element to delete from the list.
 * Note: list_empty on entry does not return true after this, the entry is
 * in an undefined state.
 */
static inline void list_del(struct list_head *entry)
{
	__list_del(entry->prev, entry->next);
	entry->next = LIST_POISON1;
	entry->prev = LIST_POISON2;
}

/**
 * list_entry - get the struct for this entry
 * @ptr:	the &struct list_head pointer.
 * @type:	the type of the struct this is embedded in.
 * @member:	the name of the list_struct within the struct.
 */
#define list_entry(ptr, type, member) \
	container_of(ptr, type, member)

/**
 * list_for_each	-	iterate over a list
 * @pos:	the &struct list_head to use as a loop counter.
 * @head:	the head for your list.
 */
#define list_for_each(pos, head) \
	for (pos = (head)->next; pos != (head); \
        	pos = pos->next)


/**
 * list_for_each_safe	-	iterate over a list safe against removal of list entry
 * @pos:	the &struct list_head to use as a loop counter.
 * @n:		another &struct list_head to use as temporary storage
 * @head:	the head for your list.
 */
#define list_for_each_safe(pos, n, head) \
	for (pos = (head)->next, n = pos->next; pos != (head); \
		pos = n, n = pos->next)

/**
 * list_empty - tests whether a list is empty
 * @head: the list to test.
 */

static inline int list_empty(const struct list_head *head)
{
	return head->next == head;
}


#define MAX_LINES 124

typedef struct index_instance_s {
  struct list_head next;  
  int line_numbers[MAX_LINES];
  int next_free;
  char file_name[MAXPATH+1];
} index_instance_t;

typedef struct index_element_s {
  struct list_head instances;
} index_element_t;


static unsigned int hash_from_key_fn( void *k )
{
  char * str = (char *) k;
  unsigned int hash = 5381;
  int c;

  while ((c = *str++))
    hash = ((hash << 5) + hash) + c; /* hash * 33 + c */
  
  return hash;
}

static int keys_equal_fn ( void *key1, void *key2 )
{
  return(!strcmp((char *) key1, (char *) key2));
}

int init_index()
{
 global_index = create_hashtable(1024, hash_from_key_fn, keys_equal_fn);
 if (global_index == NULL) {
   return(-1);
 } else {
   return(0);
 }
}

int insert_into_index(char * word, char * file_name, int line_number)
{
  index_element_t * old_value;
  index_element_t * new_value = NULL;
  char * new_word = NULL;
  index_instance_t * instance = NULL;
  int error = 0;
  int ret;

  old_value = (index_element_t *) hashtable_search(global_index, word);
  if (old_value == NULL) {
    //
    // Copy the word
    //

    new_word = strdup(word);
    if (new_word == NULL) {
      error = -ENOMEM;
      goto Cleanup;
    }

    new_value = (index_element_t *) calloc(sizeof(index_element_t), 1);
    if (new_value == NULL) {
      error = -ENOMEM;
      goto Cleanup;
    }
    INIT_LIST_HEAD(&new_value->instances);

    //
    // Create a new instance in and put it in the value list
    //

    instance = (index_instance_t *) calloc(sizeof(index_instance_t), 1);
    strncpy(instance->file_name, file_name, MAXPATH);
    instance->line_numbers[instance->next_free++] = line_number;
    list_add(&instance->next, &new_value->instances);
    ret = hashtable_insert(global_index, new_word, new_value);
    if (!ret) {
      error = -ENOMEM;
      goto Cleanup;
    }
  } else {
    struct list_head * elem;
    index_instance_t * old_instance;

    //
    // We have the word in the index. Look for a matching filename
    //
    list_for_each(elem, &old_value->instances) {
      old_instance = list_entry(elem, index_instance_t, next);
      if ((strcmp(old_instance->file_name, file_name) == 0) &&
	  (old_instance->next_free != MAX_LINES)) {
	old_instance->line_numbers[old_instance->next_free++] = line_number;
	goto Cleanup;
      }
    }
    //
    // Allocate a new instance
    //
    instance = (index_instance_t *) calloc(sizeof(index_instance_t), 1);
    strncpy(instance->file_name, file_name, MAXPATH);
    instance->line_numbers[instance->next_free++] = line_number;
    list_add(&instance->next, &old_value->instances);
  }
 Cleanup:
  if (error < 0) {
    if (new_word != NULL) 
      free(new_word);
    if (instance != NULL)
      free(instance);
    if (new_value != NULL)
      free(new_value);
  }
  return(error);
}
  
index_search_results_t * find_in_index(char * word)
{
  index_search_results_t * results = NULL;
  int num_results = 0;
  index_element_t * element;
  element = hashtable_search(global_index, word);
  if (element != NULL) {
    struct list_head * next;
    int i;
    index_instance_t * instance;
    list_for_each(next, &element->instances) {
      instance = list_entry(next, index_instance_t, next);
      for (i = 0; i < instance->next_free; i++) {
	num_results++;
      }
    }

    results = (index_search_results_t *) calloc(sizeof(index_search_results_t) +
						(num_results - 1) * sizeof(index_search_elem_t), 1);
    if (results != NULL) {

      list_for_each(next, &element->instances) {
	instance = list_entry(next, index_instance_t, next);
	for (i = 0; i < instance->next_free; i++) {
	  strcpy(results->results[results->num_results].file_name, instance->file_name);
	  results->results[results->num_results].line_number = instance->line_numbers[i];
	  results->num_results++;
	}
      }
    }   
  }
  return(results);
}