static int cmd_memorymap(int argc, char *argv[])
/*
* function definition for memory map command. Calls memmoryMap() of memory
* module to print current state of memory.
* Parameters:
* o int argc- number of arguments
* o char *argv[] - array of arguments
* Return : 0 if successful, otherwise 1
* sets my_errno to TOO_MANY_ARGUMENTS if more than one argument is passed
*/
{
if(argc > 1)//no arguments allowed, first argument is command name
{
my_errno = TOO_MANY_ARGUMENTS;
return 1;
}
memoryMap();
return 0;
}
int main()
{
unsigned int size;;
init_memory(ONE_MEGA_BYTE);
size = 4231;
fprintf(stdout,"\nTest Case# 1: Interleaved memory allocacations/deallocations and print memory map");
fprintf(stdout,"\n----------------------------------------------------------------------------------\n\n");
void* ptr1 = myMalloc(size);
fprintf(stdout,"Allocated memory of size %d at %p\n",size,ptr1);
size = 10487;
void* ptr2 = myMalloc(size);
fprintf(stdout,"Allocated memory of size %d at %p\n",size,ptr2);
size = 75934;
void* ptr3 = myMalloc(size);
fprintf(stdout,"Allocated memory of size %d at %p\n",size,ptr3);
size = 532432;
void* ptr4 = myMalloc(size);
fprintf(stdout,"Allocated memory of size %d at %p\n",size,ptr4);
size = 343253;
void* ptr5 = myMalloc(size);
fprintf(stdout,"Allocated memory of size %d at %p\n",size,ptr5);
myFree(ptr2);
fprintf(stdout,"Deallocated memory %p\n",ptr2);
myFree(ptr4);
fprintf(stdout,"Deallocated memory %p\n",ptr4);
size = 65762;
void* ptr6 = myMalloc(size);
fprintf(stdout,"Allocated memory of size %d at %p\n",size,ptr6);
size = 43234;
void* ptr7 = myMalloc(size);
fprintf(stdout,"Allocated memory of size %d at %p\n",size,ptr7);
myFree(ptr6);
fprintf(stdout,"Deallocated memory %p\n",ptr6);
size = 8364;
void* ptr8 = myMalloc(size);
fprintf(stdout,"Allocated memory of size %d at %p\n",size,ptr8);
fprintf(stdout,"\nPrinting memory map after test case# 1\n\n");
memoryMap();
fprintf(stdout,"\nTest Case# 2: Allocating memory with size in hexadecimal (0xFFFF)");
fprintf(stdout,"\n------------------------------------------------------------------\n\n");
size = 0xFFFF;
void* ptr9 = myMalloc(size);
fprintf(stdout,"Allocated memory of size %d at %p\n",size,ptr9);
fprintf(stdout,"\nPrinting memory map after test case# 2\n\n");
memoryMap();
fprintf(stdout,"\nTest Case# 3: Freeing not a previously allocated memory(0xFFFFFFFF)");
fprintf(stdout,"\n--------------------------------------------------------\n\n");
myFree((void*)0xFFFFFFFF);
fprintf(stderr,"%s\n",my_strerr(my_errno));
fprintf(stdout,"\nTest Case# 4: Freeing a null pointer");
fprintf(stdout,"\n-------------------------------------------\n\n");
myFree(0);
fprintf(stderr,"%s\n",my_strerr(my_errno));
fprintf(stdout,"\nTest Case# 5: Allocating zero sized memory");
fprintf(stdout,"\n-------------------------------------------\n\n");
void* ptr10 = myMalloc(0);
fprintf(stderr,"%p\n",ptr10);
fprintf(stdout,"\nTest Case# 6: Allocating memory of a very big size(%d bytes)",ONE_MEGA_BYTE);
fprintf(stdout,"\n-------------------------------------------\n\n");
myMalloc(ONE_MEGA_BYTE);
fprintf(stderr,"%s\n",my_strerr(my_errno));
fprintf(stdout,"\nPrinting memory map after all test cases\n\n");
memoryMap();
return 0;
}
// MAIN
int main(int argc, const char *argv[]) {
// Stat tracking variables
unsigned char max_bucket_size;
unsigned int *num_buckets_of_size;
signed long min_offset = 0;
signed long max_offset = 0;
// File and array variables
unsigned long size;
tinybucket *itable;
unsigned int *buckets;
bucket *unique_indices;
// Loop variables
unsigned int h;
unsigned int index;
signed long offset;
// STAGE 0 - Setup
///////////////////////////////////////////////////////////////////
stage = 0;
signal(SIGINT, checkStatus);
if (argc < 4) {
perror("Error: Invalid number of arguments.\n" \
"Try: ./createHashTable ./human.bin ./unique.bin ./output.hashtable\n");
exit(1);
}
FILE *output_fp = fopen(argv[3], "wb");
if (output_fp == NULL) {
perror("Error: Invalid output filename.\n");
exit(1);
}
// memory map the genome
genome = memoryMap(argv[1], &size);
// Open unique.bin
unique_indices = (bucket *) memoryMap(argv[2], &size);
number_of_entries = size / sizeof(bucket);
unsigned char bitlength = 0;
while ((1LL << bitlength) < number_of_entries) bitlength++;
mask = 0xFFFFFFFFFFFFFFFFLL;
mask = mask >> (57 - bitlength);
printf("There are %u entries. (%u bits of data)\n", number_of_entries, bitlength);
// Allocate space for the intermediate table
printf("Attempting to allocate space for table(s)\n");
itable = calloc(number_of_entries, sizeof(tinybucket)); // calloc allocates and clears
htable = calloc(number_of_entries / 8 + 1, sizeof(char)); // could compress to a bit array
if (itable == NULL || htable == NULL) {
perror("Error: insufficient memory\n");
exit(1);
}
printf("Done allocating\n");
// STAGE 1 - 1st Pass; hash every key and see which spaces in the hash
// table will have collisions
/////////////////////////////////////////////////////////////////////////
stage = 1;
max_bucket_size = 0;
printf("Starting 1st pass\n");
for (count = 0; count < number_of_entries; count++) {
getWindowAtIndex(genome, key, unique_indices[count].index);
h = hash(key, 0, number_of_entries, mask);
itable[h].index++;
if (itable[h].index > max_bucket_size) {
max_bucket_size = itable[h].index;
}
}
printf("Finished 1st pass. Max bucket size = %d\n", max_bucket_size);
// STAGE 2 - Count bucket sizes
///////////////////////////////////////////////////////////////////
stage = 2;
num_buckets_of_size = calloc(max_bucket_size + 1, sizeof(unsigned int));
for (count = 0; count < number_of_entries; count++) {
char size = itable[count].index;
num_buckets_of_size[size]++;
}
for (count = 0; count <= max_bucket_size; count++) {
printf("Size %ld : Count %d\n", count, num_buckets_of_size[count]);
}
// STAGE 3 - Sort the keys by bucket size by outputing them to files
//////////////////////////////////////////////////////////////////////////
stage = 3;
unsigned int bucket_size;
char filename[200];
FILE **temp_files = malloc(max_bucket_size * sizeof(FILE *));
printf("Starting sorting...\n");
for (bucket_size = 1; bucket_size <= max_bucket_size; bucket_size++) {
sprintf(filename, TEMP "buckets%d.bin", bucket_size);
temp_files[bucket_size - 1] = fopen(filename, "wb");
}
for (count = 0; count < number_of_entries; count++) {
index = unique_indices[count].index;
getWindowAtIndex(genome, key, index);
h = hash(key, 0, number_of_entries, mask);
unsigned char size = itable[h].index;
if (size > 1) {
fwrite(&index, sizeof(unsigned int), 1, temp_files[size - 1]);
}
}
for (bucket_size = 1; bucket_size <= max_bucket_size; bucket_size++) {
fclose(temp_files[bucket_size - 1]);
}
printf("Finished sorting...\n");
// STAGE 4 - For each bucket of size = X, update the itable to give an
// index into the buckets[] array.
//////////////////////////////////////////////////////////////////////////
stage = 4;
printf("Assigning indices for all buckets\n");
unsigned int *indices = calloc(max_bucket_size + 1, sizeof(unsigned int));
for (count = 0; count < number_of_entries; count++) {
register unsigned char size = itable[count].index;
if (size > 2) {
itable[count].index = indices[size];
indices[size] += size;
} else if (size == 2) {
itable[count].index = -1; // value indicating empty for size 2 buckets
} else if (size == 1) {
itable[count].offset = 1; // set flag
}
}
// STAGE 5 - Handle the large buckets first (size > 2); reseed
//////////////////////////////////////////////////////////////////////////
stage = 5;
FILE *current_buckets;
for (bucket_size = max_bucket_size; bucket_size > 2; bucket_size--) {
printf("Begin processing buckets of size = %d\n", bucket_size);
sprintf(filename, TEMP "buckets%d.bin", bucket_size);
current_buckets = fopen(filename, "rb");
size = bucket_size * num_buckets_of_size[bucket_size] * sizeof(unsigned int);
printf("Allocate memory for buckets (%lu bytes)\n", size);
buckets = malloc(size);
// For each key, place into a bucket in the buckets[]
printf("Processing...\n");
for (count = 0; fread(&index, sizeof(unsigned int), 1, current_buckets); count++) {
getWindowAtIndex(genome, key, index);
h = hash(key, 0, number_of_entries, mask);
buckets[itable[h].index] = index;
itable[h].index++;
if (itable[h].index % bucket_size == 0) {
// Bucket is full of genome indices! Process it
unsigned int *full_bucket = &buckets[(itable[h].index - bucket_size)];
// Returns a good seed value
itable[h].ivalue = processBucket(full_bucket, bucket_size);
}
}
free(buckets);
fclose(current_buckets);
}
// STAGE 6 - Handle buckets of size = 2; reseed
//////////////////////////////////////////////////////////////////////////
stage = 6;
unsigned int full_bucket[2];
current_buckets = fopen(TEMP "buckets2.bin", "rb");
printf("Begin processing buckets of size = 2\n");
for (count = 0; fread(&index, sizeof(unsigned int), 1, current_buckets); count++) {
getWindowAtIndex(genome, key, index);
h = hash(key, 0, number_of_entries, mask);
if (itable[h].index == -1) {
itable[h].index = index;
} else {
// Bucket is full! Process it
full_bucket[0] = itable[h].index;
full_bucket[1] = index;
// Returns a good seed value
itable[h].ivalue = processBucket(full_bucket, 2);
}
}
fclose(current_buckets);
// STAGE 6 - Handle buckets of size = 1; displace
//////////////////////////////////////////////////////////////////////////
stage = 6;
current_buckets = fopen(TEMP "buckets1.bin", "rb");
long smallest_index = 0;
printf("Displace buckets of size = 1\n");
for (count = 0; count < number_of_entries; count++) {
if (itable[count].offset != 0) {
// This is a bin that only had one key fall in it.
// Look for an empty spot in the htable
while (getBit(htable, smallest_index) != 0) {
smallest_index++;
}
offset = smallest_index - count;
if (offset > max_offset) max_offset = offset;
if (offset < min_offset) min_offset = offset;
itable[count].ivalue = offset;
setBit(htable, smallest_index);
}
}
fclose(current_buckets);
printf("Intermediate Table Created (still in memory). Statistics:\n");
printf("Offset range: %ld [%ld, %ld]\n", max_offset - min_offset, min_offset, max_offset);
printf("Largest seed: %u\n", max_seed);
// STAGE 7 - Add the values to the hash table
//////////////////////////////////////////////////////////////////////////
stage = 7;
printf("Adding genome indices and count values to intermediate table\n");
for (count = 0; count < number_of_entries; count++) {
getWindowAtIndex(genome, key, unique_indices[count].index);
h = hash(key, 0, number_of_entries, mask);
if (itable[h].offset) {
h = h + itable[h].ivalue;
} else {
h = hash(key, itable[h].ivalue, number_of_entries, mask);
}
itable[h].index = unique_indices[count].index;
itable[h].size = unique_indices[count].size - 1;
if (unique_indices[count].size > MAX_COUNT) {
itable[h].size = MAX_COUNT;
}
}
// Stage 8 - Write MPH Table
//////////////////////////////////////////////////////////////////////////
stage = 8;
printf("Dumping table to disk!\n");
fwrite(itable, number_of_entries, sizeof(tinybucket), output_fp);
// Stage 9 - Clean up
//////////////////////////////////////////////////////////////////////////
stage = 9;
printf("Done!\n");
free(itable);
free(num_buckets_of_size);
}
