int main (int argc, char *argv[]) {
int hashtable_size, i;
char value[20];
char* element;
HashTable *hashTable;
hashtable_size = 10;
hashTable = createHashtable(10);
for (i = 0; i < 20; i++) {
sprintf(value, "string%i", i);
addElementToHashtabel(hashTable, value, strlen(value));
}
while (1) {
element = nextHashTableElement(hashTable);
if (element == NULL) {
break;
}
printf("%s\n", (char*)element);
}
if (findElementInHashtable(hashTable, value, strlen(value)) == 1) {
printf("Elementul %s se afla in hashTable.\n", value);
} else {
printf("Elementul %s nu se afla in hashTable.\n", value);
}
return 0;
}
/**
* Creates a new Aggregator. Do not forget to set the callback functions, then call @c startAggregator().
* @param ruleFile filename of file containing a set of rules
* @param minBufferTime TODO
* @param maxBufferTime TODO
* @return handle to the Aggregator on success or NULL on error
*/
IpfixAggregator* createAggregator(char* ruleFile, uint16_t minBufferTime, uint16_t maxBufferTime)
{
int i;
IpfixAggregator* ipfixAggregator = (IpfixAggregator*)malloc(sizeof(IpfixAggregator));
ipfixAggregator->rules = parseRulesFromFile(ruleFile);
if (!ipfixAggregator->rules) {
msg(MSG_FATAL, "Aggregator: could not parse rules file %s", ruleFile);
return NULL;
}
Rules* rules = ipfixAggregator->rules;
for (i = 0; i < rules->count; i++) {
rules->rule[i]->hashtable = createHashtable(rules->rule[i],
minBufferTime,
maxBufferTime
);
}
if (pthread_mutex_init(&ipfixAggregator->mutex, NULL) != 0) {
msg(MSG_FATAL, "Could not init mutex");
}
if (pthread_mutex_lock(&ipfixAggregator->mutex) != 0) {
msg(MSG_FATAL, "Could not lock mutex");
}
msg(MSG_INFO, "Aggregator: Done. Parsed %d rules; minBufferTime %d, maxBufferTime %d",
rules->count, minBufferTime, maxBufferTime
);
return ipfixAggregator;
}
/**
* initializes aggregator module and creates hashtable
* @param rules rules to use for creation of hashtables
* @param minBufferTime minimum buffer time for flows in hashtable
* @param maxBufferTime maximum buffer time for flows in hashtable
*/
void BaseAggregator::buildAggregator(Rules* rules, uint16_t minBufferTime, uint16_t maxBufferTime)
{
this->rules = rules;
for (size_t i = 0; i < rules->count; i++) {
rules->rule[i]->initialize();
rules->rule[i]->hashtable = createHashtable(rules->rule[i], minBufferTime, maxBufferTime);
}
msg(MSG_INFO, "Done. Parsed %d rules; minBufferTime %d, maxBufferTime %d", rules->count, minBufferTime, maxBufferTime);
}
/**
* Starts the Aggregator. Exporter must be running.
*/
void startAggregator(char* ruleFile, uint16_t minBufferTime, uint16_t maxBufferTime) {
int i;
if (!ipfixSender) {
fatal("Exporter not started");
return;
}
if (rules) {
fatal("Concentrator already started");
return;
}
debug("reading ruleset and creating hashtables");
rules = parseRulesFromFile(ruleFile);
for (i = 0; i < rules->count; i++) {
rules->rule[i]->hashtable = createHashtable(rules->rule[i], minBufferTime, maxBufferTime);
setNewDataTemplateCallback(rules->rule[i]->hashtable, sndNewDataTemplate);
setNewDataDataRecordCallback(rules->rule[i]->hashtable, sndDataDataRecord);
setNewDataTemplateDestructionCallback(rules->rule[i]->hashtable, sndDestroyDataTemplate);
}
}
void tupleIdBitmapCreatePageTable(TupleIdBitmap* tupleIdBitmap)
{
int hashTableSize = TupleIdBitmapHashtableStartSize;
int hashTableFlags = HashElement | HashBlobs;
SHashtableSettings settings;
memset(&settings, 0, sizeof(HashtableSettings));
settings.keyLen = sizeof(BlockNumber);
settings.valLen = sizeof(PageTableEntry);
tupleIdBitmap->pageTable = createHashtable("TupleIdBitmap", hashTableSize, &settings, hashTableFlags);
if (tupleIdBitmap->status == TupleIdBitmapOnePage)
{
Bool hashTableItemFound;
PageTableEntry* page = hashInsert(tupleIdBitmap->pageTable, tupleIdBitmap->onePageEntry.blockNumber, &hashTableItemFound);
memcpy(page, &tupleIdBitmap->onePageEntry, sizeof(PageTableEntry));
}
tupleIdBitmap->status = TupleIdBitmapHash;
}
void compress()
{
Hashtable *ht = createHashtable();
linkedList *charBits = newLinkedList();
FILE *file = fopen("file.txt", "r");
FILE *backp = fopen("backp.txt","w");
FILE *backp2 = fopen("backp2.txt","w");
PriorityQueue * queue = createPriorityQueue();
int characters[256],i,count=0,j,count2=0;
unsigned char currentChar;
memset(characters,0,sizeof(characters));
if (file == NULL)
{
printf ("ERROR 404: FILE NOT FOUND\n");
exit(0);
}
else
{
do
{
currentChar = fgetc(file);
if(currentChar == UNSIGNED_EOF)
{
break;
}
characters[currentChar]++;
count2++;
}
while(currentChar != UNSIGNED_EOF);
}
for(i=0; i < UNSIGNED_EOF; i++)
{
if(characters[i]!=0)
{
count++;
enqueue(queue,i,characters[i],NULL);
}
}
for(j = count ; j>1 ; j--)
{
dequeue(queue);
}
generateBits(ht, charBits, returnBT(queue));
freeBT(returnBT(queue));
rewind(file);
do
{
currentChar = fgetc(file);
if(currentChar == UNSIGNED_EOF)
{
break;
}
fprintf(backp,"%s",get(ht,currentChar));
}
while(currentChar != UNSIGNED_EOF);
fprintf(backp2,"%d\n",count); // SALVA QUANTAS LINHAS DE HASH TERÁ NO ARQUIVO
for(i=0; i < UNSIGNED_EOF; i++)
{
if(characters[i]!=0)
{
fprintf(backp2,"%c %s\n",i,get(ht,i)); //SALVA QUAL CARACTERE E O SEU CODIGO ASCII REDUZIDO NA HASH
}
}
fprintf(backp2,"%d\n",count2); // SALVA A QUANTIDADE DE CHARS QUE EXISTIRÃO
fclose(backp);
writeCompressedData(backp2);
fclose(backp2);
fclose(file);
rename("backp2.txt","compressed.txt");
remove("backp.txt");
}
void decompress()
{
FILE * file = fopen("compressed.txt","r");
if (file == NULL)
{
printf ("ERROR 404: FILE NOT FOUND\n");
exit(0);
}
int htLines,i,nmbChar,j;
unsigned char Char;
char bits[MAX_SIZE_BINARY_ARRAY];
Hashtable * ht = createHashtable();
unsigned Char2;
char binaryText[MAX_SIZE_BINARY_ARRAY];
fscanf(file,"%d",&htLines);//QUANTAS LINHAS SERÃO INSERIDAS NA HASH
fgetc(file);
for(i=0;i<htLines;++i)
{
Char = fgetc(file);
fgetc(file); // PULAR O ESPAÇO ENTRE O CARACTER E OS BITS
fgets(bits,MAX_SIZE_BINARY,file);
bits[strlen(bits) - 1] = '\0'; // REMOVENDO \N QUE O FGETS COLOCA.
put(ht,Char,Char,bits,0);
}
fscanf(file,"%d",&nmbChar);//PEGA QUANTOS CARACTERES TEM NA COMPRESSÃO
fgetc(file);
FILE* decomp = fopen("decomp.txt","w");
do
{
Char = fgetc(file);
if(Char == UNSIGNED_EOF)
{
break;
}
char binaryText[MAX_SIZE_BINARY_ARRAY];
int * binary;
binary = intToBin(Char);
for(i=0;i<8;i++)
{
binaryText[i] = binary[i] + 48;
}
binaryText[8] = '\0';
fprintf(decomp,"%s",binaryText);
}
while(Char != UNSIGNED_EOF);
fclose(decomp);
decomp = fopen("decomp.txt","r");
FILE * decompressed = fopen("decompressed.txt","w");
for(i=0;i<nmbChar;)
{
for(j=0;j<MAX_SIZE_BINARY;j++)
{
binaryText[j] = fgetc(decomp);
binaryText[j+1] = '\0';
unsigned char charByBit = getByBits(ht,binaryText);
if(charByBit != UNSIGNED_EOF)
{
i++;
fprintf(decompressed,"%c",charByBit);
break;
}
}
}
removeht(ht);
fclose(decompressed);
fclose(file);
fclose(decomp);
remove("decomp.txt");
}
int main(int argc, char *argv[]) {
clock_t startTime, endTime;
startTime = clock();
int p, my_rank;
/* initialize MPI stuff */
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD,&p);
MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
srand(time(NULL));
int row_num, nz, col_num, i;
FILE *fp;
fp = fopen("crs48x48.txt", "r");
fscanf(fp, "%d", &nz);
while (fgetc(fp) != '\n');
fscanf(fp, "%d", &row_num);
while (fgetc(fp) != '\n');
fscanf(fp, "%d", &col_num);
while (fgetc(fp) != '\n');
printf("%d => NZ = %d\n",my_rank, nz);
FILE *fpseed;
int seed[p];
//int *column_partition = (int *)malloc(sizeof(int)*col_num);
int *column_ptr;
int *hash_weights;
int num_cols_per_process[p];
int *YPartition = (int*)calloc(row_num, sizeof(int));
int *YmaxPartition = (int*)calloc(row_num, sizeof(int));
const int nitems = 2;
int blocklengths[2] = {1, 1};
MPI_Datatype types[2] = {MPI_INT, MPI_INT};
MPI_Datatype mpi_pair;
MPI_Aint offsets[2];
offsets[0] = offsetof(pair, col);
offsets[1] = offsetof(pair, nz);
MPI_Type_create_struct(nitems, blocklengths, offsets, types, &mpi_pair);
MPI_Type_commit(&mpi_pair);
//printf("datatype created\n");
pair A_partition_column[p];
pair *A_partition[p];
pair *my_columns;
column_ptr = (int *)malloc(sizeof(int) * (col_num+1));
// I need how many non-zeros in each column in the matrix data
for (i=0; i <= col_num; i++) {
fscanf(fp, "%d", &column_ptr[i]);
while (fgetc(fp) != '\n');
}
//column_ptr[i] = nz;
if (my_rank == 0) {
fpseed = fopen("seed48x48.txt", "r");
for(i=0; i<p; i++)
{
fscanf(fpseed, "%d\n", &seed[i]);
printf("seed[%d]: %d\n", i, seed[i]);
}
fclose(fpseed);
int i;
int prime_arr[prime_arr_len] = { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97,
101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181,
191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281,
283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379, 383, 389, 397,
401, 409, 419, 421, 431, 433, 439, 443, 449, 457, 461, 463, 467, 479, 487, 491, 499, 503,
509, 521, 523, 541
};
hash_weights = (int *)malloc(sizeof(int)*row_num);
genHashWeightsArr(hash_weights, row_num, prime_arr);
/*for (i=0; i<row_num; i++) {
printf("hashweights[%d]: %d\n",i, hash_weights[i]);
}*/
int *current_column_rows = (int *)malloc(sizeof(int)*row_num);
//printf("check 1\n");
HASHTABLE hash_columns;
hash_columns = createHashtable(p, row_num);
// read row_arr and insert in the hashtable for each column
int j,c, flag;
//insert seed cols
for (c=0; c<p; c++) {
j = seed[c];
int nz_in_current_col = column_ptr[j+1] - column_ptr[j];
fseek(fp, col_block_size*(col_num+1) + init_block_size*3 + rowVal_block_size*(column_ptr[j]), SEEK_SET);
//printf("inserting\n");
for (i=0; i<nz_in_current_col; i++) {
fscanf(fp, "%d,", ¤t_column_rows[i]);
while (fgetc(fp) != '\n');
}
hash_columns = insert_hash(hash_columns, current_column_rows, nz_in_current_col, j, p, hash_weights, row_num, c);
}
printf("\nSeeds:\n");
print_hash(hash_columns, p);
fseek(fp, col_block_size*(col_num+1) + init_block_size*3, SEEK_SET);
//#pragma omp parallel for private(fp, j) num_threads(8)
for (j=0; j<col_num; j++) {
int nz_in_current_col = column_ptr[j+1] - column_ptr[j];
flag =1;
for (i=0; i<nz_in_current_col; i++) {
fscanf(fp, "%d,", ¤t_column_rows[i]);
while (fgetc(fp) != '\n');
}
//current_column_rows[i] = -1;
/*if (j==0)
{
for (i=0; i<nz_in_current_col; i++) {
printf("cur col[%d]: %d\n",i, current_column_rows[i]);
}
*/
for(c =0 ; c<p; c++)
{
if(seed[c] == j)
{
flag = 0;
}
}
if(flag == 1)
{
hash_columns = insert_hash(hash_columns, current_column_rows, nz_in_current_col, j, p, hash_weights, row_num, -1);
}
//}
}
// Load balancing
//printf("inserted in hash\n");
print_hash(hash_columns, p);
// Generate a column-wise index storing the partition alloted to each column
NODE temp;
int max;
#pragma omp parallel for num_threads(p)
for (i=0; i<p; i++) {
max = 0;
A_partition_column[i].col = hash_columns->col_counts[i];
A_partition[i] = (pair *)malloc(sizeof(pair)*A_partition_column[i].col);
temp = hash_columns->buckets[i];
for (j = 0; j < A_partition_column[i].col; j++) {
A_partition[i][j].col = temp->col_index;
A_partition[i][j].nz = temp->col_nz;
if (temp->col_nz > max) {
max = temp->col_nz;
}
temp = temp->next;
}
for (j=0; j<row_num; j++) {
if(hash_columns->row_indices[i][j] > YmaxPartition[j]) {
YmaxPartition[j] = hash_columns->row_indices[i][j];
YPartition[j] = i;
}
}
A_partition_column[i].nz = max;
}
}
// Broadcast the column-wise partition array
MPI_Bcast(A_partition_column, p, mpi_pair, 0, MPI_COMM_WORLD);
if (my_rank == 0) {
my_columns = *A_partition;
}
else {
my_columns = (pair *)malloc(sizeof(struct _pair)*A_partition_column[my_rank].col);
}
if (my_rank == 0) {
for (i=1; i<p; i++) {
MPI_Send(A_partition[i], A_partition_column[i].col, mpi_pair, i, 0, MPI_COMM_WORLD);
}
}
else {
MPI_Recv(my_columns, A_partition_column[my_rank].col, mpi_pair, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
MPI_Bcast(YPartition, row_num, MPI_INT, 0, MPI_COMM_WORLD);
//check what recvd in mycolumns
/*for(i=0; i<A_partition_column[my_rank].col; i++)
{
printf("Rank %d , Col no: %d, myNz : %d\n", my_rank, my_columns[i].col, my_columns[i].nz);
}*/
//partition_fp = (FILE **)malloc(sizeof(FILE*)*p);
FILE *my_output;
char f_name[20];
int colIndex, myNz, rowIndex, j;
float val;
char *buffer;
float *Y;
Y = (float*)calloc(row_num, sizeof(float));
//Read X
FILE *fp2;
fp2 = fopen("Xvector_algo2.txt", "r");
int *X;
X = (int*)malloc(sizeof(int)*A_partition_column[my_rank].col);
printf("Rank %d recvd %d columns\n", my_rank, A_partition_column[my_rank].col);
#pragma omp parallel for private(fp2, colIndex, i)
for(i=0; i<A_partition_column[my_rank].col; i++)
{
colIndex = my_columns[i].col;
fseek(fp2, colIndex*vector_block_size, SEEK_SET);
fscanf(fp2, "%d\n", &X[i]);
}
fclose(fp2);
/*for(i=0; i<A_partition_column[my_rank].col; i++)
{
printf("Rank %d ::, X[%d] = %d\n",my_rank, i, X[i]);
}*/
//for each column in A_partition_column[my_rank]...
//Read non zeroes and multiply (computing local Y)...
#pragma omp parallel for private(fp, colIndex, myNz, rowIndex, val)
for(i=0; i<A_partition_column[my_rank].col; i++)
{
//printf("proc: %d, Operating on col %d \n", my_rank, colIndex);
colIndex = my_columns[i].col;
myNz = my_columns[i].nz;
//seek to non-zeroes corresponding to this column in file
fseek(fp, col_block_size*(col_num+1) + init_block_size*3 + rowVal_block_size*(column_ptr[colIndex]), SEEK_SET);
//fread(buffer, myNz*rowVal_block_size,1,fp);
//for each non zero...
for(j=0; j<myNz; j++)
{
fscanf(fp, "%d, %f", &rowIndex, &val);
while (fgetc(fp) != '\n');
if(rowIndex>=row_num)
{
//printf("\n\n***********ERROR %d\n\n\n\n", rowIndex);
}
#pragma omp atomic
Y[rowIndex]+= X[i]*val;
}
}
//printf("end of loop: %d\n", my_rank);
pairF *sendOthers[p];
int numRowsInPartition[p], part;
//numRowsInPartition = (int*) malloc(sizeof(int)*p);
#pragma omp parallel for
for(i=0; i<row_num; i++)
{
numRowsInPartition[i] = 0;
}
/* for(i=0; i<row_num; i++)
{
printf("YPartition[%d] = %d\n", i, YPartition[i]);
}
*/
#pragma omp parallel for
for(i=0; i<row_num; i++)
{
part = YPartition[i];
#pragma omp atomic
numRowsInPartition[part]++;
}
if (my_rank == 0) {
for(i=0;i < p; i++)
{
printf("Rank %d got %d rows of Y vector\n", i, numRowsInPartition[i]);
}
}
//make the arrays that have to be sent to other processes.
//pair arrays that store rowIndex and val.
//allocate!
for(i=0; i<p;i++)
{
//if(i!=my_rank)
//{
sendOthers[i] = (pairF*)malloc(sizeof(pairF)*numRowsInPartition[i]);
//}
}
int *current = (int*) calloc(p, sizeof(int));
int other, other_pos;
//populate!
for(i=0; i<row_num; i++)
{
other = YPartition[i];
//if(other!=my_rank)
//{
other_pos = current[other];
sendOthers[other][other_pos].row = i;
sendOthers[other][other_pos].val = Y[i];
current[other]++;
//}
}
//write to respective files
FILE *partition_fp[p];
//open output files
for (i=0; i< p; i++)
{
sprintf(f_name, "%d", i);
//printf("open file %d\n", i);
partition_fp[i] = fopen(f_name, "a");
}
//FILE *fp21 = fopen("hehe.txt", "a");
for(i=0; i<p; i++)
{
if(i!=my_rank)
{
other = i;
for(j=0; j< numRowsInPartition[other]; j++)
{
if(sendOthers[other][j].val!=0){
fprintf(partition_fp[other], "%d, %f, process %d\n",sendOthers[other][j].row, sendOthers[other][j].val, my_rank);
}
}
}
}
//read from respective files and add!
MPI_Barrier(MPI_COMM_WORLD);
for (i = 0; i < p; ++i)
{
fclose(partition_fp[i]);
}
//printf("all files closed by rank %d\n", my_rank);
sprintf(f_name, "%d", my_rank);
partition_fp[my_rank] = fopen(f_name, "r");
strcat(f_name, "_output.txt");
my_output = fopen(f_name, "w");
while (fscanf(partition_fp[my_rank], "%d, %f", &rowIndex, &val) > 0) // expect 1 successful conversion
{
while (fgetc(partition_fp[my_rank]) != '\n');
//update local y
//printf("\n****\nRank %d read value %f\n\n", my_rank, val);
Y[rowIndex]+=val;
}
for(i=0; i<numRowsInPartition[my_rank]; i++)
{
rowIndex = sendOthers[my_rank][i].row;
sendOthers[my_rank][i].val = Y[rowIndex];
//these are the final values!
fprintf(my_output, "%d, %f, process %d\n",sendOthers[my_rank][i].row, sendOthers[my_rank][i].val, my_rank);
}
fclose(partition_fp[my_rank]);
fclose(my_output);
endTime = clock();
printf("\nrank = %d, Time taken: %lf\n", my_rank, (double)(endTime - startTime)/CLOCKS_PER_SEC);
MPI_Finalize();
return 0;
}
