void word_count(int m, int r, char *address){ FILE * fp= NULL; char * line = NULL; size_t len = 0; ssize_t read; char* address1 = "/input"; char* address2 = "/output"; char **text = malloc(sizeof(char*) * m); int x = 0; for(x=0;x<m;x++){ char *n = malloc (sizeof(char)*2); snprintf(n , 10, "%d", x); char *put = join3(address,address1); char *input = join3(put,n); /*creat the input file address*/ fp =fopen(input,"r"); if (fp == NULL) exit(EXIT_FAILURE); /*open the file*/ char * result = ""; while ((read = getline(&line, &len, fp)) != -1) { int length = strlen(line); line[length-1] = ' '; result = join3(result , line); /*join all the lines in one file together to one single line*/ } text[x] = result; /*store the line in the corresponding line in the **text */ fclose(fp); } mymap = (map_t *)malloc(sizeof(map_t ) * r); int number=0; for(number =0; number <r;number ++) mymap[number] = hashmap_new(); int q =0; queue = (struct buffered_queue *)malloc(sizeof(struct buffered_queue) * r); for( q=0; q<r; q++){ buffered_queue_init(&queue[q],20); } /*initialize the hashtable and the queue*/ int num =0; if (m>=r) num =m; else num = r; struct ma arg[num]; int z =0; for( z=0; z<num; z++){ arg[z].text = (char **)malloc(sizeof(char*) * m); arg[z].id = z; arg[z].r = r; arg[z].m = m; arg[z].text = text; } /*the information passed to the thread store in the struct*/ pthread_t map[m]; pthread_t re[r]; int a = 0; int b =0; int c=0; int d=0; for( a=0; a<m; a++){ pthread_create(&map[a], NULL, mapper, &arg[a]); } for( b=0; b<r; b++){ pthread_create(&re[b], NULL, reducer, &arg[b]); } for( c=0; c<m; c++){ pthread_join(map[c], NULL); } for( d=0; d<r; d++){ pthread_join(re[d], NULL); } buffered_queue_destroy(queue); /*creat and run the thread */ int t = 0; for(t=0;t<r;t++){ char *z = malloc (sizeof(char)*2); snprintf(z , 10, "%d", t); char *put = join3(address,address2); /*creat the input text address*/ char *input = join3(put,z); fp = fopen (input, "w+"); /*open the file*/ int j =0; hashmap_map * m; m = (hashmap_map *) mymap[t]; for(j = 0; j< m->table_size; j++){ if(m->data[j].in_use != 0) { char *key = m->data[j].key; int data = m->data[j].data; fprintf(fp, "%s %d\n", key,data); /* copy one hashtable to on corresponding output file*/ } } fclose(fp); } return ; }
void word_count(int m, int r, char **files, char *directory){ map_t mymap[r]; pthread_t mappers[m]; // Thread for each mapper pthread_t reducers[r]; // Thread for each reducer int *signal = malloc(sizeof(*signal)); // Keep track of finished mappers *signal = m; pthread_mutex_t lock; pthread_mutex_init(&lock, NULL); //printf("metting up buffered queue connections\n"); struct buffered_queue **conns = malloc(sizeof(*conns) * r);// Hold connections for (int i = 0; i < r; i++) { struct buffered_queue *conn = malloc(sizeof(*conn)); buffered_queue_init(conn, 2*m); // Let queue fill & wait since text won't fit in memory conns[i] = conn; } for (int i = 0; i < m; i++){ // Create mapper threads struct mapperArgs *mArgs = malloc(sizeof(*mArgs)); // Get arguments for map function mArgs->mapperId = i; mArgs->reducerCount = r; mArgs->file = files[i]; mArgs->conns = conns; mArgs->signal = signal; mArgs->signal_lock = lock; //printf("Beginning mapper functions\n"); pthread_create(&mappers[i], NULL, map, (void*)mArgs); // Create thread } //printf("Beginning reducer processes\n"); for (int i = 0; i < r; i++){ printf("%s\n", directory); char *file_no = malloc(sizeof(*file_no)); sprintf(file_no,"%d", i); char *file_path = malloc(strlen(directory) + strlen(file_no) + 8); strcpy(file_path, directory); strcat(file_path, "/output"); strcat(file_path, file_no); printf("Reducer %d will write to file %s\n", i, file_path); mymap[i] = hashmap_new(file_path); struct reducerArgs *rArgs = malloc(sizeof(*rArgs)); rArgs->mapperCount = m; rArgs->reducerId = i; rArgs->map = &(mymap[i]); rArgs->conns = conns; pthread_create(&reducers[i], NULL, reduce, (void*)rArgs); } for (int i = 0; i < m; i++){ //printf("Waiting for mapper %d to finish\n", i); pthread_join(mappers[i], NULL); } for (int i = 0; i < r; i++){ //printf("Waiting for reducer %d to finish\n", i); pthread_join(reducers[i], NULL); } // printf("Result:\n"); // hashmap_iterate(mymap, print_result); }