int main(int narg, char **args)
{
MPI_Init(&narg,&args);
int me,nprocs;
MPI_Comm_rank(MPI_COMM_WORLD,&me);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
if (narg <= 1) {
if (me == 0) printf("Syntax: wordfreq file1 file2 ...\n");
MPI_Abort(MPI_COMM_WORLD,1);
}
MapReduce *mr = new MapReduce(MPI_COMM_WORLD);
mr->verbosity = 2;
mr->timer = 1;
//mr->memsize = 1;
//mr->outofcore = 1;
MPI_Barrier(MPI_COMM_WORLD);
double tstart = MPI_Wtime();
int nwords = mr->map(narg-1,&args[1],0,1,0,fileread,NULL);
int nfiles = mr->mapfilecount;
mr->collate(NULL);
int nunique = mr->reduce(sum,NULL);
MPI_Barrier(MPI_COMM_WORLD);
double tstop = MPI_Wtime();
mr->sort_values(&ncompare);
Count count;
count.n = 0;
count.limit = 10;
count.flag = 0;
mr->map(mr,output,&count);
mr->gather(1);
mr->sort_values(ncompare);
count.n = 0;
count.limit = 10;
count.flag = 1;
mr->map(mr,output,&count);
delete mr;
if (me == 0) {
printf("%d total words, %d unique words\n",nwords,nunique);
printf("Time to process %d files on %d procs = %g (secs)\n",
nfiles,nprocs,tstop-tstart);
}
MPI_Finalize();
}
void WordFreq::run()
{
int me;
MPI_Comm_rank(MPI_COMM_WORLD,&me);
// MR = word : NULL
int nfiles = 0;
MapReduce *mr = obj->input(1,read_words,&nfiles);
uint64_t nwords = mr->kv_stats(0);
int nfiles_all;
MPI_Allreduce(&nfiles,&nfiles_all,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
// unique words and their count
// before processing it, make a copy of input MR if it is permanent
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->collate(NULL);
uint64_t nunique = mr->reduce(count,NULL);
obj->output(1,mr,print_string_int,NULL);
// frequency stats
// before processing it, make a copy of output MR if it is permanent
if (ntop) {
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->sort_values(-1);
Count count;
count.n = 0;
count.limit = 10;
count.flag = 0;
mr->map(mr,output,&count);
mr->gather(1);
mr->sort_values(-1);
count.n = 0;
count.limit = ntop;
count.flag = 1;
mr->map(mr,output,&count);
}
char msg[128];
sprintf(msg,"WordFreq: %d files, %lu words, %lu unique",
nfiles_all,nwords,nunique);
if (me == 0) error->message(msg);
obj->cleanup();
}
int MR_sort_values(void *MRptr, int (*mycompare)(char *, int, char *, int))
{
MapReduce *mr = (MapReduce *) MRptr;
return mr->sort_values(mycompare);
}
