void WordFreq::run()
{
int me;
MPI_Comm_rank(MPI_COMM_WORLD,&me);
// MR = word : NULL
int nfiles = 0;
MapReduce *mr = obj->input(1,read_words,&nfiles);
uint64_t nwords = mr->kv_stats(0);
int nfiles_all;
MPI_Allreduce(&nfiles,&nfiles_all,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
// unique words and their count
// before processing it, make a copy of input MR if it is permanent
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->collate(NULL);
uint64_t nunique = mr->reduce(count,NULL);
obj->output(1,mr,print_string_int,NULL);
// frequency stats
// before processing it, make a copy of output MR if it is permanent
if (ntop) {
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->sort_values(-1);
Count count;
count.n = 0;
count.limit = 10;
count.flag = 0;
mr->map(mr,output,&count);
mr->gather(1);
mr->sort_values(-1);
count.n = 0;
count.limit = ntop;
count.flag = 1;
mr->map(mr,output,&count);
}
char msg[128];
sprintf(msg,"WordFreq: %d files, %lu words, %lu unique",
nfiles_all,nwords,nunique);
if (me == 0) error->message(msg);
obj->cleanup();
}
void Histo::run()
{
int me,nprocs;
MPI_Comm_rank(MPI_COMM_WORLD,&me);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
// MR = key : NULL
MapReduce *mr = obj->input(1);
uint64_t ntotal = mr->kv_stats(0);
// unique keys and their count
// before processing it, make a copy of input MR if it is permanent
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->collate(NULL);
uint64_t nunique = mr->reduce(count,NULL);
obj->output(1,mr);
// histogram stats
// before processing it, make a copy of output MR if it is permanent
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->map(mr,invert,NULL);
mr->collate(NULL);
mr->reduce(count,NULL);
mr->gather(1);
mr->sort_keys(-1);
char msg[128];
sprintf(msg,"Histo: %lu total keys, %lu unique",ntotal,nunique);
if (me == 0) error->message(msg);
mr->scan(print,NULL);
obj->cleanup();
}
void MR_kv_stats(void *MRptr, int level)
{
MapReduce *mr = (MapReduce *) MRptr;
mr->kv_stats(level);
}
