void WordFreq::run()
{
  int me;
  MPI_Comm_rank(MPI_COMM_WORLD,&me);

  // MR = word : NULL

  int nfiles = 0;
  MapReduce *mr = obj->input(1,read_words,&nfiles);
  uint64_t nwords = mr->kv_stats(0);
  int nfiles_all;
  MPI_Allreduce(&nfiles,&nfiles_all,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);

  // unique words and their count
  // before processing it, make a copy of input MR if it is permanent

  if (obj->permanent(mr)) mr = obj->copy_mr(mr);
  mr->collate(NULL);
  uint64_t nunique = mr->reduce(count,NULL);

  obj->output(1,mr,print_string_int,NULL);

  // frequency stats
  // before processing it, make a copy of output MR if it is permanent

  if (ntop) {
    if (obj->permanent(mr)) mr = obj->copy_mr(mr);
    mr->sort_values(-1);

    Count count;
    count.n = 0;
    count.limit = 10;
    count.flag = 0;
    mr->map(mr,output,&count);

    mr->gather(1);
    mr->sort_values(-1);

    count.n = 0;
    count.limit = ntop;
    count.flag = 1;
    mr->map(mr,output,&count);
  }

  char msg[128];
  sprintf(msg,"WordFreq: %d files, %lu words, %lu unique",
  	  nfiles_all,nwords,nunique);
  if (me == 0) error->message(msg);

  obj->cleanup();
}
void Histo::run()
{
  int me,nprocs;
  MPI_Comm_rank(MPI_COMM_WORLD,&me);
  MPI_Comm_size(MPI_COMM_WORLD,&nprocs);

  // MR = key : NULL

  MapReduce *mr = obj->input(1);
  uint64_t ntotal = mr->kv_stats(0);

  // unique keys and their count
  // before processing it, make a copy of input MR if it is permanent

  if (obj->permanent(mr)) mr = obj->copy_mr(mr);
  mr->collate(NULL);
  uint64_t nunique = mr->reduce(count,NULL);

  obj->output(1,mr);

  // histogram stats
  // before processing it, make a copy of output MR if it is permanent

  if (obj->permanent(mr)) mr = obj->copy_mr(mr);

  mr->map(mr,invert,NULL);
  mr->collate(NULL);
  mr->reduce(count,NULL);
  mr->gather(1);
  mr->sort_keys(-1);

  char msg[128];
  sprintf(msg,"Histo: %lu total keys, %lu unique",ntotal,nunique);
  if (me == 0) error->message(msg);
  mr->scan(print,NULL);

  obj->cleanup();
}
Example #3
0
void MR_kv_stats(void *MRptr, int level)
{
  MapReduce *mr = (MapReduce *) MRptr;
  mr->kv_stats(level);
}