void Histo::run()
{
int me,nprocs;
MPI_Comm_rank(MPI_COMM_WORLD,&me);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
// MR = key : NULL
MapReduce *mr = obj->input(1);
uint64_t ntotal = mr->kv_stats(0);
// unique keys and their count
// before processing it, make a copy of input MR if it is permanent
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->collate(NULL);
uint64_t nunique = mr->reduce(count,NULL);
obj->output(1,mr);
// histogram stats
// before processing it, make a copy of output MR if it is permanent
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->map(mr,invert,NULL);
mr->collate(NULL);
mr->reduce(count,NULL);
mr->gather(1);
mr->sort_keys(-1);
char msg[128];
sprintf(msg,"Histo: %lu total keys, %lu unique",ntotal,nunique);
if (me == 0) error->message(msg);
mr->scan(print,NULL);
obj->cleanup();
}
int MR_sort_keys(void *MRptr, int (*mycompare)(char *, int, char *, int))
{
MapReduce *mr = (MapReduce *) MRptr;
return mr->sort_keys(mycompare);
}
int main(int narg, char **args)
{
int me, nprocs;
int nwords, nunique;
double tstart, tstop;
Count count;
pthread_t thread1;
MPI_Init(&narg, &args);
MPI_Comm_rank(MPI_COMM_WORLD, &me);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (narg <= 1)
{
if (me == 0)
{
printf("Syntax: cwordfreq file1 file2 ...\n");
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
if (me != 0)
{
pthread_create(&thread1, NULL, hb_function, &me);
}
else
{
printf("PID %d on ready for attach\n", getpid());
pthread_create(&thread1, NULL, pingRecv, &me);
}
MapReduce *mr = new MapReduce(MPI_COMM_WORLD);
MapReduce *mr2 = new MapReduce(MPI_COMM_WORLD);
//MapReduce *mra = new MapReduce(MPI_COMM_WORLD);
//MapReduce *mrb = new MapReduce(MPI_COMM_WORLD);
MapReduce *mra = new MapReduce(MPI_COMM_WORLD);
MapReduce *mrb = new MapReduce(MPI_COMM_WORLD);
// mra->open();
mr2->open();
MPI_Barrier(MPI_COMM_WORLD);
int test = 1;
void *arg[5] = {args[1], args[2], mra->kv, mrb->kv, mr2->kv};
//printf("%s %s\n", arg[0], arg[1]);
printf("starting map\n");
nwords = mr->map(nprocs, &fileread, arg);
MPI_Barrier(MPI_COMM_WORLD);
printf("map done\n");
mr2->close();
// mrb->close();
printf("%d mr closed\n", me);
mr->collate(NULL);
MPI_Barrier(MPI_COMM_WORLD);
printf("collate done\n");
nunique = mr->reduce(&sum2, NULL);
MPI_Barrier(MPI_COMM_WORLD);
printf("reduce done\n");
mr->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
mr2->collate(NULL);
mr2->reduce(&sum,NULL);
mr2->gather(1);
mr->add(mr2);
mr->sort_keys(&ncompare);
MPI_Barrier(MPI_COMM_WORLD);
FILE * pFile;
pFile = fopen("result.out", "w");
mr->map(mr, &output, pFile);
MPI_Barrier(MPI_COMM_WORLD);
fclose(pFile);
//sum done
//getHistogram(mra, "hist.a");
//getHistogram(mr, "hist.a");
//getHistogram(mrb, "hist.b");
/* FILE * pFilea;
printf("sum done\n");
pFilea = fopen("test", "w");
mra->gather(1);
mra->map(mra, &histoutput, pFilea);
fclose(pFilea);*/
printf("%d sum done\n", me);
mr->collate(NULL);
MPI_Barrier(MPI_COMM_WORLD);
mr->reduce(&sum, NULL);
MPI_Barrier(MPI_COMM_WORLD);
mr->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
mr->sort_keys(&ncompare);
MPI_Barrier(MPI_COMM_WORLD);
FILE * pFile2;
pFile2 = fopen("hist.c", "w");
mr->map(mr, &histoutput, pFile2);
MPI_Barrier(MPI_COMM_WORLD);
fclose(pFile2);
delete mr;
printf("%d c done\n", me);
mra->open();
mrb->open();
void *arg2[4] = {args[1], args[2], mra->kv, mrb->kv};
printf("%p %p\n", mra->kv, arg2[2]);
nwords = mra->map(nprocs, &fileread2, arg2);
mra->close();
mrb->close();
mra->collate(NULL);
MPI_Barrier(MPI_COMM_WORLD);
mra->reduce(&sum, NULL);
MPI_Barrier(MPI_COMM_WORLD);
mra->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
mra->sort_keys(&ncompare);
MPI_Barrier(MPI_COMM_WORLD);
mra->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
FILE * pFile3;
pFile3 = fopen("hist.a", "w");
mra->map(mra, &histoutput, pFile3);
fclose(pFile3);
delete mra;
mrb->collate(NULL);
MPI_Barrier(MPI_COMM_WORLD);
mrb->reduce(&sum, NULL);
MPI_Barrier(MPI_COMM_WORLD);
mrb->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
mrb->sort_keys(&ncompare);
MPI_Barrier(MPI_COMM_WORLD);
FILE * pFile4;
pFile4 = fopen("hist.b", "w");
mrb->map(mrb, &histoutput, pFile4);
fclose(pFile4);
delete mrb;
MPI_Barrier(MPI_COMM_WORLD);
pthread_cancel(thread1);
MPI_Finalize();
//exit(0);
}
