int main(int narg, char **args)
{
MPI_Init(&narg,&args);
int me,nprocs;
MPI_Comm_rank(MPI_COMM_WORLD,&me);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
if (narg <= 1) {
if (me == 0) printf("Syntax: wordfreq file1 file2 ...\n");
MPI_Abort(MPI_COMM_WORLD,1);
}
MapReduce *mr = new MapReduce(MPI_COMM_WORLD);
mr->verbosity = 2;
mr->timer = 1;
//mr->memsize = 1;
//mr->outofcore = 1;
MPI_Barrier(MPI_COMM_WORLD);
double tstart = MPI_Wtime();
int nwords = mr->map(narg-1,&args[1],0,1,0,fileread,NULL);
int nfiles = mr->mapfilecount;
mr->collate(NULL);
int nunique = mr->reduce(sum,NULL);
MPI_Barrier(MPI_COMM_WORLD);
double tstop = MPI_Wtime();
mr->sort_values(&ncompare);
Count count;
count.n = 0;
count.limit = 10;
count.flag = 0;
mr->map(mr,output,&count);
mr->gather(1);
mr->sort_values(ncompare);
count.n = 0;
count.limit = 10;
count.flag = 1;
mr->map(mr,output,&count);
delete mr;
if (me == 0) {
printf("%d total words, %d unique words\n",nwords,nunique);
printf("Time to process %d files on %d procs = %g (secs)\n",
nfiles,nprocs,tstop-tstart);
}
MPI_Finalize();
}
void WordFreq::run()
{
int me;
MPI_Comm_rank(MPI_COMM_WORLD,&me);
// MR = word : NULL
int nfiles = 0;
MapReduce *mr = obj->input(1,read_words,&nfiles);
uint64_t nwords = mr->kv_stats(0);
int nfiles_all;
MPI_Allreduce(&nfiles,&nfiles_all,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
// unique words and their count
// before processing it, make a copy of input MR if it is permanent
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->collate(NULL);
uint64_t nunique = mr->reduce(count,NULL);
obj->output(1,mr,print_string_int,NULL);
// frequency stats
// before processing it, make a copy of output MR if it is permanent
if (ntop) {
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->sort_values(-1);
Count count;
count.n = 0;
count.limit = 10;
count.flag = 0;
mr->map(mr,output,&count);
mr->gather(1);
mr->sort_values(-1);
count.n = 0;
count.limit = ntop;
count.flag = 1;
mr->map(mr,output,&count);
}
char msg[128];
sprintf(msg,"WordFreq: %d files, %lu words, %lu unique",
nfiles_all,nwords,nunique);
if (me == 0) error->message(msg);
obj->cleanup();
}
Vector* Vector::add(Vector* other) {
MapReduce* sum = MapReduce::copy();
sum->add(other);
sum->collate(NULL);
// Gets set to true if, in addReduce, it is discovered that the vectors were
// of unequal length.
bool unequalLength = false;
sum->reduce(&addReduce, &unequalLength);
sum->gather(1);
if (unequalLength)
return NULL;
return static_cast<Vector*>(sum);
}
void Histo::run()
{
int me,nprocs;
MPI_Comm_rank(MPI_COMM_WORLD,&me);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
// MR = key : NULL
MapReduce *mr = obj->input(1);
uint64_t ntotal = mr->kv_stats(0);
// unique keys and their count
// before processing it, make a copy of input MR if it is permanent
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->collate(NULL);
uint64_t nunique = mr->reduce(count,NULL);
obj->output(1,mr);
// histogram stats
// before processing it, make a copy of output MR if it is permanent
if (obj->permanent(mr)) mr = obj->copy_mr(mr);
mr->map(mr,invert,NULL);
mr->collate(NULL);
mr->reduce(count,NULL);
mr->gather(1);
mr->sort_keys(-1);
char msg[128];
sprintf(msg,"Histo: %lu total keys, %lu unique",ntotal,nunique);
if (me == 0) error->message(msg);
mr->scan(print,NULL);
obj->cleanup();
}
int MR_gather(void *MRptr, int numprocs)
{
MapReduce *mr = (MapReduce *) MRptr;
return mr->gather(numprocs);
}
int main(int narg, char **args)
{
int me, nprocs;
int nwords, nunique;
double tstart, tstop;
Count count;
pthread_t thread1;
MPI_Init(&narg, &args);
MPI_Comm_rank(MPI_COMM_WORLD, &me);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if (narg <= 1)
{
if (me == 0)
{
printf("Syntax: cwordfreq file1 file2 ...\n");
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
if (me != 0)
{
pthread_create(&thread1, NULL, hb_function, &me);
}
else
{
printf("PID %d on ready for attach\n", getpid());
pthread_create(&thread1, NULL, pingRecv, &me);
}
MapReduce *mr = new MapReduce(MPI_COMM_WORLD);
MapReduce *mr2 = new MapReduce(MPI_COMM_WORLD);
//MapReduce *mra = new MapReduce(MPI_COMM_WORLD);
//MapReduce *mrb = new MapReduce(MPI_COMM_WORLD);
MapReduce *mra = new MapReduce(MPI_COMM_WORLD);
MapReduce *mrb = new MapReduce(MPI_COMM_WORLD);
// mra->open();
mr2->open();
MPI_Barrier(MPI_COMM_WORLD);
int test = 1;
void *arg[5] = {args[1], args[2], mra->kv, mrb->kv, mr2->kv};
//printf("%s %s\n", arg[0], arg[1]);
printf("starting map\n");
nwords = mr->map(nprocs, &fileread, arg);
MPI_Barrier(MPI_COMM_WORLD);
printf("map done\n");
mr2->close();
// mrb->close();
printf("%d mr closed\n", me);
mr->collate(NULL);
MPI_Barrier(MPI_COMM_WORLD);
printf("collate done\n");
nunique = mr->reduce(&sum2, NULL);
MPI_Barrier(MPI_COMM_WORLD);
printf("reduce done\n");
mr->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
mr2->collate(NULL);
mr2->reduce(&sum,NULL);
mr2->gather(1);
mr->add(mr2);
mr->sort_keys(&ncompare);
MPI_Barrier(MPI_COMM_WORLD);
FILE * pFile;
pFile = fopen("result.out", "w");
mr->map(mr, &output, pFile);
MPI_Barrier(MPI_COMM_WORLD);
fclose(pFile);
//sum done
//getHistogram(mra, "hist.a");
//getHistogram(mr, "hist.a");
//getHistogram(mrb, "hist.b");
/* FILE * pFilea;
printf("sum done\n");
pFilea = fopen("test", "w");
mra->gather(1);
mra->map(mra, &histoutput, pFilea);
fclose(pFilea);*/
printf("%d sum done\n", me);
mr->collate(NULL);
MPI_Barrier(MPI_COMM_WORLD);
mr->reduce(&sum, NULL);
MPI_Barrier(MPI_COMM_WORLD);
mr->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
mr->sort_keys(&ncompare);
MPI_Barrier(MPI_COMM_WORLD);
FILE * pFile2;
pFile2 = fopen("hist.c", "w");
mr->map(mr, &histoutput, pFile2);
MPI_Barrier(MPI_COMM_WORLD);
fclose(pFile2);
delete mr;
printf("%d c done\n", me);
mra->open();
mrb->open();
void *arg2[4] = {args[1], args[2], mra->kv, mrb->kv};
printf("%p %p\n", mra->kv, arg2[2]);
nwords = mra->map(nprocs, &fileread2, arg2);
mra->close();
mrb->close();
mra->collate(NULL);
MPI_Barrier(MPI_COMM_WORLD);
mra->reduce(&sum, NULL);
MPI_Barrier(MPI_COMM_WORLD);
mra->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
mra->sort_keys(&ncompare);
MPI_Barrier(MPI_COMM_WORLD);
mra->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
FILE * pFile3;
pFile3 = fopen("hist.a", "w");
mra->map(mra, &histoutput, pFile3);
fclose(pFile3);
delete mra;
mrb->collate(NULL);
MPI_Barrier(MPI_COMM_WORLD);
mrb->reduce(&sum, NULL);
MPI_Barrier(MPI_COMM_WORLD);
mrb->gather(1);
MPI_Barrier(MPI_COMM_WORLD);
mrb->sort_keys(&ncompare);
MPI_Barrier(MPI_COMM_WORLD);
FILE * pFile4;
pFile4 = fopen("hist.b", "w");
mrb->map(mrb, &histoutput, pFile4);
fclose(pFile4);
delete mrb;
MPI_Barrier(MPI_COMM_WORLD);
pthread_cancel(thread1);
MPI_Finalize();
//exit(0);
}