void DoMasterAMR(char *buf, int size, int who, int tag, thread_arg_t *targ) {
MasterGamlConfig *conf = targ->conf;
Population *pop = targ->pop;
int count, start_shield, which;
char *tree_strings, *model_buf;
double *models, score;
assert(tag == TAG_SCORE);
memcpy(&score, buf, sizeof(double));
//delete [] buf;
// print some messages
debug_mpi("SYNCHRONOUS COMM (%d, AMR, %f)", who, pop->bestFitness);
debug_mpi("\trecv: score %f", score);
if (score > pop->bestFitness) {
SendMPIMessage(NULL, 0, who, TAG_TREE_STRINGS_REQUEST);
debug_mpi("\tsent: TAG_TREE_STRINGS_REQUEST");
RecvMPIMessage(&buf, &size, who, &tag, true);
assert(tag == TAG_TREE_STRINGS);
tree_strings = buf;
debug_mpi("\trecv: %d tree strings", CountTreeStrings(tree_strings));
RecvMPIMessage(&buf, &size, who, &tag, true);
assert(tag == TAG_MODEL);
models=(double*)buf;
which = pop->total_size-1;
pop->ReplaceSpecifiedIndividuals(1, &which, tree_strings, models);
pop->CalcAverageFitness();
debug_mpi("score sent: %f, score calced: %f", score, pop->IndivFitness(which));
//assert(abs(score - pop->IndivFitness(which))<.00001);
}
else {
which = (int)pop->cumfit[pop->total_size-1][0];
pop->GetSpecifiedTreeStrings(&tree_strings, 1, &which);
int model_size=pop->GetSpecifiedModels(&models, 1, &which);
SendMPIMessage(tree_strings, strlen2(tree_strings)+2, who, TAG_TREE_STRINGS);
debug_mpi("\tsent: %d tree strings", CountTreeStrings(tree_strings));
model_buf = (char*)models;
SendMPIMessage(model_buf, sizeof(double)*model_size, who, TAG_MODEL);
debug_mpi("\tsent: %d models", 1);
}
delete [] tree_strings;
delete [] models;
}
void DoMasterSM(char *buf, int size, int who, int tag, thread_arg_t *targ) {
MasterGamlConfig *conf = targ->conf;
Population *pop = targ->pop;
int count, start_shield, *which = new int;//[conf->gc.numshields];
char *tree_strings;
double *models;
assert(tag == TAG_TREE_STRINGS);
tree_strings = buf;
count = CountTreeStrings(tree_strings);
RecvMPIMessage(&buf, &size, who, &tag, true);
assert(tag == TAG_MODEL);
models=(double*)buf;
// print some messages
debug_mpi("SYNCHRONOUS COMMUNICATION (%d, SM)", who);
debug_mpi("\trecv: %d tree strings", count);
debug_mpi("\trecv: %d models", count);
bool poo=true;
//while(poo);
*which = start_shield = pop->params->nindivs + (who-1);
// for (int i = 0; i < conf->gc.numshields; ++i)
// which[i] = start_shield++;
pop->ReplaceSpecifiedIndividuals(count, which, tree_strings, models);
pop->CalcAverageFitness();
delete [] which;
//this will be deleted back where the initial call to RecvMPIMessage was made
//delete [] tree_strings;
delete [] (char*)models;
//under certain conditions, tell the remote to become an AMR node
if(((double)rand()/RAND_MAX)<.002){
debug_mpi("sent message to change to AMR to remote %d", who);
SendMPIMessage(NULL, 0, who, TAG_REMOTE_TYPE_SWITCH);
//get rid of any SM messages that might be waiting
while(RecvMPIMessage(&buf, &size, who, &tag, false)==true);
//update the remote_types array
remote_types[who]=AMR;
}
}
int DoMasterSW(char *buf, int size, int who, int tag, thread_arg_t *targ) {
MasterGamlConfig *conf = targ->conf;
Population *pop = targ->pop;
int count, start_shield, *which = new int;//[conf->gc.numshields];
char *tree_strings, *model_buf;
char *out_tree_strings;
char *defbuf, *subbuf;
double *out_models;
double *models;
int remoteSubtreeDef, remoteSubtreeNode;
ParallelManager *paraMan=(pop->paraMan);
//first get the tree and model from the remote and include it in
//the master population. If there are multiple messages, chuck
//earlier ones and just get the most recent
bool firstmessage=true;
do{
debug_mpi("Remote %d", who);
assert(tag == TAG_TREE_STRINGS || tag == TAG_QUIT);
if(tag==TAG_QUIT) return 1;
tree_strings = buf;
count = CountTreeStrings(tree_strings);
// debug_mpi("about to get model strings...");
RecvMPIMessage(&buf, &size, who, &tag, true);
assert(tag == TAG_MODEL);
models=(double*)buf;
// debug_mpi("about to get subdef strings...");
//determine what the remote was doing when it sent this tree
RecvMPIMessage(&defbuf, &size, who, &tag, true);
assert(tag==TAG_SUBTREE_ITERATION);
remoteSubtreeDef=atoi(defbuf);
if(remoteSubtreeDef>0){
RecvMPIMessage(&subbuf, &size, who, &tag, true);
assert(tag==TAG_SUBTREE_DEFINE);
remoteSubtreeNode=atoi(subbuf);
if(remoteSubtreeDef==paraMan->subtreeDefNumber)
paraMan->localSubtreeAssign[who]=remoteSubtreeNode;
else paraMan->localSubtreeAssign[who]=0;
delete []subbuf;
}
//DJZ 5-18-05
else {
paraMan->localSubtreeAssign[who]=0;
remoteSubtreeNode=0;
}
double score;
char *scoreBuf;
RecvMPIMessage(&scoreBuf, &size, who, &tag, true);
assert(tag==TAG_SCORE);
memcpy(&score, scoreBuf, sizeof(double));
// debug_mpi("recieved score of %f", score);
delete []scoreBuf;
if(firstmessage==false) debug_mpi("\tfound another tree from remote %d", who);
*which = start_shield = pop->params->nindivs + (who-1);
pop->ReplaceSpecifiedIndividuals(count, which, tree_strings, models);
pop->indiv[*which].SetFitness(score);
if(firstmessage==false) delete []tree_strings;
delete [](char*)models;
delete []defbuf;
firstmessage=false;
}while(RecvMPIMessage(&buf, &size, who, &tag, false)==true);
bool subtreesCurrent = ((remoteSubtreeDef == paraMan->subtreeDefNumber) && remoteSubtreeDef > 0);
if(paraMan->subtreeModeActive==false || subtreesCurrent==false){
pop->indiv[*which].accurateSubtrees=false;
pop->newindiv[*which].accurateSubtrees=false;
}
else {
pop->indiv[*which].accurateSubtrees=true;
pop->newindiv[*which].accurateSubtrees=true;
}
// debug_mpi("about to CalcFitness...");
double prevBestScore=pop->BestFitness();
// pop->indiv[*which].CalcFitness(0);
pop->indiv[*which].treeStruct->calcs=calcCount;
pop->CalcAverageFitness();
//reclaim clas if the new tree has essentially no chance of reproducing
if(((pop->indiv[*which].Fitness() - pop->indiv[pop->bestIndiv].Fitness()) < (-11.5/pop->params->selectionIntensity))){
// debug_mpi("about to reclaim...");
pop->indiv[*which].treeStruct->ReclaimUniqueClas();
}
//Now, take a look at what we got from the remote and decide what to do
double inscore=pop->indiv[*which].Fitness();
double scorediff=prevBestScore - inscore;
debug_mpi("\tnew ind - def %d - node %d - lnL: %f", remoteSubtreeDef, remoteSubtreeNode, inscore);
if(scorediff < 0) debug_mpi("\tPrev Best=%f, diff=%f (new best)", prevBestScore, scorediff);
else debug_mpi("\tPrev Best=%f, diff=%f", prevBestScore, scorediff);
// debug_mpi("\tbest=%d, bestAc=%d, bestlnL=%f, bestAcclnL=%f", pop->bestIndiv, pop->bestAccurateIndiv, pop->BestFitness(), pop->indiv[pop->bestAccurateIndiv].Fitness());
bool recalcSubtrees=false;
if(scorediff < -0.01){
pop->LogNewBestFromRemote(-scorediff, *which);
}
int subtreeNum;
bool send=false;
//there are really 8 possible cases here
//1. Subtree mode active, got accurate tree, score good -> do nothing
//2. score bad -> send best accurate tree
//3. inaccurate tree, score good -> recalc subtrees, send?
//4. score bad -> send best accurate tree
//5. Subtree mode inactive, got accurate tree, score good -> send best tree
//6. score bad -> send best tree
//7. inaccurate tree, score good -> do nothing
//8. score bad -> send best tree
//so, 2 "do nothings" 3 "send best", 2 "send best accurate" and 1 "subtree recalc"
//if subtree mode isn't active, send the remote our best tree if the
//tree we got from it is worse by some amount, or if it is still working
//on a subtree
double updateThresh=paraMan->updateThresh;
//DEBUG
// if(paraMan->subtreeModeActive==false && (paraMan->needToSend[who]==true || ((scorediff > nonSubThresh && paraMan->beforeFirstSubtree==false) || (scorediff > startThresh && paraMan->beforeFirstSubtree==true) || remoteSubtreeDef>0))){
if(paraMan->subtreeModeActive==false){
if((paraMan->perturbModeActive==false && (scorediff > updateThresh || remoteSubtreeDef>0))/* || (paraMan->needToSend[who]==true)*/){
debug_mpi("\tupdate thresh = %f, send indiv", updateThresh);
//cases 5, 6 and 8
*which = (int)pop->cumfit[pop->total_size-1][0];
subtreeNum=0;
send=true;
}
else debug_mpi("\tupdate thresh = %f", updateThresh);
}
else if(paraMan->subtreeModeActive==true){
//cases 1-4
if((scorediff > updateThresh) || (subtreesCurrent==false)/* || paraMan->perturb==true*/){
//cases 2 and 4. send the best accurate tree
*which=pop->bestAccurateIndiv;
if(paraMan->remoteSubtreeAssign[who] != 0) subtreeNum=paraMan->remoteSubtreeAssign[who];
else subtreeNum=paraMan->ChooseSubtree();
debug_mpi("\tsend best accurate ind, %f (best=%f)", pop->indiv[*which].Fitness(), pop->bestFitness);
// debug_mpi("\tperturb=%d, bestFit=%f, indFit=%f", paraMan->perturb, pop->bestFitness, pop->indiv[*which].Fitness());
send=true;
}
else if(recalcSubtrees==true && subtreesCurrent==false){
//case 3
//if the new inaccurate tree that came in is better than what we have,
//recalcuate the subtrees, and send the same tree back, but with a
//subtree asignment
pop->StartSubtreeMode();
debug_mpi("Recalculating subtrees");
subtreeNum=paraMan->ChooseSubtree();
send=true;
}
}
if(paraMan->needToSend[who]){
char pertbuf[5];
int perttype = (pop->pertMan->pertType > 0 ? pop->pertMan->pertType : (int)(rnd.uniform() * 2 + 1));
sprintf(pertbuf, "%d", perttype);
SendMPIMessage(pertbuf, strlen(pertbuf)+2, who, TAG_PERTURB);
debug_mpi("sending pertub message to %d, type %d", who, perttype);
paraMan->needToSend[who]=false;
}
if(send==true){
pop->GetSpecifiedTreeStrings(&out_tree_strings, 1, which);
assert(*out_tree_strings == '(');
int model_size=pop->GetSpecifiedModels(&out_models, 1, which);
SendMPIMessage(out_tree_strings, strlen2(out_tree_strings)+2, who, TAG_TREE_STRINGS);
SendMPIMessage((char*)out_models, sizeof(double)*model_size, who, TAG_MODEL);
/* if(paraMan->needToSend[who]){
char pertbuf[5];
int perttype = (pop->pertMan->pertType > 0 ? pop->pertMan->pertType : (rnd.uniform() * 2 + 1));
sprintf(pertbuf, "%d", subtreeNum);
SendMPIMessage(NULL, 0, who, TAG_PERTURB);
debug_mpi("sending pertub message to %d, type %d", who, perttype);
paraMan->needToSend[who]=false;
}
*/
// else{
char stn[5];
sprintf(stn, "%d", subtreeNum);
SendMPIMessage(stn, strlen(stn)+2, who, TAG_SUBTREE_DEFINE);
debug_mpi("\tsent ind %d, lnL %f", *which, pop->indiv[*which].Fitness());
if(subtreeNum > 0){
//if this node was already assigned a subtree, be sure to subtract the old one from the assigned array
sprintf(stn, "%d", paraMan->subtreeDefNumber);
debug_mpi("\tsubdef %d, node %d", paraMan->subtreeDefNumber, subtreeNum);
SendMPIMessage(stn, strlen(stn)+2, who, TAG_SUBTREE_ITERATION);
}
// }
paraMan->remoteSubtreeAssign[who]=subtreeNum;
delete []out_models;
delete []out_tree_strings;
}
#ifndef NDEBUG
if(paraMan->subtreeModeActive && paraMan->subtreeDefNumber==remoteSubtreeDef){
//DEBUG
//if we think that this remote gave us a tree with accurate subtrees, check
paraMan->CheckSubtreeAccuracy(pop->indiv[which[0]].treeStruct);
}
#endif
//the tree_strings that were passed in will be deleted back
//where the initial call to RecvMPIMessage was made
delete [] which;
pop->CalcAverageFitness();
return 0;
}
/* i would prefer that the thread initialization code happen in MPIMain(), but
* it needs Population pop, which is declared here */
int MasterMaster(MasterGamlConfig& conf, HKYData& data) {
Parameters params;
params.SetParams(conf, data);
LogParams(params);
int nprocs;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
bool poo=true;
// while(poo);
Tree::alpha = params.gammaShapeBrlen;
Tree::meanBrlenMuts = params.meanBrlenMuts;
Population pop;
// debug_mpi("about to setup");
pop.Setup(params, &conf, nprocs, 0);
g_sw=&pop.stopwatch;
// debug_mpi("setup");
g_gen = &pop.gen;
pop.CalcAverageFitness();
// start the thread
pthread_t thread;
thread_arg_t targ;
pthread_mutex_init(&lock_pm, NULL);
pthread_mutex_init(&lock_pop, NULL);
pthread_cond_init(&cond_pm, NULL);
g_quit_time = false;
g_processing_message = false;
targ.conf = &const_cast<MasterGamlConfig&>(conf);
targ.pop = &pop;
targ.nprocs = nprocs;
pthread_create(&thread, NULL, thread_func2, (void*)&targ);
cout << "Master running..." << endl;
pop.gen=0;
while (!g_quit_time){
pthread_mutex_lock(&lock_pop);
pop.keepTrack();
pop.OutputFate();
if (pop.gen % conf.logevery == 0) pop.OutputLog();
++pop.gen;
pop.NextGeneration();
if(pop.gen % pop.params->saveEvery == 0) pop.CreateTreeFile( pop.params->treefname );
if(pop.gen % pop.adap->intervalLength == 0){
if(pop.enforceTermConditions == true
&& pop.gen-pop.lastTopoImprove > pop.lastTopoImproveThresh
&& pop.adap->improveOverStoredIntervals < pop.improveOverStoredIntervalsThresh
&& pop.adap->branchOptPrecision == pop.adap->minOptPrecision){
// && pop.paraMan->updateThresh == pop.paraMan->minUpdateThresh){
cout << "Reached termination condition!\nlast topological improvement at gen " << pop.lastTopoImprove << endl;
cout << "Improvement over last " << pop.adap->intervalsToStore*pop.adap->intervalLength << " gen = " << pop.adap->improveOverStoredIntervals << endl;
g_quit_time=true;
break;
}
pop.CheckSubtrees();
//6-20-05 Changing this to deterministically reduce the replace thresh
//every subtreeInterval generations. Tying the reduction of this to
//how often the master recieved new bests from the remotes had odd properties,
//and didn't scale well with differing numbers of processors.
//and it's not clear what a better automated approach would be
//pop.CheckRemoteReplaceThresh();
if(pop.gen % pop.paraMan->subtreeInterval == 0) pop.paraMan->ReduceUpdateThresh();
#ifdef INCLUDE_PERTURBATION
pop.CheckPerturbParallel();
#endif
}
pthread_mutex_unlock(&lock_pop);
pthread_mutex_lock(&lock_pm);
while (g_processing_message)
pthread_cond_wait(&cond_pm, &lock_pm);
pthread_mutex_unlock(&lock_pm);
}
pop.FinalOptimization();
pthread_join(thread, NULL);
return 0;
}
int RemoteSubtreeWorker(Population& pop, const GeneralGamlConfig& conf){
int *which, size, rank, tag;
char *tree_strings, *buf;
double score, *models;
bool perturb;
which=new int[5];
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
pop.CalcAverageFitness();
int lastSend=g_sw->SplitTime();
cout << "Remote number " << rank << " running..." << endl;
for (pop.gen = 1; pop.gen < conf.stopgen;){
pop.keepTrack();
pop.OutputFate();
if (pop.gen % conf.logevery == 0)
pop.OutputLog();
++pop.gen;
pop.NextGeneration();
if(g_sw->SplitTime() - lastSend > conf.sendInterval){
debug_mpi("SYNCH COMM (node 0)");
//send our best individual to the master
RemoteSendBestTree(pop);
lastSend=g_sw->SplitTime();
if(pop.params->stoptime - g_sw->SplitTime() < 0){
debug_mpi("time limit of %d seconds reached...", pop.params->stoptime);
break;
}
}
//Check for a new tree from the master
bool firstmessage=true;
bool gotmessage=false;
int subtreeNode;
while(RecvMPIMessage(&tree_strings, &size, 0, &tag, false)==true){
//check for a quit message
if(tag == TAG_QUIT) {
debug_mpi("\trecv: quit message");
delete [] which;
debug_mpi("quitting");
return 0;
}
//
bool gotNewIndiv=false;
int recievedDefNumber;
debug_mpi("SYNCH COMM (node 0)");
gotmessage=true;
assert(tag == TAG_TREE_STRINGS || tag==TAG_PERTURB);
if(firstmessage==false) debug_mpi("\tfound a newer message...");
if(tag != TAG_PERTURB){
gotNewIndiv=true;
RecvMPIMessage(&buf, &size, 0, &tag, true);
assert(tag == TAG_MODEL);
models=(double*) buf;
debug_mpi("\tgot new ind" );
RecvMPIMessage(&buf, &size, 0, &tag, true);
// if(tag != TAG_PERTURB){
perturb=false;
assert(tag == TAG_SUBTREE_DEFINE);
subtreeNode=atoi(buf);
if(subtreeNode!=0){
delete []buf;
RecvMPIMessage(&buf, &size, 0, &tag, true);
assert(tag == TAG_SUBTREE_ITERATION);
recievedDefNumber=atoi(buf);
debug_mpi("\tworking on subtree def %d, node %d", recievedDefNumber, subtreeNode);
}
else recievedDefNumber=0;
}
else{
pop.pertMan->pertType=atoi(tree_strings);
perturb=true;
}
//if the current best and the new tree are either both accurate for the same subtree def or both
//inaccurate for subtrees, just replace the worst individual, rather than the
// whole pop, that way if the tree is old and worse that what the remote
// already has it won't matter
if(gotNewIndiv){
*which=(int)pop.cumfit[0][0];
debug_mpi("\treplacing indiv %d", *which);
pop.ReplaceSpecifiedIndividuals(1, which, tree_strings, models);
if(recievedDefNumber!=pop.subtreeDefNumber || (pop.subtreeNode!=0 && subtreeNode!=0)){
pop.AssignSubtree(subtreeNode, *which);
pop.CalcAverageFitness();
debug_mpi("\tfilling pop with clones of %d", *which);
pop.SetNewBestIndiv(*which);
pop.FillPopWithClonesOfBest();
pop.subtreeDefNumber=recievedDefNumber;
}
delete [] models;
delete [] buf;
}
#ifdef INCLUDE_PERTURBATION
if(perturb==true){
pop.CalcAverageFitness();
if(pop.pertMan->pertType==1){
debug_mpi("peforming NNI perturbation...");
int toReplace=(pop.bestIndiv == 0 ? 1 : 0);
pop.AppendTreeToTreeLog(-1, pop.bestIndiv);
pop.NNIPerturbation(pop.bestIndiv, toReplace);
pop.SetNewBestIndiv(toReplace);
pop.FillPopWithClonesOfBest();
pop.AppendTreeToTreeLog(-1, pop.bestIndiv);
}
else if(pop.pertMan->pertType==2){
debug_mpi("peforming SPR perturbation...");
int toReplace=(pop.bestIndiv == 0 ? 1 : 0);
pop.AppendTreeToTreeLog(-1, pop.bestIndiv);
pop.SPRPerturbation(pop.bestIndiv, toReplace);
pop.SetNewBestIndiv(toReplace);
pop.FillPopWithClonesOfBest();
pop.AppendTreeToTreeLog(-1, pop.bestIndiv);
}
else assert(0);
}
#endif
delete [] tree_strings;
tag=0;
firstmessage=false;
}
if(gotmessage==true){
// if(pop.subtreeNode != subtreeNode) pop.AssignSubtree(subtreeNode);
pop.CalcAverageFitness();
debug_mpi("\tbest score= %f", pop.indiv[*which].Fitness());
//DEBUG
//pop.AppendTreeToTreeLog(rank, -1, *which);
}
}
SendMPIMessage(NULL, 0, 0, TAG_QUIT);
debug_mpi("\tsent: quit message");
delete [] which;
debug_mpi("quitting");
return 0;
}