int main(int argc,char **argv)
{
//Override the number of threads if using MPI
HCUBE::NUM_THREADS=1;
char str[1024];
initcake(str);
SgInit();
GoInit();
int retval=0;
CommandLineParser commandLineParser(argc,argv);
try
{
/*
for(int a=0;a<argc;a++)
{
cout << "ARGUMENT: " << argv[a] << endl;
}
*/
int numtasks, rank, rc;
rc = MPI_Init(&argc,&argv);
if (rc != MPI_SUCCESS)
{
printf ("Error starting MPI program. Terminating.\n");
MPI_Abort(MPI_COMM_WORLD, rc);
return 1;
}
MPI_Comm_size(MPI_COMM_WORLD,&numtasks);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
cout << "NUM TASKS: " << numtasks << " RANK: " << rank << endl;
for (int a=0;a<argc;a++)
{
cout << "ARGUMENT: " << argv[a] << endl;
}
if(
commandLineParser.HasSwitch("-I") &&
commandLineParser.HasSwitch("-O")
)
{
NEAT::Globals::init(commandLineParser.GetSafeArgument("-I",0,"input.dat"));
if(commandLineParser.HasSwitch("-R"))
{
NEAT::Globals::getSingleton()->seedRandom(stringTo<unsigned int>(commandLineParser.GetSafeArgument("-R",0,"0")));
}
int experimentType = int(NEAT::Globals::getSingleton()->getParameterValue("ExperimentType")+0.001);
cout << "Loading Experiment: " << experimentType << endl;
HCUBE::MPIExperimentRun experimentRun;
experimentRun.setupExperiment(experimentType,commandLineParser.GetSafeArgument("-O",0,"output.xml"));
if(rank==0)
{
experimentRun.createPopulation();
}
cout << "Experiment set up\n";
runOrContinueExperiment(experimentRun);
}
else if(
commandLineParser.HasSwitch("-C") &&
commandLineParser.HasSwitch("-O")
)
{
HCUBE::MPIExperimentRun experimentRun;
string backupFile = commandLineParser.GetArgument("-C",0);
string outputFile = commandLineParser.GetArgument("-O",0);
//This also calls NEAT::Globals::init(...)
experimentRun.setupExperimentInProgress(backupFile,outputFile);
cout << "Experiment set up\n";
runOrContinueExperiment(experimentRun);
}
else
{
cout << rank << ") Invalid parameters (count: " << argc << ")!\n";
}
MPI_Finalize();
exitcake();
GoFini();
SgFini();
}
catch (const std::exception &ex)
{
cout << "An exception has occured: " << ex.what() << endl;
}
catch (string s)
{
cout << "An exception has occured: " << s << endl;
}
catch (...)
{
cout << "An error has occured!\n";
}
return 0;
}
int main(int argc,char **argv)
{
try
{
/*
for(int a=0;a<argc;a++)
{
cout << "ARGUMENT: " << argv[a] << endl;
}
*/
int numtasks, rank, rc;
rc = MPI_Init(&argc,&argv);
if (rc != MPI_SUCCESS)
{
printf ("Error starting MPI program. Terminating.\n");
MPI_Abort(MPI_COMM_WORLD, rc);
return 1;
}
MPI_Comm_size(MPI_COMM_WORLD,&numtasks);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
for (int a=0;a<argc;a++)
{
cout << "ARGUMENT: " << argv[a] << endl;
}
if (argc==1)
{
cout << "You must pass the parameters and the output file as command "
<< "parameters!\n";
}
/*
else if (argc==2)
{
//Run the post-hoc analysis on every generation
HCUBE::ExperimentRun experimentRun;
experimentRun.setupExperimentInProgress(
string(argv[1]),
""
);
int numGenerations = experimentRun.getPopulation()->getGenerationCount();
HCUBE::Experiment *experiment = experimentRun.getExperiment()->clone();
{
string outString = (erase_tail_copy(string(argv[1]),4)+string("_fitness.out"));
cout << "Creating file " << outString << endl;
ofstream outfile( outString.c_str() );
string previousSummary;
bool doOnce=false;
for (int generation=0;generation<numGenerations;generation++)
{
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
if ( generation &&
(*(experimentRun.getIndividual(generation,0).get())) == (*(experimentRun.getIndividual(generation-1,0).get())) )
{
outfile << (generation+1) << ' ' << previousSummary << endl;
continue;
}
shared_ptr<NEAT::GeneticIndividual> indiv =
shared_ptr<NEAT::GeneticIndividual>(
new NEAT::GeneticIndividual(
*(experimentRun.getIndividual(generation,0).get())
)
);
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
cout << "Beginning fitness evaluation " << (generation+1) << "/" << numGenerations << "...";
experiment->addIndividualToGroup(indiv);
experiment->processGroup(experimentRun.getGeneration(generation));
experiment->clearGroup();
cout << "done!\n";
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
if (indiv->getUserData())
{
if (!doOnce)
{
doOnce=true;
outfile
<< "#Generation: "
<< indiv->getUserData()->summaryHeaderToString()
<< endl;
}
previousSummary = indiv->getUserData()->summaryToString();
outfile << (generation+1) << ' ' << previousSummary << endl;
}
else
{
throw CREATE_LOCATEDEXCEPTION_INFO("No user data!\n");
}
}
}
{
string outString = (erase_tail_copy(string(argv[1]),4)+string("_lowres.out"));
cout << "Creating file " << outString << endl;
ofstream outfile( outString.c_str() );
string previousSummary;
bool doOnce=false;
for (int generation=0;generation<numGenerations;generation++)
{
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
if ( generation &&
(*(experimentRun.getIndividual(generation,0).get())) == (*(experimentRun.getIndividual(generation-1,0).get())) )
{
outfile << (generation+1) << ' ' << previousSummary << endl;
continue;
}
shared_ptr<NEAT::GeneticIndividual> indiv =
shared_ptr<NEAT::GeneticIndividual>(
new NEAT::GeneticIndividual(
*(experimentRun.getIndividual(generation,0).get())
)
);
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
cout << "Beginning post-hoc evaluation " << (generation+1) << "/" << numGenerations << "...";
experiment->processIndividualPostHoc(indiv);
cout << "done!\n";
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
if (indiv->getUserData())
{
if (!doOnce)
{
doOnce=true;
outfile
<< "#Generation: "
<< indiv->getUserData()->summaryHeaderToString()
<< endl;
}
previousSummary = indiv->getUserData()->summaryToString();
outfile << (generation+1) << ' ' << previousSummary << endl;
}
}
}
cout << "Done with run! Running medium resolution tests...\n";
((HCUBE::FindClusterExperiment*)experiment)->increaseResolution();
{
string outString = (erase_tail_copy(string(argv[1]),4)+string("_midres.out"));
ofstream outfile( outString.c_str() );
cout << "Creating file " << outString << endl;
string previousSummary;
bool doOnce=false;
for (int generation=0;generation<numGenerations;generation++)
{
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
if ( generation &&
(*(experimentRun.getIndividual(generation,0).get())) == (*(experimentRun.getIndividual(generation-1,0).get())) )
{
outfile << (generation+1) << ' ' << previousSummary << endl;
continue;
}
shared_ptr<NEAT::GeneticIndividual> indiv =
shared_ptr<NEAT::GeneticIndividual>(
new NEAT::GeneticIndividual(
*(experimentRun.getIndividual(generation,0).get())
)
);
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
cout << "Beginning post-hoc evaluation " << (generation+1) << "/" << numGenerations << "...";
experiment->processIndividualPostHoc(indiv);
cout << "done!\n";
//CREATE_PAUSE(string("Error!: ")+toString(__LINE__));
if (indiv->getUserData())
{
if (!doOnce)
{
doOnce=true;
outfile
<< "#Generation: "
<< indiv->getUserData()->summaryHeaderToString()
<< endl;
}
previousSummary = indiv->getUserData()->summaryToString();
outfile << (generation+1) << ' ' << previousSummary << endl;
}
}
}
}
*/
else if (argc==3)
{
NEAT::Globals::init(string(argv[1]));
int experimentType = int(NEAT::Globals::getSingleton()->getParameterValue("ExperimentType")+0.001);
cout << "Loading Experiment: " << experimentType << endl;
HCUBE::MPIExperimentRun experimentRun;
experimentRun.setupExperiment(experimentType,string(argv[2]));
cout << "Experiment set up\n";
if (rank==0)
{
experimentRun.createPopulation();
experimentRun.setCleanup(true);
cout << "Population Created\n";
experimentRun.start();
}
else
{
shared_ptr<HCUBE::Experiment> experiment(
experimentRun.getExperiment()->clone()
);
//cout << "Experiment pointer: " << experiment << endl;
char *buffer=NULL;
int bufferSize=0;
while (true)
{
#if DEBUG_MPI_MAIN
cout << "Listening for individual chunk size...\n";
#endif
int msgSize;
{
MPI_Status Stat;
MPI_Recv (&msgSize,1,MPI_INT,0,1,MPI_COMM_WORLD,&Stat);
}
#if DEBUG_MPI_MAIN
cout << "Got chunk size: " << msgSize << endl;
#endif
if (msgSize==-1)
{
break;
}
if (msgSize==0)
{
continue; // ???? This shouldn't happen, but handle anyways
}
if (bufferSize<msgSize)
{
bufferSize = msgSize;
buffer = (char*)realloc(buffer,bufferSize);
}
memset(buffer,0,bufferSize);
#if DEBUG_MPI_MAIN
cout << "Getting buffer...\n";
#endif
{
MPI_Status Stat;
MPI_Recv (buffer,msgSize,MPI_CHAR,0,1,MPI_COMM_WORLD,&Stat);
}
#if DEBUG_MPI_MAIN
cout << "Got Buffer\n";
//cout << buffer << endl;
#endif
istringstream istr(buffer);
#if DEBUG_MPI_MAIN
cout << "Loaded stringstream\n";
#endif
int testCount;
istr >> testCount;
shared_ptr<NEAT::GeneticGeneration> generation;
{
generation = shared_ptr<NEAT::GeneticGeneration>(
new NEAT::GeneticGeneration(0)
);
}
#if DEBUG_MPI_MAIN
cout << "Test count: " << testCount << endl;
#endif
int individualCount;
istr >> individualCount;
#if DEBUG_MPI_MAIN
cout << "Individualcount: " << individualCount << endl;
#endif
double *newFitness = (double*)malloc(sizeof(double)*individualCount);
#if DEBUG_MPI_MAIN
cout << "Fitness buffer created\n";
#endif
for (int a=0;a<individualCount;a+=experiment->getGroupCapacity())
{
for (int b=0;b<experiment->getGroupCapacity();b++)
{
shared_ptr<NEAT::GeneticIndividual> ind(
new NEAT::GeneticIndividual(istr)
);
experiment->addIndividualToGroup(ind);
}
experiment->processGroup(generation);
for (int b=0;b<experiment->getGroupCapacity();b++)
{
newFitness[a+b] = experiment->getGroupMember(b)->getFitness();
}
//Clear the experiment for the next individuals
experiment->clearGroup();
}
#if DEBUG_MPI_MAIN
cout << "Sending new fitness values\n";
#endif
MPI_Send (newFitness,individualCount,MPI_DOUBLE,0,1,MPI_COMM_WORLD);
#if DEBUG_MPI_MAIN
cout << "Cleaning up new fitness values\n";
#endif
free(newFitness);
}
}
}
else
{
cout << "Invalid # of parameters (" << argc << ")!\n";
}
MPI_Finalize();
}
void runOrContinueExperiment(HCUBE::MPIExperimentRun &experimentRun)
{
int numtasks, rank;
MPI_Comm_size(MPI_COMM_WORLD,&numtasks);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
if (rank==0)
{
experimentRun.setCleanup(true);
cout << rank << ") Population Created\n";
experimentRun.start();
int numThreads;
MPI_Comm_size(MPI_COMM_WORLD,&numThreads);
cout << "Number of threads to kill: " << numThreads << endl;
for (rank = 1; rank < numThreads; ++rank)
{
MPI_Send(0, 0, MPI_INT, rank, DIE_TAG, MPI_COMM_WORLD);
}
}
else
{
shared_ptr<HCUBE::Experiment> experiment(
experimentRun.getExperiment()->clone()
);
//cout << "Experiment pointer: " << experiment << endl;
char *buffer=NULL;
int bufferSize=0;
int curGenNumber=0;
if(experimentRun.getPopulation())
{
cout << "Resuming at generation: " << experimentRun.getPopulation()->getGenerationCount()-1 << endl;
curGenNumber=experimentRun.getPopulation()->getGenerationCount()-1;
}
for(;;curGenNumber++)
{
#if DEBUG_MPI_MAIN
cout << rank << ") Listening for individual chunk size...\n";
#endif
int msgSize;
{
MPI_Status Stat;
MPI_Recv (&msgSize,1,MPI_INT,0,MPI_ANY_TAG,MPI_COMM_WORLD,&Stat);
if(Stat.MPI_TAG==DIE_TAG)
{
break;
}
}
#if DEBUG_MPI_MAIN
cout << rank << ") Got chunk size: " << msgSize << endl;
#endif
if (msgSize==-1)
{
break;
}
if (msgSize==0)
{
continue; // ???? This shouldn't happen, but handle anyways
}
if (bufferSize<msgSize)
{
bufferSize = msgSize;
buffer = (char*)realloc(buffer,bufferSize);
}
memset(buffer,0,bufferSize);
#if DEBUG_MPI_MAIN
cout << rank << ") Getting buffer...\n";
#endif
{
MPI_Status Stat;
MPI_Recv (buffer,msgSize,MPI_CHAR,0,INDIVIDUAL_GENOMES_TAG,MPI_COMM_WORLD,&Stat);
}
#if DEBUG_MPI_MAIN
cout << rank << ") Got Buffer\n";
//cout << buffer << endl;
#endif
istringstream istr(buffer);
#if DEBUG_MPI_MAIN
cout << rank << ") Loaded stringstream\n";
#endif
int testCount;
istr >> testCount;
shared_ptr<NEAT::GeneticGeneration> generation;
#ifdef EPLEX_INTERNAL
if(testCount)
{
generation = shared_ptr<NEAT::GeneticGeneration>(
new NEAT::CoEvoGeneticGeneration(
0,
dynamic_pointer_cast<NEAT::CoEvoExperiment>(experiment)
)
);
}
else
#endif
{
generation = shared_ptr<NEAT::GeneticGeneration>(
new NEAT::GeneticGeneration(curGenNumber)
);
}
#if DEBUG_MPI_MAIN
cout << rank << ") Test count: " << testCount << endl;
#endif
#ifdef EPLEX_INTERNAL
for(int a=0;a<testCount;a++)
{
shared_ptr<NEAT::GeneticIndividual> testInd(new NEAT::GeneticIndividual(istr));
shared_ptr<NEAT::CoEvoGeneticGeneration> coEvoGen =
static_pointer_cast<NEAT::CoEvoGeneticGeneration>(generation);
coEvoGen->addTestHack(testInd);
}
#endif
int individualCount;
istr >> individualCount;
#if DEBUG_MPI_MAIN
cout << rank << ") Individualcount: " << individualCount << endl;
#endif
//double *newFitness = (double*)malloc(sizeof(double)*individualCount);
ostringstream ostr;
#if DEBUG_MPI_MAIN
cout << rank << ") Fitness buffer created\n";
#endif
for (int a=0;a<individualCount;a+=experiment->getGroupCapacity())
{
#if DEBUG_MPI_MAIN
cout << rank << ") Adding to group...\n";
#endif
for (int b=0;b<experiment->getGroupCapacity();b++)
{
shared_ptr<NEAT::GeneticIndividual> ind(
new NEAT::GeneticIndividual(istr)
);
experiment->addIndividualToGroup(ind);
}
#if DEBUG_MPI_MAIN
cout << rank << ") Processing...\n";
#endif
experiment->processGroup(generation);
#if DEBUG_MPI_MAIN
cout << rank << ") Dumping...\n";
#endif
for (int b=0;b<experiment->getGroupCapacity();b++)
{
experiment->getGroupMember(b)->dump(ostr);
//newFitness[a+b] = experiment->getGroupMember(b)->getFitness();
}
#if DEBUG_MPI_MAIN
cout << rank << ") Clearing...\n";
#endif
//Clear the experiment for the next individuals
experiment->clearGroup();
#if DEBUG_MPI_MAIN
cout << rank << ") End of process...\n";
#endif
}
#if DEBUG_MPI_MAIN
cout << rank << ") Sending new fitness values\n";
#endif
string str = ostr.str();
char *buffer = new char[str.length()+1];
memcpy(buffer,str.c_str(),str.length());
buffer[str.length()] = '\0'; //null terminate
int lengthInt = (int)str.length() + 1;
#if DEBUG_MPI_MAIN
cout << rank << ") Sending message of size " << lengthInt << "\n";
#endif
//MPI_Send (newFitness,individualCount,MPI_DOUBLE,0,NEW_FITNESSES_TAG,MPI_COMM_WORLD);
MPI_Send (&lengthInt,1,MPI_INT,0,NEW_INDIVIDUALS_TAG,MPI_COMM_WORLD);
MPI_Send (buffer,lengthInt,MPI_CHAR,0,NEW_INDIVIDUALS_TAG,MPI_COMM_WORLD);
#if DEBUG_MPI_MAIN
cout << rank << ") Cleaning up new fitness values\n";
#endif
//free(newFitness);
}
}
}