std::string run_model(Properties& props, boost::mpi::communicator& world) {
try {
repast::relogo::SimulationRunner runner(&world);
if (world.rank() == 0) {
std::string time;
repast::timestamp(time);
std::cout << "Start Time: " << time << std::endl;
}
repast::Timer timer;
timer.start();
runner.run<ZombieObserver, repast::relogo::Patch>(props);
if (world.rank() == 0) {
std::string time;
repast::timestamp(time);
std::cout << "End Time: " << time << "\nElapsed Time: "
<< timer.stop() << std::endl;
}
} catch (std::exception& exp) {
// catch any exception (e.g. if data files couldn't be opened) and
// print out the errors.
std::cerr << "ERROR: " << exp.what() << std::endl;
}
return props.getProperty(OUTPUT_KEY);
}
void generate_data(mpi::communicator local, mpi::communicator world)
{
using std::srand;
using std::rand;
// The rank of the collector within the world communicator
int master_collector = local.size();
srand(time(0) + world.rank());
// Send out several blocks of random data to the collectors.
int num_data_blocks = rand() % 3 + 1;
for (int block = 0; block < num_data_blocks; ++block) {
// Generate some random data
int num_samples = rand() % 1000;
std::vector<int> data;
for (int i = 0; i < num_samples; ++i) {
data.push_back(rand());
}
// Send our data to the master collector process.
std::cout << "Generator #" << local.rank() << " sends some data..."
<< std::endl;
world.send(master_collector, msg_data_packet, data);
}
// Wait for all of the generators to complete
(local.barrier)();
// The first generator will send the message to the master collector
// indicating that we're done.
if (local.rank() == 0)
world.send(master_collector, msg_finished);
}
void
printit(const boost::mpi::communicator& comm,
const std::vector<T>& things,
const std::string& caption)
{
if (!caption.empty() && comm.rank() == 0) {
std::cout << caption << std::endl;
std::cout.flush();
}
for (int p = 0; p < comm.size(); ++p) {
if (comm.rank() == p) {
std::cout << p << ": ";
std::copy(things.begin(), things.end(),
std::ostream_iterator<T>(std::cout, ","));
std::cout << std::endl;
std::cout.flush();
}
comm.barrier();
}
comm.barrier();
size_t global_size;
boost::mpi::reduce(comm, things.size(), global_size, std::plus<size_t>(), 0);
if (comm.rank() == 0) {
if (!caption.empty()) {
std::cout << caption;
}
std::cout << "Number of things: " << global_size << std::endl;
}
comm.barrier();
std::cout << comm.rank() << ": leaving printit()" << std::endl;
}
void
all_gatherv_test(const mpi::communicator& comm, Generator generator,
std::string kind)
{
typedef typename Generator::result_type value_type;
using boost::mpi::all_gatherv;
std::vector<value_type> myvalues, expected, values;
std::vector<int> sizes;
for(int r = 0; r < comm.size(); ++r) {
value_type value = generator(r);
sizes.push_back(r+1);
for (int k=0; k < r+1; ++k) {
expected.push_back(value);
if(comm.rank() == r) {
myvalues.push_back(value);
}
}
}
if (comm.rank() == 0) {
std::cout << "Gathering " << kind << "...";
std::cout.flush();
}
mpi::all_gatherv(comm, myvalues, values, sizes);
BOOST_CHECK(values == expected);
if (comm.rank() == 0 && values == expected)
std::cout << "OK." << std::endl;
(comm.barrier)();
}
void collect_data(mpi::communicator local, mpi::communicator world)
{
// The rank of the collector within the world communicator
int master_collector = world.size() - local.size();
if (world.rank() == master_collector) {
while (true) {
// Wait for a message
mpi::status msg = world.probe();
if (msg.tag() == msg_data_packet) {
// Receive the packet of data
std::vector<int> data;
world.recv(msg.source(), msg.tag(), data);
// Tell each of the collectors that we'll be broadcasting some data
for (int dest = 1; dest < local.size(); ++dest)
local.send(dest, msg_broadcast_data, msg.source());
// Broadcast the actual data.
broadcast(local, data, 0);
} else if (msg.tag() == msg_finished) {
// Receive the message
world.recv(msg.source(), msg.tag());
// Tell each of the collectors that we're finished
for (int dest = 1; dest < local.size(); ++dest)
local.send(dest, msg_finished);
break;
}
}
} else {
while (true) {
// Wait for a message from the master collector
mpi::status msg = local.probe();
if (msg.tag() == msg_broadcast_data) {
// Receive the broadcast message
int originator;
local.recv(msg.source(), msg.tag(), originator);
// Receive the data broadcasted from the master collector
std::vector<int> data;
broadcast(local, data, 0);
std::cout << "Collector #" << local.rank()
<< " is processing data from generator #" << originator
<< "." << std::endl;
} else if (msg.tag() == msg_finished) {
// Receive the message
local.recv(msg.source(), msg.tag());
break;
}
}
}
}
void TwoParticleGF::compute(bool clear, const boost::mpi::communicator & comm)
{
if (Status < Prepared) throw (exStatusMismatch());
if (Status >= Computed) return;
if (!Vanishing) {
// Create a "skeleton" class with pointers to part that can call a compute method
pMPI::mpi_skel<ComputeAndClearWrap> skel;
bool fill_container = m_data_.NBosonic() > 0 && m_data_.NFermionic() > 0;
skel.parts.reserve(parts.size());
for (size_t i=0; i<parts.size(); i++) {
skel.parts.push_back(ComputeAndClearWrap(&m_data_, parts[i], clear, fill_container, 1));
};
std::map<pMPI::JobId, pMPI::WorkerId> job_map = skel.run(comm, true); // actual running - very costly
int rank = comm.rank();
int comm_size = comm.size();
// Start distributing data
//DEBUG(comm.rank() << getIndex(0) << getIndex(1) << getIndex(2) << getIndex(3) << " Start distributing data");
comm.barrier();
if (!clear) {
for (size_t p = 0; p<parts.size(); p++) {
boost::mpi::broadcast(comm, parts[p]->NonResonantTerms, job_map[p]);
boost::mpi::broadcast(comm, parts[p]->ResonantTerms, job_map[p]);
if (rank == job_map[p]) {
parts[p]->Status = TwoParticleGFPart::Computed;
};
};
comm.barrier();
}
};
Status = Computed;
}
void print_section (const std::string& str)
{
if (!comm.rank()) {
std::cout << std::string(str.size(),'=') << std::endl;
std::cout << str << std::endl;
std::cout << std::string(str.size(),'=') << std::endl;
};
}
void reduce_and_check(const boost::mpi::communicator &comm, bool local_value) {
if(comm.rank() == 0) {
bool total;
boost::mpi::reduce(comm, local_value, total, std::logical_and<bool>(), 0);
BOOST_CHECK(total);
} else {
boost::mpi::reduce(comm, local_value, std::logical_and<bool>(), 0);
}
}
void gather_resample_weight() const
{
weight_.resize(this->size());
this->read_weight(weight_.data());
if (world_.rank() == 0)
::boost::mpi::gather(world_, weight_, weight_all_, 0);
else
::boost::mpi::gather(world_, weight_, 0);
}
void
report_features(mpi::communicator const& comm) {
if (comm.rank() == 0) {
std::cout << "Assuming working MPI_Improbe:" <<
#if defined(BOOST_MPI_USE_IMPROBE)
"yes" << '\n';
#else
"no" << '\n';
#endif
}
}
void TwoParticleGFContainer::computeAll_split(bool clearTerms, const boost::mpi::communicator & comm)
{
// split communicator
size_t ncomponents = NonTrivialElements.size();
size_t ncolors = std::min(int(comm.size()), int(NonTrivialElements.size()));
RealType color_size = 1.0*comm.size()/ncolors;
std::map<int,int> proc_colors;
std::map<int,int> elem_colors;
std::map<int,int> color_roots;
bool calc = false;
for (size_t p=0; p<comm.size(); p++) {
int color = int (1.0*p / color_size);
proc_colors[p] = color;
color_roots[color]=p;
}
for (size_t i=0; i<ncomponents; i++) {
int color = i*ncolors/ncomponents;
elem_colors[i] = color;
};
if (!comm.rank()) {
INFO("Splitting " << ncomponents << " components in " << ncolors << " communicators");
for (size_t i=0; i<ncomponents; i++)
INFO("2pgf " << i << " color: " << elem_colors[i] << " color_root: " << color_roots[elem_colors[i]]);
};
comm.barrier();
int comp = 0;
boost::mpi::communicator comm_split = comm.split(proc_colors[comm.rank()]);
for(std::map<IndexCombination4, boost::shared_ptr<TwoParticleGF> >::iterator iter = NonTrivialElements.begin(); iter != NonTrivialElements.end(); iter++, comp++) {
bool calc = (elem_colors[comp] == proc_colors[comm.rank()]);
if (calc) {
INFO("C" << elem_colors[comp] << "p" << comm.rank() << ": computing 2PGF for " << iter->first);
if (calc) static_cast<TwoParticleGF&>(*(iter->second)).compute(clearTerms, comm_split);
};
};
comm.barrier();
// distribute data
if (!comm.rank()) INFO_NONEWLINE("Distributing 2PGF container...");
comp = 0;
for(std::map<IndexCombination4, boost::shared_ptr<TwoParticleGF> >::iterator iter = NonTrivialElements.begin(); iter != NonTrivialElements.end(); iter++, comp++) {
int sender = color_roots[elem_colors[comp]];
TwoParticleGF& chi = *((iter)->second);
for (size_t p = 0; p<chi.parts.size(); p++) {
// if (comm.rank() == sender) INFO("P" << comm.rank() << " 2pgf " << p << " " << chi.parts[p]->NonResonantTerms.size());
boost::mpi::broadcast(comm, chi.parts[p]->NonResonantTerms, sender);
boost::mpi::broadcast(comm, chi.parts[p]->ResonantTerms, sender);
if (comm.rank() != sender) {
chi.setStatus(TwoParticleGF::Computed);
};
};
}
comm.barrier();
if (!comm.rank()) INFO("done.");
}
// Posle pozadavek o praci nahodne vybranemu procesu
void sendRequest() {
int pn = rand() % com.size();
srand(time(NULL));
// Pokud by se nahodne cislo trefilo na moje cislo, generuj nove
if (pn == com.rank()) {
pn = rand() % com.size();
}
//cout << "Sending WORK_REQUEST to " << pn << endl;
com.send(pn, WORK_REQUEST);
}
/// Reduce the results of the measures, and reports some statistics
void collect_results(boost::mpi::communicator const & c) {
uint64_t nmeasures_tot;
MCSignType sum_sign_tot;
boost::mpi::reduce(c, nmeasures, nmeasures_tot, std::plus<uint64_t>(), 0);
boost::mpi::reduce(c, sum_sign, sum_sign_tot, std::plus<MCSignType>(), 0);
report(3) << "[Node "<<c.rank()<<"] Acceptance rate for all moves:\n" << AllMoves.get_statistics(c);
report(3) << "[Node "<<c.rank()<<"] Simulation lasted: " << double(Timer) << " seconds" << std::endl;
report(3) << "[Node "<<c.rank()<<"] Number of measures: " << nmeasures << std::endl;
report(3) << "[Node "<<c.rank()<<"] Average sign: " << sum_sign / double(nmeasures) << std::endl << std::endl << std::flush;
if (c.rank()==0) {
sign_av = sum_sign_tot / double(nmeasures_tot);
report(2) << "Total number of measures: " << nmeasures_tot << std::endl;
report(2) << "Average sign: " << sign_av << std::endl << std::endl << std::flush;
}
boost::mpi::broadcast(c, sign_av, 0);
AllMeasures.collect_results(c);
}
int main()
{
int numClients = 2;
// client settings
float minClientActionDelay = 0.2f; // min wait time for a client before starting new actions
float maxClientActionDelay = 0.4f; // max wait time for a client before it has to complete a new action
float clientQueryWeight = 30.0f; // possibility of a query to happen
float clientReplyWeight = 30.0f; // possibility of a reply to happen
float clientPostWeight = 40.0f; // possibility of a new post (update) to happen
// set the global MPI variabes
gRank = gWorld.rank();
gNumFE = numClients;
gNumRM = gWorld.size() - numClients;
// early out if there are not enough nodes for at least one RM
if(numClients + 1 > gWorld.size() && gWorld.rank() == 0) {
std::cout << "ERROR: there are not enough nodes for at least 1 RM, please increase the number of nodes" << std::endl;
exit(-1);
}
if (gWorld.rank() == 0) {
std::cout << " num RM: " << gNumRM << " num FE: " << gNumFE << std::endl;
}
Log::singleton().open("Bulletin_" + std::to_string(gRank) + ".log"); // set log file
Log::singleton().setVerbosity(LV_Normal);
//the last 'numClients' ranks are front ends
if (gWorld.rank() >= gWorld.size() - numClients) {
std::cout << "P" << gWorld.rank() << ": assigned as a client" << std::endl;
// create client instance
Client client;
// set client variables as defined above
client.setMinActionDelay(minClientActionDelay);
client.setMaxActionDelay(maxClientActionDelay);
client.setQueryWeight(clientQueryWeight);
client.setReplyWeight(clientReplyWeight);
client.setPostWeight(clientPostWeight);
// run the client
// the client will now call the Frontend classes specific functions
// whenever it wants to complete an action.
client.run();
}
else
{
std::cout << "P" << gWorld.rank() << ": assigned as a replicator" << std::endl;
ReplicaManager RM;
RM.run();
}
return 0;
}
void
all_gather_test(const mpi::communicator& comm, Generator generator,
std::string kind)
{
typedef typename Generator::result_type value_type;
value_type value = generator(comm.rank());
std::vector<value_type> values;
if (comm.rank() == 0) {
std::cout << "Gathering " << kind << "...";
std::cout.flush();
}
mpi::all_gather(comm, value, values);
std::vector<value_type> expected_values;
for (int p = 0; p < comm.size(); ++p)
expected_values.push_back(generator(p));
BOOST_CHECK(values == expected_values);
if (comm.rank() == 0 && values == expected_values)
std::cout << "OK." << std::endl;
(comm.barrier)();
}
void
broadcast_test(const mpi::communicator& comm, const T& bc_value,
std::string const& kind, int root) {
using boost::mpi::broadcast;
T value;
if (comm.rank() == root) {
value = bc_value;
std::cout << "Broadcasting " << kind << " from root " << root << "...";
std::cout.flush();
}
broadcast(comm, value, root);
BOOST_CHECK(value == bc_value);
if (comm.rank() == root) {
if (value == bc_value) {
std::cout << "OK." << std::endl;
} else {
std::cout << "FAIL." << std::endl;
}
}
comm.barrier();
}
// Hlavni metoda hledani podsekvence.
// Generuje nove stringy z pocatecniho stringu a urcuje, jestli se jedna o podsekvence
void findAndGenerate() {
stack<string> inputStack;
unsigned int counter = 0;
// Vlozeni pocatecniho stringu na zasobnik
inputStack.push(data.startString);
cout << com.rank() << ": starting on: " << data.startString << endl;
while (!inputStack.empty()) {
// Jednou za cas provede kontrolu prichozich zprav
counter = (counter + 1) % 100;
if (counter == 0) {
// Pokud prijme zpravu o nalezeni nejlepsiho mozneho vysledku, ukonci praci
if (activeRecv(inputStack)) {
return;
}
}
string current = inputStack.top();
inputStack.pop();
// Behem hledani byla nalezena maximalni mozna podsekvence.
if (findSubsequence(current, inputStack)) {
cout << com.rank() << ": found best: " << myLongest << endl;
broadcastMessage(FOUND_BEST);
return;
}
// Generuje vetsi stringy, dokud se nedosahne delky nejkratsiho ze vstupnich stringu
if (current.length() < data.shortestLen) {
generateSubsequences(current, inputStack);
}
}
cout << com.rank() << ": found : " << myLongest << endl;
}
// Vyrizeni pozadavku o praci
void handleWorkRequest(int source, stack<string>& workStack) {
string response = splitStack(workStack);
// Nemam co poslat
if (response.empty()) {
//cout << "Sending NO_WORK to " << source << endl;
com.send(source, NO_WORK);
} else {
// Pokud bych posilal data procesu s nizsim indexem, prebarvim se na cerno
if (source < com.rank()) {
isWhiteProcess = false;
}
//cout << "Sending DATA to " << source << endl;
com.send(source, DATA, response);
}
}
void
test_version(mpi::communicator const& comm) {
#if defined(MPI_VERSION)
int mpi_version = MPI_VERSION;
int mpi_subversion = MPI_SUBVERSION;
#else
int mpi_version = 0;
int mpi_subversion = 0;
#endif
std::pair<int,int> version = mpi::environment::version();
if (comm.rank() == 0) {
std::cout << "MPI Version: " << version.first << ',' << version.second << '\n';
}
BOOST_CHECK(version.first == mpi_version);
BOOST_CHECK(version.second == mpi_subversion);
}
void FieldOperator::compute(const boost::mpi::communicator& comm)
{
if (Status < Prepared) throw (exStatusMismatch());
if (Status >= Computed) return;
if (!comm.rank()) INFO_NONEWLINE("Computing " << *O << " in eigenbasis of the Hamiltonian: ");
/*
pMPI::mpi_skel<pMPI::ComputeWrap<FieldOperatorPart>> skel;
skel.parts.resize(parts.size());
for (size_t i=0; i<parts.size(); i++) { skel.parts[i] = pMPI::ComputeWrap<FieldOperatorPart>(*parts[i]);};
std::map<pMPI::JobId, pMPI::WorkerId> job_map = skel.run(comm, true); // actual running - very costly
int rank = comm.rank();
int comm_size = comm.size();
// Start distributing data
comm.barrier();
for (size_t p = 0; p<parts.size(); p++) {
int nelem = 0;
if (rank == job_map[p]) nelem = parts[p]->elementsColMajor.nonZeros();
boost::mpi::broadcast(comm, nelem, job_map[p]);
auto data = parts[p]->elementsColMajor.data(); // Eigen::internal::CompressedStorage
auto& value_start = data.value(0);
auto& index_start = data.index(0);
if (comm.rank()) parts[p]->elementsColMajor.resize(nelem);
exit(0);
//boost::mpi::broadcast(comm, parts[p]->elementsColMajor.data(), nelem, job_map[p]);
if (rank == job_map[p]) {
parts[p]->Status = FieldOperatorPart::Computed;
};
};
comm.barrier();
*/
size_t Size = parts.size();
for (size_t BlockIn = 0; BlockIn < Size; BlockIn++){
INFO_NONEWLINE( (int) ((1.0*BlockIn/Size) * 100 ) << " " << std::flush);
parts[BlockIn]->compute();
};
INFO("");
Status = Computed;
}
harp::mpi_spec::mpi_spec ( boost::mpi::communicator const & comm, std::string const & type, boost::property_tree::ptree const & props ) {
comm_ = comm;
int rank = comm.rank();
plugin_registry & reg = plugin_registry::get();
if ( rank == 0 ) {
// instantiate
local_.reset ( reg.create_spec ( type, props ) );
}
// broadcast to all processes
mpi_comm_bcast ( comm_, local_, 0 );
}
// Poslani zpravy vsem ostatnim procesum
void broadcastMessage(int msgType) {
for (int i = 0; i < com.size(); i++) {
// Sobe nic neposilam
if (i == com.rank()) { continue; }
// Pokud jsem nasel vysledek, poslu ho
if (msgType == FOUND || msgType == FOUND_BEST) {
//cout << "Sending (BEST)FOUND to " << i << endl;
com.send(i, msgType, myLongest);
}
// Pri oznameni konce vypoctu neni treba posilat zadna data
else if (msgType == END) {
//cout << "Sending end to " << i << endl;
com.send(i, msgType);
}
}
}
void
random_scattered_vector(const boost::mpi::communicator& comm,
const int& global_size,
std::vector<I>& local_values)
{
int me(comm.rank());
int nproc(comm.size());
std::vector< std::vector<I> > toscatter(nproc);
if (me == 0) {
for (int i = 0; i < global_size; ++i) {
boost::random::uniform_int_distribution<> dist(0, nproc-1);
int p(dist(gen));
toscatter[p].push_back(i);
}
}
scatter(comm, toscatter, local_values, 0);
}
static void master(mpi::communicator world){
int ntasks, rank;
vector<int> data;
int work;
int result;
for(int i = 0; i< 10; i++){
data.push_back(i);
}
const int size_work = (int)data.size();
rank = world.rank(); //int rank(ID) of processor
ntasks = world.size();//int total number of processors
for (rank = 1; rank < ntasks; ++rank) {
get_next_work_item(work,size_work,data);
world.send(rank,WORKTAG,work);
}
int ret = get_next_work_item(work,size_work,data);
while (ret == 0){
mpi::status status = world.recv(mpi::any_source,mpi::any_tag,result);
world.send(status.source(),WORKTAG,work);
ret = get_next_work_item(work,size_work,data);
}
for (rank = 1; rank < ntasks; ++rank) {
world.recv( mpi::any_source, mpi::any_tag,result);
}
for (rank = 1; rank < ntasks; ++rank) {
world.send(rank,DIETAG,0);
}
}
void mpi_broadcast_workaround(const boost::mpi::communicator& comm,
T& value, int root)
{
if(comm.rank() == root)
{
// serialize T into a string
std::ostringstream oss;
boost::archive::text_oarchive oa(oss);
oa << value;
std::string s = oss.str();
boost::mpi::broadcast(comm, s, root);
}
else
{
std::string s;
boost::mpi::broadcast(comm, s, root);
// serialize into T
std::istringstream iss(s);
boost::archive::text_iarchive ia(iss);
ia >> value;
}
}
scalar_type sum_counts(Species_tree * infer_tree,const mpi::communicator world)
{
//########################## GLOBAL #########################
int server = 0;
int rank = world.rank();
int size = world.size();
scalar_type client_ll,broadcast_ll;
vector<scalar_type> gather_ll;
scalar_type client_pea_count;
scalar_type broadcast_pea_count;
vector<scalar_type> gather_pea_count;
scatter_scalar gather_O_counts;
scatter_scalar gather_D_counts;
scatter_scalar gather_T_counts;
scatter_scalar gather_L_counts;
scatter_scalar gather_B_counts;
scatter_scalar gather_S_counts;
scatter_scalar gather_F_counts;
scatter_scalar gather_G_counts;
//vector< vector< vector<scalar_type> > > gather_Ttf;
vector<scalar_type> broadcast_O_counts;
vector<scalar_type> broadcast_D_counts;
vector<scalar_type> broadcast_T_counts;
vector<scalar_type> broadcast_branch_zero;
vector<scalar_type> broadcast_branch_count;
vector<scalar_type> broadcast_branch_sum;
vector<scalar_type> broadcast_from_count;
vector<scalar_type> broadcast_genome_size;
int start=1;
//########################## GLOBAL #########################
//########################## SERVER #########################
//########################## SERVER #########################
if (rank != server)
{
//########################## CLIENT #########################
client_pea_count=infer_tree->tmp_pea_sum;
client_ll=infer_tree->tmp_ll;
//########################## CLIENT #########################
}
//########################## COMMON #########################
gather(world, infer_tree->O_counts, gather_O_counts, server);
gather(world, infer_tree->D_counts, gather_D_counts, server);
gather(world, infer_tree->T_counts, gather_T_counts, server);
gather(world, infer_tree->branch_zero, gather_L_counts, server);
gather(world, infer_tree->branch_count, gather_B_counts, server);
gather(world, infer_tree->branch_sum, gather_S_counts, server);
gather(world, infer_tree->from_count, gather_F_counts, server);
gather(world, infer_tree->branch_genome_size, gather_G_counts, server);
//TTF
// gather(world, infer_tree->Ttf, gather_Ttf, server);
gather(world, client_pea_count, gather_pea_count, server);
gather(world, client_ll, gather_ll, server);
//########################## COMMON #########################
if (rank == server)
{
//########################## SERVER #########################
for (int i = 0;i<infer_tree->N_slices+infer_tree->tree->getNumberOfLeaves()-1;i++)
{
broadcast_O_counts.push_back(0);
broadcast_D_counts.push_back(0);
broadcast_T_counts.push_back(0);
broadcast_branch_zero.push_back(0);
broadcast_branch_count.push_back(0);
broadcast_branch_sum.push_back(0);
broadcast_from_count.push_back(0);
broadcast_genome_size.push_back(0);
}
scalar_type tmp_O=0;
scalar_type tmp_D=0;
scalar_type tmp_T=0;
scalar_type tmp_z=0;
scalar_type tmp_c=0;
scalar_type tmp_s=0;
scalar_type tmp_f=0;
//cout << " start summing - sum " <<endl;
//for (int i = 0;i<infer_tree->N_slices+infer_tree->tree->getNumberOfLeaves()-1;i++)
// for (int k = 0;k<infer_tree->N_slices+infer_tree->tree->getNumberOfLeaves()-1;k++)
// infer_tree->Ttf[i][k]=0;
for (int j=1;j<size;j++)
{
for (int i = 0;i<infer_tree->N_slices+infer_tree->tree->getNumberOfLeaves()-1;i++)
{
//for (int k = 0;k<infer_tree->N_slices+infer_tree->tree->getNumberOfLeaves()-1;k++)
// infer_tree->Ttf[i][k]+=gather_Ttf[j][i][k];
broadcast_O_counts[i]+=gather_O_counts[j][i];
broadcast_D_counts[i]+=gather_D_counts[j][i];
broadcast_T_counts[i]+=gather_T_counts[j][i];
broadcast_branch_zero[i]+=gather_L_counts[j][i];
broadcast_branch_count[i]+=gather_B_counts[j][i];
broadcast_branch_sum[i]+=gather_S_counts[j][i];
broadcast_from_count[i]+=gather_F_counts[j][i];
broadcast_genome_size[i]+=gather_G_counts[j][i];
tmp_O+=gather_O_counts[j][i];
tmp_D+=gather_D_counts[j][i];
tmp_T+=gather_T_counts[j][i];
tmp_z+=gather_L_counts[j][i];
tmp_c+=gather_B_counts[j][i];
tmp_s+=gather_S_counts[j][i];
tmp_f+=gather_F_counts[j][i];
}
}
/*
for (int i = 0;i<infer_tree->N_slices+infer_tree->tree->getNumberOfLeaves()-1;i++)
{
cout << "tag "<<i;
cout << " " << broadcast_O_counts[i];
cout << " " << broadcast_D_counts[i];
cout << " " << broadcast_T_counts[i];
cout << " " << broadcast_branch_zero[i];
cout << " " << broadcast_branch_sum[i];
cout << " " << broadcast_branch_count[i];
cout << " " << broadcast_from_count[i];
cout << endl;
}
*/
//if (rank==server) cout << rank << " : "<<tmp_O << " " << tmp_D << " " << tmp_T << " " << tmp_z << " " << tmp_c << " " << tmp_s << " " << tmp_f << endl;
broadcast_pea_count=0.;
broadcast_ll=0.;
//cout << " start summing - llsum " << endl;
for (int j=1;j<size;j++)
{
broadcast_pea_count+=gather_pea_count[j];
broadcast_ll+=gather_ll[j];
}
// ### sum events
//########################## SERVER #########################
//cout << " end summing" << endl;
}
//########################## CLIENT #########################
//########################## CLIENT #########################
//########################## COMMON #########################
broadcast(world, broadcast_O_counts,server);
broadcast(world, broadcast_D_counts,server);
broadcast(world, broadcast_T_counts,server);
broadcast(world, broadcast_branch_zero,server);
broadcast(world, broadcast_branch_count,server);
broadcast(world, broadcast_branch_sum,server);
broadcast(world, broadcast_from_count,server);
broadcast(world, broadcast_genome_size,server);
broadcast(world, broadcast_pea_count,server);
broadcast(world, broadcast_ll,server);
//########################## COMMON #########################
//########################## SEVER #########################
//########################## CLIENT #########################
// if (rank==server)
// {
for (int i = 0;i<infer_tree->N_slices+infer_tree->tree->getNumberOfLeaves()-1;i++)
{
infer_tree->O_counts[i]=broadcast_O_counts[i];
infer_tree->D_counts[i]=broadcast_D_counts[i];
infer_tree->T_counts[i]=broadcast_T_counts[i];
infer_tree->branch_zero[i]=broadcast_branch_zero[i];
infer_tree->branch_count[i]=broadcast_branch_count[i];
infer_tree->branch_sum[i]=broadcast_branch_sum[i];
infer_tree->from_count[i]=broadcast_from_count[i];
infer_tree->branch_genome_size[i]=broadcast_genome_size[i];
}
//infer_tree->remember_rates();
// }
//infer_tree->apparent_rates(broadcast_pea_count,true,true);
//########################## CLIENT #########################
//########################## SERVER #########################
gather_ll.clear();
gather_pea_count.clear();
for (scatter_scalar::iterator it=gather_O_counts.begin();it!=gather_O_counts.end();it++) (*it).clear();
gather_O_counts.clear();
for (scatter_scalar::iterator it=gather_D_counts.begin();it!=gather_D_counts.end();it++) (*it).clear();
gather_D_counts.clear();
for (scatter_scalar::iterator it=gather_T_counts.begin();it!=gather_T_counts.end();it++) (*it).clear();
gather_T_counts.clear();
for (scatter_scalar::iterator it=gather_L_counts.begin();it!=gather_L_counts.end();it++) (*it).clear();
gather_L_counts.clear();
for (scatter_scalar::iterator it=gather_B_counts.begin();it!=gather_B_counts.end();it++) (*it).clear();
gather_B_counts.clear();
for (scatter_scalar::iterator it=gather_S_counts.begin();it!=gather_S_counts.end();it++) (*it).clear();
gather_S_counts.clear();
for (scatter_scalar::iterator it=gather_F_counts.begin();it!=gather_F_counts.end();it++) (*it).clear();
gather_F_counts.clear();
for (scatter_scalar::iterator it=gather_G_counts.begin();it!=gather_G_counts.end();it++) (*it).clear();
gather_G_counts.clear();
broadcast_O_counts.clear();
broadcast_D_counts.clear();
broadcast_T_counts.clear();
broadcast_branch_zero.clear();
broadcast_branch_count.clear();
broadcast_branch_sum.clear();
broadcast_from_count.clear();
broadcast_genome_size.clear();
return broadcast_pea_count;
}
pair<Species_tree *,string > sim_init_LL_mpi(string tree_file,string sim_file,int outgroup,
const mpi::communicator world)
{
//########################## GLOBAL #########################
string S_tree_string;
tree_type* S;
int server = 0;
int rank = world.rank();
int size = world.size();
scalar_type delta,tau,lambda,omega,delta_var,tau_var,lambda_var,omega_var,noise;
int N;
long seed;
vector < vector <scalar_type> > gather_sim_O_counts;
vector < vector <scalar_type> > gather_sim_D_counts;
vector < vector <scalar_type> > gather_sim_T_counts;
vector < vector <scalar_type> > gather_sim_L_counts;
vector < vector <scalar_type> > gather_sim_O_profile;
//########################## GLOBAL #########################
if (rank==server)
{
seed=good_seed();
cout << "#SEED:" << seed <<endl;
}
broadcast(world,seed,server);
RandomTools::setSeed(seed*rank+rank);
if (rank == server)
{
string sim_file_name=sim_file;
string tree_file_name=tree_file;
vector <string> forest,codes;
string line;
int i=0;
ifstream file_stream (sim_file_name.c_str());
scalar_type alpha=0.5;
scalar_type rho=0.5;
scalar_type height=0.5;
noise=0;
omega=0.5;
delta_var=0.;
tau_var=0.;
lambda_var=0.;
omega_var=0.;
N=20;
if (file_stream.is_open()) // ########## read sim params ############
{
while (! file_stream.eof() )
{
getline (file_stream,line);
if (line.find("#N")!=line.npos)
{
vector<string> tokens;
Tokenize(line,tokens," ");
N=atoi(tokens[1].c_str());
}
if (line.find("#noise")!=line.npos)
{
vector<string> tokens;
Tokenize(line,tokens," ");
noise=atof(tokens[1].c_str());
}
if (line.find("#ALPHA")!=line.npos)
{
vector<string> tokens;
Tokenize(line,tokens," ");
alpha=atof(tokens[1].c_str());
}
if (line.find("#RHO")!=line.npos)
{
vector<string> tokens;
Tokenize(line,tokens," ");
rho=atof(tokens[1].c_str());
}
if (line.find("#HEIGHT")!=line.npos)
{
vector<string> tokens;
Tokenize(line,tokens," ");
height=atof(tokens[1].c_str());
}
if (line.find("#OMEGA")!=line.npos)
{
vector<string> tokens;
Tokenize(line,tokens," ");
omega=atof(tokens[1].c_str());
}
if (line.find('#VAR')!=line.npos)
{
vector<string> tokens;
Tokenize(line,tokens," ");
delta_var=atof(tokens[1].c_str());
tau_var=atof(tokens[1].c_str());
lambda_var=atof(tokens[1].c_str());
}
if (line.find("#OMEGAVAR")!=line.npos)
{
vector<string> tokens;
Tokenize(line,tokens," ");
omega_var=atof(tokens[1].c_str());
}
}
file_stream.close();
}
delta=(1-alpha)*(rho)*height;
tau=(alpha)*(rho)*height;
lambda=(1-rho)*height;
if (rank==server) cout <<"D,T,L,O,N: "<< delta << " " << tau << " " << lambda << " " << omega << " " << N <<endl;
if (rank==server) cout <<"Dv,Tv,Lv,Ov,NOISE: "<< delta_var << " " << tau_var << " " << lambda_var << " " << omega_var<<" " << noise<<endl;
string S_string="";
file_stream.open(tree_file_name.c_str());
if (file_stream.is_open()) // ########## read an input S tree ############
{
while (! file_stream.eof() )
{
getline (file_stream,line);
if (line.find('(')!=line.npos && S_string=="")
{
vector<string> tokens;
Tokenize(line,tokens," ");
for (int i=0;i<tokens.size();i++)
if (tokens[i].find('(')!=tokens[i].npos)
S_string=tokens[i];
}
}
file_stream.close();
}
//########################## DISTRIBUTE ######################
S_tree_string=S_string;
}
//########################## DISTRIBUTE ######################
// SERVER CHECK POINT 1 - client trees sent -
broadcast(world, N, server);
broadcast(world, delta, server);
broadcast(world, tau, server);
broadcast(world, lambda, server);
broadcast(world, omega, server);
broadcast(world, delta_var, server);
broadcast(world, tau_var, server);
broadcast(world, lambda_var, server);
broadcast(world, omega_var, server);
broadcast(world, noise, server);
broadcast(world, S_tree_string, server);
//########################## infer_tree init #######################
S = TreeTemplateTools::parenthesisToTree(S_tree_string);
tree_type * So=S->clone();
Node * root=So->getRootNode();
double stem_len=0.1;
if (outgroup || true)// ########## add outgroup to root of S tree ############
{
Node * new_root = new Node();
Node * out_group = new Node();
root->addSon(new_root);
new_root->addSon(out_group);
out_group->setDistanceToFather(1.+stem_len);
out_group -> setName("OUT_GROUP");
So->rootAt(new_root);
Node * root=So->getRootNode();
root->getSon(0)->setDistanceToFather(stem_len);
root->getSon(1)->setDistanceToFather(stem_len);
}
Species_tree * infer_tree = new Species_tree(So);
infer_tree->init_x();
//cout << "rank " << rank << " " << delta << " " << tau << " " << lambda << endl;
infer_tree->init_sim(0.1, delta, tau, lambda, delta_var*delta, tau_var*tau, lambda_var*lambda,omega,omega*omega_var,0.);
stringstream out;
out<< rank << "trees.trees";
ofstream trees_file(out.str().c_str());
infer_tree->event_stream=&trees_file;
vector <string> client_trees;
vector <string> client_codes;
vector <string> client_stream;
if (rank!=server)
{
infer_tree->emit_G_trees(N,&client_trees,&client_codes,&client_stream);
for (int i=0;i<client_trees.size();i++)
{
if (noise>0)
{
scalar_type p0 = exp(-noise);
scalar_type p=1;
int k=-1;
bool stop=false;
while(!stop)
{
k++;
scalar_type u = RandomTools::giveRandomNumberBetweenZeroAndEntry(1);
p*=u;
if (p<=p0)
stop=true;
}
for (int nni=0;nni<k;nni++)
{
vector <string> allnnis=all_NNIs(client_trees[i]);
client_trees[i]=allnnis[RandomTools::giveIntRandomNumberBetweenZeroAndEntry(allnnis.size())];
}
}
}
infer_tree->unrooted_register(client_trees);
for (int i=0;i<client_trees.size();i++)
infer_tree->codes[client_trees[i]]=client_codes[i];
}
//########################## infer_tree init #######################
if (noise==0)
{
infer_tree->world=world;
infer_tree->branchwise=1;
infer_tree->count_labels(client_trees);
scalar_type pea_count = sum_counts(infer_tree,world);
for (int i = 0;i<infer_tree->N_slices+infer_tree->tree->getNumberOfLeaves();i++)
{
infer_tree->sim_O_count[i]=infer_tree->O_counts[i];
infer_tree->sim_D_count[i]=infer_tree->D_counts[i];
infer_tree->sim_T_count[i]=infer_tree->T_counts[i];
infer_tree->sim_L_count[i]=infer_tree->branch_zero[i];
infer_tree->sim_branch_sum[i]=infer_tree->branch_sum[i];
infer_tree->sim_branch_count[i]=infer_tree->branch_count[i];
infer_tree->sim_branch_genome_size[i]=infer_tree->branch_genome_size[i];
}
}
vector < vector <string> > gather_trees;
gather(infer_tree->world, infer_tree->in_trees, gather_trees, server);
if (rank==server)
{
vector <string> forest;
//GG
for (int j=1;j<size;j++)
for (vector<string>::iterator tit=gather_trees[j].begin();tit!=gather_trees[j].end();tit++)
forest.push_back((*tit));
infer_tree->panj_string= PANJ(S_tree_string,forest);
cout << "#PANJ "<< infer_tree->panj_string <<endl;
cout << "#PANJ "<< TreeTools::robinsonFouldsDistance(*TreeTemplateTools::parenthesisToTree(S_tree_string),*TreeTemplateTools::parenthesisToTree(infer_tree->panj_string)) << endl;
}
pair<Species_tree *,string > return_pair;
return_pair.first=infer_tree;
return_pair.second=S_tree_string;
return return_pair;
//########################## CLIENT #########################
}
scalar_type LL_mpi(Species_tree * infer_tree,string Sstring,string mode,bool outgroup,scalar_type delta,scalar_type tau, scalar_type lambda)
{
const mpi::communicator world = infer_tree->world;
int server = 0;
int rank = world.rank();
int size = world.size();
scalar_type ll;
scalar_type return_ll;
if (rank==server) cout << mode << " " << Sstring <<endl;
broadcast(world, Sstring,server);
tree_type * S=TreeTemplateTools::parenthesisToTree(Sstring);
if (outgroup)
{
Node * root=S->getRootNode();
vector <Node*> nodes=S->getNodes();
for (vector<Node * > :: iterator it=nodes.begin();it!=nodes.end();it++)
if ((*it)->hasFather())
if ((*it)->getDistanceToFather()==0)
(*it)->setDistanceToFather(0.001);
infer_tree->node_makeclocklike_height((S->getRootNode()));
S->scaleTree(1./TreeTemplateTools::getDistanceBetweenAnyTwoNodes(*root,*(S->getLeaves()[0])));
double stem_len=0.1;
Node * new_root = new Node();
Node * out_group = new Node();
root->addSon(new_root);
new_root->addSon(out_group);
out_group->setDistanceToFather(1.+stem_len);
out_group -> setName("OUT_GROUP");
S->rootAt(new_root);
root=S->getRootNode();
if (root->getSon(0)->isLeaf())
root->getSon(1)->setDistanceToFather(stem_len);
else
root->getSon(0)->setDistanceToFather(stem_len);
//root->getSon(1)->setDistanceToFather(stem_len);
infer_tree->reconstruct(S);
}
vector <string> modes;
Tokenize(mode,modes,"_");
if (modes[0]=="clock" )
{
infer_tree->intp=0;
infer_tree->w=0.;
infer_tree->reset();
infer_tree->branchwise=0;
string clock_string=Sstring;
if (rank==server) cout << modes[0] << " " << modes[1] <<endl;
clock_string=DTL_clock(clock_string,infer_tree,modes[1]);
//ll=LL_mpi(infer_tree, clock_string, "Hest",outgroup,delta,tau, lambda);
ll=infer_tree->tmp_ll;
infer_tree->tmp_ll=ll;
delete S;
if (rank==server) cout <<"$$ "<< clock_string <<endl;
infer_tree->tmp_Sstring=clock_string;
infer_tree->tmp_ll=ll;
return ll;
}
if (mode=="Nest" )
{
infer_tree->intp=0;
infer_tree->w=0.;
infer_tree->reset(delta,tau,lambda);
infer_tree->branchwise=0;
ll=oLL(infer_tree,S,delta,tau,lambda,1,"estimate");
//ll=oLL(infer_tree,S,delta,tau,lambda,1,"intbck");
infer_tree->omega_avg=infer_tree->branchwise_omega[0]+infer_tree->branchwise_omega[1]+infer_tree->branchwise_omega[2];
delete S;
infer_tree->tmp_ll=ll;
return ll;
}
if (mode=="noestimate" )
{
ll= oLL(infer_tree,S,delta,tau,lambda,1,"noestimate") ;
delete S;
return ll;
}
if (mode=="Pest" )
{
infer_tree->intp=0.9;
infer_tree->w=0.;
// !! no T
infer_tree->reset();
infer_tree->branchwise=0;
ll=oLL(infer_tree,S,delta,tau,lambda,1,"intbck");
scalar_type pea_count = sum_counts(infer_tree,world);
delete S;
infer_tree->tmp_ll=ll;
return ll;
}
if (mode=="Test" )
{
infer_tree->intp=0.9;
infer_tree->w=0.;
// !! no T
infer_tree->reset();
infer_tree->branchwise=0;
ll=oLL(infer_tree,S,delta,tau,lambda,1,"intbck");
scalar_type pea_count = sum_counts(infer_tree,world);
delete S;
scalar_type Tll=0;
for (int i = 0;i<infer_tree->N_slices+infer_tree->tree->getNumberOfLeaves()-1;i++)
Tll+=infer_tree->T_counts[i];
infer_tree->tmp_ll=-Tll;
return -Tll;
}
if (mode=="FASTest" )
{
infer_tree->intp=0.9;
infer_tree->w=0.;
// !! no T
infer_tree->reset();
infer_tree->branchwise=0;
ll=oLL(infer_tree,S,delta,tau,lambda,1,"intbck");
scalar_type pea_count = sum_counts(infer_tree,world);
infer_tree->apparent_rates(pea_count,true,true);
//if (rank==server) cout<<"#p "<< ll << " " << " "<< infer_tree->delta_avg << " " << infer_tree->tau_avg << " " << infer_tree->lambda_avg<< " " << infer_tree->omega_avg << endl ;
ll=oLL(infer_tree,S,delta,tau,lambda,1,"estimate");
infer_tree->tmp_ll=ll;
infer_tree->omega_avg=infer_tree->branchwise_omega[0]+infer_tree->branchwise_omega[1]+infer_tree->branchwise_omega[2];
delete S;
return ll;
}
if (mode=="Hest" || mode=="IHest")
{
if (mode=="Hest")
infer_tree->nclasses=5;
// !! no T
//tau=1e-30;
//infer_tree->reset(0.01,1e-30,0.01);
infer_tree->reset();
infer_tree->intp=0.9;
infer_tree->w=0.;
infer_tree->branchwise=0;
ll=oLL(infer_tree,S,delta,tau,lambda,1,"intbck");
infer_tree->tmp_ll=ll;
scalar_type pea_count = sum_counts(infer_tree,world);
if (rank==server) cout<<"#p "<< ll << " " << " "<< infer_tree->delta_avg << " " << infer_tree->tau_avg << " " << infer_tree->lambda_avg<< " " << infer_tree->omega_avg <<" "<<pea_count << endl ;
infer_tree->apparent_rates(pea_count,mode=="Hest",true);
//if (rank==server) infer_tree->print_rates();
// ### sum events
scalar_type ll2=ll-2;
while (ll>ll2 && abs(ll-ll2)>1 )
{
ll2=ll;
// given rates estimate p_omega
ll=oLL(infer_tree,S,delta,tau,lambda,1,"estimate");
infer_tree->tmp_ll=ll;
if (ll2>ll)
{
infer_tree->recall_rates();
//scalar_type re_ll=oLL(infer_tree,S,delta,tau,lambda,1,"noestimate") ;
ll=ll2;
break;
}
infer_tree->omega_avg=infer_tree->branchwise_omega[0]+infer_tree->branchwise_omega[1]+infer_tree->branchwise_omega[2];
// given p_omega estimate rates
// formal position of remember
oLL(infer_tree,S,delta,tau,lambda,1,"intbck");
scalar_type pea_count = sum_counts(infer_tree,world);
infer_tree->apparent_rates(pea_count,mode=="Hest",true);
if (rank==server) cout<<"#e "<< ll << " " << " "<< infer_tree->delta_avg << " " << infer_tree->tau_avg << " " << infer_tree->lambda_avg<< " " << infer_tree->omega_avg << endl ;
// ### sum events
}
return_ll=ll;
infer_tree->tmp_ll=ll;
delete S;
return return_ll;
}
}
pair<Species_tree *,string > init_LL_mpi(string tree_file,string forest_file,int outgroup,
const mpi::communicator world)
{
//########################## GLOBAL #########################
tree_type * S;
string S_tree_string;
string panj_string;
int server = 0;
int rank = world.rank();
int size = world.size();
vector <string> client_trees;
vector <string> client_codes;
vector < vector <string> > scatter_trees;
vector < vector <string> > scatter_codes;
//########################## GLOBAL #########################
if (rank == server)
{
//########################## SERVER #########################
string forest_file_name=forest_file;
string tree_file_name=tree_file;
vector <string> forest,codes;
string line;
int i=0;
ifstream file_stream (forest_file_name.c_str());
if (file_stream.is_open()) // ########## read a forest ############
{
while (! file_stream.eof() )
{
getline (file_stream,line);
if (line.find('(')!=line.npos)
{
i++;
forest.push_back(line);
}
else if (line.find('#')!=line.npos)
codes.push_back(line);
}
file_stream.close();
}
string S_string="";
file_stream.open(tree_file_name.c_str());
if (file_stream.is_open()) // ########## read an input S tree ############
{
while (! file_stream.eof() )
{
getline (file_stream,line);
if (line.find('(')!=line.npos && S_string=="")
{
vector<string> tokens;
Tokenize(line,tokens," ");
for (int i=0;i<tokens.size();i++)
if (tokens[i].find('(')!=tokens[i].npos)
S_string=tokens[i];
}
}
file_stream.close();
}
//########################## DISTRIBUTE ######################
/// SAMPLE
vector <int> lottery;
for (int i=0;i<forest.size();i++)
lottery.push_back(i);
vector <int> winners;
winners.resize(forest.size()/10.);
RandomTools::getSample(lottery,winners);
vector<string> sample_forest;
vector<string> sample_codes;
for (int i=0;i<winners.size();i++)
{
int j=winners[i];
sample_forest.push_back(forest[j]);
sample_codes.push_back(codes[j]);
}
forest.clear();
codes.clear();
for (int i=0;i<winners.size();i++)
{
forest.push_back(sample_forest[i]);
codes.push_back(sample_codes[i]);
}
/// SAMPLE
S_tree_string=S_string;
panj_string= PANJ(S_string,forest);
cout << "#PANJ "<<panj_string <<endl;
cout << "#PANJ "<< TreeTools::robinsonFouldsDistance(*TreeTemplateTools::parenthesisToTree(S_string),*TreeTemplateTools::parenthesisToTree(panj_string)) << endl;
map <scalar_type,string> sort_trees;
map <scalar_type,string> sort_codes;
for (int i=0;i<forest.size();i++)
{
tree_type * S=TreeTemplateTools::parenthesisToTree(forest[i]);
scalar_type treesize=S->getNumberOfLeaves();
while ( sort_trees.count(treesize) )
treesize+=RandomTools::giveRandomNumberBetweenZeroAndEntry(1e-2);
sort_trees[treesize]=forest[i];
sort_codes[treesize]=codes[i];
delete S;
}
int new_i=0;
for ( map <scalar_type,string> ::iterator it=sort_trees.begin();it!=sort_trees.end();it++)
{
forest[new_i]=it->second;
codes[new_i]=sort_codes[it->first];
new_i+=1;
}
map <int,int> large_trees;
for (int i=0;i<forest.size();i++)
{
tree_type * S=TreeTemplateTools::parenthesisToTree(forest[i]);
if (S->getNumberOfLeaves()>150)
{
large_trees[i]=S->getNumberOfLeaves();
}
//large_trees[S->getNumberOfLeaves()]++;
delete S;
}
//int cum=0;
//for ( map <int,int> ::iterator it=large_trees.begin();it!=large_trees.end();it++)
// {
// cum+=(*it).second;
// cout << (*it).first << " " << (*it).second << " " << cum << endl;
// }
map <int,int> scatter_sizes;
for (int j=0;j<size;j++)
{
vector <string> tmp;
scatter_trees.push_back(tmp);
vector <string> tmp2;
scatter_codes.push_back(tmp2);
}
//cout << "distribute" <<endl;
for (int i=0;i<forest.size();i++)
{
int j = i%(size-1);
if (large_trees.count(i)==0)
{
scatter_trees[j+1].push_back(forest[i]);
scatter_codes[j+1].push_back(codes[i]);
tree_type * S=TreeTemplateTools::parenthesisToTree(forest[i]);
scatter_sizes[j]+=S->getNumberOfLeaves();
delete S;
}
}
for ( map <int,int> ::iterator it=scatter_sizes.begin();it!=scatter_sizes.end();it++)
cout << it->first << " " << it->second << endl;
//cout << "distribute" <<endl;
//.optimization could come here
//########################## DISTRIBUTE ######################
// SERVER CHECK POINT 1 - client trees sent -
broadcast(world, S_tree_string, server);
broadcast(world, panj_string, server);
scatter(world, scatter_trees, client_trees, server);
scatter(world, scatter_codes, client_codes, server);
//########################## count_tree init #######################
S = TreeTemplateTools::parenthesisToTree(S_tree_string);
tree_type * So=S->clone();
Node * root=So->getRootNode();
double stem_len=0.1;
//!!
if (outgroup || true)// ########## add outgroup to root of S tree ############
{
Node * new_root = new Node();
Node * out_group = new Node();
root->addSon(new_root);
new_root->addSon(out_group);
out_group->setDistanceToFather(1.+stem_len);
out_group -> setName("OUT_GROUP");
So->rootAt(new_root);
Node * root=So->getRootNode();
if (root->getSon(0)->isLeaf())
root->getSon(1)->setDistanceToFather(stem_len);
else
root->getSon(0)->setDistanceToFather(stem_len);
}
Species_tree * count_tree = new Species_tree(So);
count_tree->panj_string=panj_string;
count_tree->init_x();
//########################## count_tree init #######################
// SERVER CHECK POINT 1 - client trees sent -
count_tree->world=world;
pair<Species_tree *,string > return_pair;
return_pair.first=count_tree;
return_pair.second=S_tree_string;
return return_pair;
//########################## SERVER #########################
}
else
{
//########################## CLIENT #########################
// CLIENT CHECK POINT 1 - client trees recived -
broadcast(world, S_tree_string, server);
broadcast(world, panj_string, server);
scatter(world, scatter_trees, client_trees, server);
scatter(world, scatter_codes, client_codes, server);
//########################## infer_tree init #######################
S = TreeTemplateTools::parenthesisToTree(S_tree_string);
tree_type * So=S->clone();
Node * root=So->getRootNode();
double stem_len=0.1;
if (outgroup || true)// ########## add outgroup to root of S tree ############
{
Node * new_root = new Node();
Node * out_group = new Node();
root->addSon(new_root);
new_root->addSon(out_group);
out_group->setDistanceToFather(1.+stem_len);
out_group -> setName("OUT_GROUP");
So->rootAt(new_root);
Node * root=So->getRootNode();
root->getSon(0)->setDistanceToFather(stem_len);
root->getSon(1)->setDistanceToFather(stem_len);
}
Species_tree * infer_tree = new Species_tree(So);
infer_tree->panj_string=panj_string;
infer_tree->init_x();
infer_tree->unrooted_register(client_trees);
for (int i=0;i<client_trees.size();i++)
infer_tree->codes[client_trees[i]]=client_codes[i];
//########################## infer_tree init #######################
// CLIENT CHECK POINT 1 - client trees recived -
infer_tree->world=world;
pair<Species_tree *,string > return_pair;
return_pair.first=infer_tree;
return_pair.second=S_tree_string;
return return_pair;
//########################## CLIENT #########################
}
}
MCCResults mccrun_master(
const Options& opts, const Eigen::VectorXd& vpar, unsigned int num_bins,
const set<observables_t>& obs, const mpi::communicator& mpicomm )
{
cout << "========== NEW MONTE CARLO CYCLE ==========" << endl;
cout << ":: Preparing the simulation" << endl;
HubbardModelVMC model = prepare_model( opts, vpar, mpicomm );
vector< unique_ptr<Observable> > obscalc = prepare_obscalcs( obs, opts );
ObservableCache obscache;
unsigned int finished_workers = 0;
unsigned int scheduled_bins = 0;
unsigned int completed_bins = 0;
unsigned int enqueued_bins = num_bins;
// define procedure to query the slaves for new work requests
function<void()> mpiquery_work_requests( [&]() {
while ( boost::optional<mpi::status> status
= mpicomm.iprobe( mpi::any_source, MSGTAG_S_M_REQUEST_BINS ) ) {
// receive the request and hand out new bins to the source
mpicomm.recv( status->source(), MSGTAG_S_M_REQUEST_BINS );
if ( enqueued_bins > 0 ) {
mpicomm.send( status->source(), MSGTAG_M_S_DISPATCHED_BINS, 1 );
scheduled_bins += 1;
enqueued_bins -= 1;
} else {
mpicomm.send( status->source(), MSGTAG_M_S_DISPATCHED_BINS, 0 );
++finished_workers;
}
}
} );
// define procedure to query the slaves for finished work
function<void()> mpiquery_finished_work( [&]() {
while ( boost::optional<mpi::status> status
= mpicomm.iprobe( mpi::any_source, 2 ) ) {
mpicomm.recv( status->source(), 2 );
--scheduled_bins;
++completed_bins;
}
} );
cout << ":: Equilibrating the system" << endl;
for (
unsigned int mcs = 0;
mcs < opts["calc.num-mcs-equil"].as<unsigned int>();
++mcs ) {
// take care of the slaves
mpiquery_finished_work();
mpiquery_work_requests();
// perform a Monte Carlo step
model.mcs();
}
unsigned int completed_bins_master = 0;
cout << ":: Performing Monte Carlo cycle" << endl;
cout << endl;
cout << " Progress:" << endl;
while ( enqueued_bins > 0 ) {
cout << '\r' << " Bin "
<< completed_bins << "/" << num_bins;
cout.flush();
--enqueued_bins;
++scheduled_bins;
for (
unsigned int mcs = 0;
mcs < opts["calc.num-binmcs"].as<unsigned int>();
++mcs ) {
// take care of the slaves
mpiquery_finished_work();
mpiquery_work_requests();
// perform a Monte Carlo step
model.mcs();
// measure observables
for ( const unique_ptr<Observable>& o : obscalc ) {
o->measure( model, obscache );
}
obscache.clear();
}
// tell the observables that a bin has been completed
for ( const unique_ptr<Observable>& o : obscalc ) {
o->completebin();
}
--scheduled_bins;
++completed_bins_master;
++completed_bins;
}
++finished_workers;
while ( completed_bins != num_bins ||
static_cast<int>( finished_workers ) < mpicomm.size() ) {
if ( boost::optional<mpi::status> status
= mpicomm.iprobe( mpi::any_source, MSGTAG_S_M_FINISHED_BINS ) ) {
mpicomm.recv( status->source(), MSGTAG_S_M_FINISHED_BINS );
--scheduled_bins;
++completed_bins;
cout << '\r' << " Bin " << completed_bins << "/" << num_bins;
cout.flush();
}
if ( boost::optional<mpi::status> status
= mpicomm.iprobe( mpi::any_source, MSGTAG_S_M_REQUEST_BINS ) ) {
// receive the request for more work
mpicomm.recv( status->source(), MSGTAG_S_M_REQUEST_BINS );
// tell him there is no more work
mpicomm.send( status->source(), MSGTAG_M_S_DISPATCHED_BINS, 0 );
++finished_workers;
}
}
assert( enqueued_bins == 0 );
assert( scheduled_bins == 0 );
cout << '\r' << " Bin " << completed_bins << "/" << num_bins << endl;
cout.flush();
// check for floating point precision problems
cout << endl;
cout << " Floating point precision control" << endl;
vector<FPDevStat> W_devstats;
assert( mpicomm.rank() == 0 );
mpi::gather( mpicomm, model.get_W_devstat(), W_devstats, 0 );
FPDevStat W_devstat_combined =
accumulate(
W_devstats.begin(), W_devstats.end(),
FPDevStat( opts["fpctrl.W-deviation-target"].as<double>() )
);
cout << " W: " << W_devstat_combined.recalcs
<< "/" << W_devstat_combined.misses
<< "/" << W_devstat_combined.mag1_misses << endl;
vector<FPDevStat> T_devstats;
assert( mpicomm.rank() == 0 );
mpi::gather( mpicomm, model.get_T_devstat(), T_devstats, 0 );
FPDevStat T_devstat_combined =
accumulate(
T_devstats.begin(), T_devstats.end(),
FPDevStat( opts["fpctrl.T-deviation-target"].as<double>() )
);
cout << " T: " << T_devstat_combined.recalcs
<< "/" << T_devstat_combined.misses
<< "/" << T_devstat_combined.mag1_misses << endl;
if ( W_devstat_combined.mag1_misses > 0 ||
T_devstat_combined.mag1_misses > 0 ) {
cout << " Precision targets missed by more than an order of magnitude!" << endl
<< " WARNING: Your results might be unreliable!!!" << endl << endl;
} else if ( W_devstat_combined.misses > 0 ||
T_devstat_combined.misses > 0 ) {
cout << " Some precision targets were missed, but your results should be fine."
<< endl << endl;
} else {
cout << " No missed precision targets." << endl << endl;
}
// collect results from the slaves and return results the scheduler
MCCResults results;
for ( const unique_ptr<Observable>& o : obscalc ) {
o->collect_and_write_results( mpicomm, results );
}
results.success = true;
return results;
}
