void get_master_ranks(std::set<size_t>& master_ranks) { uint32_t local_ip = get_local_ip(); std::vector<uint32_t> all_ips; all_gather(local_ip, all_ips); std::set<uint32_t> visited_ips; master_ranks.clear(); for(size_t i = 0; i < all_ips.size(); ++i) { if(visited_ips.count(all_ips[i]) == 0) { visited_ips.insert(all_ips[i]); master_ranks.insert(i); } } }
Index<Distance>::Index(const std::string& file_name, const std::string& dataset_name, const IndexParams& params) { boost::mpi::communicator world; flann_algorithm_t index_type = params.getIndexType(); if (index_type == SAVED) { throw FLANNException("Saving/loading of MPI indexes is not currently supported."); } flann::mpi::load_from_file(dataset, file_name, dataset_name); flann_index = new flann::Index<Distance>(dataset, params); std::vector<int> sizes; // get the sizes of all MPI indices all_gather(world, flann_index->size(), sizes); size_ = 0; offset_ = 0; for (size_t i = 0;i < sizes.size();++i) { if (i < world.rank()) offset_ += sizes[i]; size_ += sizes[i]; } }
void GatherWorkspaces::exec() { // Every process in an MPI job must hit this next line or everything hangs! mpi::communicator world; // The communicator containing all processes inputWorkspace = getProperty("InputWorkspace"); // Create a new communicator that includes only those processes that have an // input workspace const int haveWorkspace(inputWorkspace ? 1 : 0); included = world.split(haveWorkspace); // If the present process doesn't have an input workspace then its work is // done if (!haveWorkspace) { g_log.information("No input workspace on this process, so nothing to do."); return; } // Get the number of bins in each workspace and check they're all the same numBins = inputWorkspace->blocksize(); std::vector<std::size_t> all_numBins; all_gather(included, numBins, all_numBins); if (std::count(all_numBins.begin(), all_numBins.end(), numBins) != (int)all_numBins.size()) { // All the processes will error out if all the workspaces don't have the // same number of bins throw Exception::MisMatch<std::size_t>( numBins, 0, "All input workspaces must have the same number of bins"); } // Also check that all workspaces are either histogram or not // N.B. boost mpi doesn't seem to like me using booleans in the all_gather hist = inputWorkspace->isHistogramData(); std::vector<int> all_hist; all_gather(included, hist, all_hist); if (std::count(all_hist.begin(), all_hist.end(), hist) != (int)all_hist.size()) { // All the processes will error out if we don't have either all histogram or // all point-data workspaces throw Exception::MisMatch<int>( hist, 0, "The input workspaces must be all histogram or all point data"); } // How do we accumulate the data? std::string accum = this->getPropertyValue("AccumulationMethod"); // Get the total number of spectra in the combined inputs totalSpec = inputWorkspace->getNumberHistograms(); sumSpec = totalSpec; if (accum == "Append") { reduce(included, totalSpec, sumSpec, std::plus<std::size_t>(), 0); } else if (accum == "Add") { // barrier only helps when memory is too low for communication // included.barrier(); } eventW = boost::dynamic_pointer_cast<const EventWorkspace>(inputWorkspace); if (eventW != NULL) { if (getProperty("PreserveEvents")) { // Input workspace is an event workspace. Use the other exec method this->execEvent(); return; } } // The root process needs to create a workspace of the appropriate size MatrixWorkspace_sptr outputWorkspace; if (included.rank() == 0) { g_log.debug() << "Total number of spectra is " << sumSpec << "\n"; // Create the workspace for the output outputWorkspace = WorkspaceFactory::Instance().create( inputWorkspace, sumSpec, numBins + hist, numBins); setProperty("OutputWorkspace", outputWorkspace); ExperimentInfo_sptr inWS = inputWorkspace; outputWorkspace->copyExperimentInfoFrom(inWS.get()); } for (size_t wi = 0; wi < totalSpec; wi++) { if (included.rank() == 0) { const ISpectrum *inSpec = inputWorkspace->getSpectrum(wi); if (accum == "Add") { outputWorkspace->dataX(wi) = inputWorkspace->readX(wi); reduce(included, inputWorkspace->readY(wi), outputWorkspace->dataY(wi), vplus(), 0); reduce(included, inputWorkspace->readE(wi), outputWorkspace->dataE(wi), eplus(), 0); } else if (accum == "Append") { // Copy over data from own input workspace outputWorkspace->dataX(wi) = inputWorkspace->readX(wi); outputWorkspace->dataY(wi) = inputWorkspace->readY(wi); outputWorkspace->dataE(wi) = inputWorkspace->readE(wi); const int numReqs(3 * (included.size() - 1)); mpi::request reqs[numReqs]; int j(0); // Receive data from all the other processes // This works because the process ranks are ordered the same in // 'included' as // they are in 'world', but in general this is not guaranteed. TODO: // robustify for (int i = 1; i < included.size(); ++i) { size_t index = wi + i * totalSpec; reqs[j++] = included.irecv(i, 0, outputWorkspace->dataX(index)); reqs[j++] = included.irecv(i, 1, outputWorkspace->dataY(index)); reqs[j++] = included.irecv(i, 2, outputWorkspace->dataE(index)); ISpectrum *outSpec = outputWorkspace->getSpectrum(index); outSpec->clearDetectorIDs(); outSpec->addDetectorIDs(inSpec->getDetectorIDs()); } // Make sure everything's been received before exiting the algorithm mpi::wait_all(reqs, reqs + numReqs); } ISpectrum *outSpec = outputWorkspace->getSpectrum(wi); outSpec->clearDetectorIDs(); outSpec->addDetectorIDs(inSpec->getDetectorIDs()); } else { if (accum == "Add") { reduce(included, inputWorkspace->readY(wi), vplus(), 0); reduce(included, inputWorkspace->readE(wi), eplus(), 0); } else if (accum == "Append") { mpi::request reqs[3]; // Send the spectrum to the root process reqs[0] = included.isend(0, 0, inputWorkspace->readX(0)); reqs[1] = included.isend(0, 1, inputWorkspace->readY(0)); reqs[2] = included.isend(0, 2, inputWorkspace->readE(0)); // Make sure the sends have completed before exiting the algorithm mpi::wait_all(reqs, reqs + 3); } } } }