void
GslVector::mpiAllQuantile(double probability, const MpiComm& opComm, GslVector& resultVec) const
{
// Filter out those nodes that should not participate
if (opComm.MyPID() < 0) return;
queso_require_msg(!((probability < 0.) || (1. < probability)), "invalid input");
unsigned int size = this->sizeLocal();
queso_require_equal_to_msg(size, resultVec.sizeLocal(), "different vector sizes");
for (unsigned int i = 0; i < size; ++i) {
double auxDouble = (int) (*this)[i];
std::vector<double> vecOfDoubles(opComm.NumProc(),0.);
opComm.Gather((void *) &auxDouble, 1, RawValue_MPI_DOUBLE, (void *) &vecOfDoubles[0], (int) 1, RawValue_MPI_DOUBLE, 0,
"GslVector::mpiAllQuantile()",
"failed MPI.Gather()");
std::sort(vecOfDoubles.begin(), vecOfDoubles.end());
double result = vecOfDoubles[(unsigned int)( probability*((double)(vecOfDoubles.size()-1)) )];
opComm.Bcast((void *) &result, (int) 1, RawValue_MPI_DOUBLE, 0,
"GslVector::mpiAllQuantile()",
"failed MPI.Bcast()");
resultVec[i] = result;
}
return;
}
bool
MiscCheckForSameValueInAllNodes(T& inputValue, // Yes, 'not' const
double acceptableTreshold,
const MpiComm& comm,
const char* whereString)
{
// Filter out those nodes that should not participate
if (comm.MyPID() < 0) return true;
double localValue = (double) inputValue;
double sumValue = 0.;
comm.Allreduce((void *) &localValue, (void *) &sumValue, (int) 1, RawValue_MPI_DOUBLE, RawValue_MPI_SUM,
whereString,
"failed MPI on 'sumValue' inside MiscCheckForSameValueInAllNodes()");
double totalNumNodes = (double) comm.NumProc();
double testValue = fabs(1. - localValue/(sumValue/totalNumNodes));
unsigned int boolSum = 0;
#if 1
unsigned int boolResult = 0;
if (testValue > acceptableTreshold) boolResult = 1;
comm.Allreduce((void *) &boolResult, (void *) &boolSum, (int) 1, RawValue_MPI_UNSIGNED, RawValue_MPI_SUM,
whereString,
"failed MPI on 'boolSum' inside MiscCheckForSameValueInAllNodes()");
if (boolSum > 0) {
comm.Barrier();
for (int i = 0; i < comm.NumProc(); ++i) {
if (i == comm.MyPID()) {
std::cerr << "WARNING, "
<< whereString
<< ", inside MiscCheckForSameValueInAllNodes()"
<< ", rank (in this communicator) = " << i
<< ": boolSum = " << boolSum
<< ", localValue = " << localValue
<< ", sumValue = " << sumValue
<< ", totalNumNodes = " << totalNumNodes
<< ", avgValue = " << (sumValue/totalNumNodes)
<< ", relativeTest = " << testValue
<< std::endl;
}
comm.Barrier();
}
comm.Barrier();
comm.Bcast((void *) &localValue, (int) 1, RawValue_MPI_DOUBLE, 0,
whereString,
"failed MPI on 'boolSum' inside MiscCheckForSameValueInAllNodes()");
inputValue = localValue; // IMPORTANT
}
#else
UQ_FATAL_TEST_MACRO(testValue > acceptableTreshold,
UQ_UNAVAILABLE_RANK,
whereString,
"not all nodes have the same value inside MiscCheckForSameValueInAllNodes()");
#endif
return (boolSum == 0);
}
void
GslVector::mpiBcast(int srcRank, const MpiComm& bcastComm)
{
// Filter out those nodes that should not participate
if (bcastComm.MyPID() < 0) return;
// Check 'srcRank'
queso_require_msg(!((srcRank < 0) || (srcRank >= bcastComm.NumProc())), "invalud srcRank");
// Check number of participant nodes
double localNumNodes = 1.;
double totalNumNodes = 0.;
bcastComm.Allreduce((void *) &localNumNodes, (void *) &totalNumNodes, (int) 1, RawValue_MPI_DOUBLE, RawValue_MPI_SUM,
"GslVector::mpiBcast()",
"failed MPI.Allreduce() for numNodes");
queso_require_equal_to_msg(((int) totalNumNodes), bcastComm.NumProc(), "inconsistent numNodes");
// Check that all participant nodes have the same vector size
double localVectorSize = this->sizeLocal();
double sumOfVectorSizes = 0.;
bcastComm.Allreduce((void *) &localVectorSize, (void *) &sumOfVectorSizes, (int) 1, RawValue_MPI_DOUBLE, RawValue_MPI_SUM,
"GslVector::mpiBcast()",
"failed MPI.Allreduce() for vectorSize");
if ( ((unsigned int) sumOfVectorSizes) != ((unsigned int)(totalNumNodes*localVectorSize)) ) {
std::cerr << "rank " << bcastComm.MyPID()
<< ": sumOfVectorSizes = " << sumOfVectorSizes
<< ", totalNumNodes = " << totalNumNodes
<< ", localVectorSize = " << localVectorSize
<< std::endl;
}
bcastComm.Barrier();
queso_require_equal_to_msg(((unsigned int) sumOfVectorSizes), ((unsigned int)(totalNumNodes*localVectorSize)), "inconsistent vectorSize");
// Ok, bcast data
std::vector<double> dataBuffer((unsigned int) localVectorSize, 0.);
if (bcastComm.MyPID() == srcRank) {
for (unsigned int i = 0; i < dataBuffer.size(); ++i) {
dataBuffer[i] = (*this)[i];
}
}
bcastComm.Bcast((void *) &dataBuffer[0], (int) localVectorSize, RawValue_MPI_DOUBLE, srcRank,
"GslVector::mpiBcast()",
"failed MPI.Bcast()");
if (bcastComm.MyPID() != srcRank) {
for (unsigned int i = 0; i < dataBuffer.size(); ++i) {
(*this)[i] = dataBuffer[i];
}
}
return;
}
void HypCutRegion::cb_collexit_handler(
const pearl::CallbackManager& cbmanager,
int user_event, const pearl::Event& event,
pearl::CallbackData* cdata)
{
silas::CallbackData* data = static_cast<silas::CallbackData*>(cdata);
MpiComm *comm = event->get_comm();
int cut = (calltree_depth > 0 ? 1 : 0);
MPI_Allreduce(&cut, 1, MPI_INT, MPI_MAX, comm->get_comm());
}
void
GslVector::mpiAllReduce(RawType_MPI_Op mpiOperation, const MpiComm& opComm, GslVector& resultVec) const
{
// Filter out those nodes that should not participate
if (opComm.MyPID() < 0) return;
unsigned int size = this->sizeLocal();
queso_require_equal_to_msg(size, resultVec.sizeLocal(), "different vector sizes");
for (unsigned int i = 0; i < size; ++i) {
double srcValue = (*this)[i];
double resultValue = 0.;
opComm.Allreduce((void *) &srcValue, (void *) &resultValue, (int) 1, RawValue_MPI_DOUBLE, mpiOperation,
"GslVector::mpiAllReduce()",
"failed MPI.Allreduce()");
resultVec[i] = resultValue;
}
return;
}
