int
sock_send_ints(int purpose, int *vals, int num)
{
if (accept_if_needed(purpose) != -1)
return send_ints(purpose_table[purpose], vals, num);
return -1;
}
int fei::Vector_core::scatterToOverlap()
{
if (fei::numProcs(comm_) == 1 || haveFEVector()) {
return(0);
}
#ifndef FEI_SER
if (!overlapAlreadySet_) {
setOverlap();
}
//...and now the overlap is whatever is in our remotelyOwned_ vectors.
//first find out which procs we'll be receiving from.
std::vector<int> recvProcs;
for(unsigned i=0; i<remotelyOwned_.size(); ++i) {
if ((int)i == fei::localProc(comm_)) continue;
if (remotelyOwned_[i]->size() == 0) continue;
recvProcs.push_back((int)i);
}
//find out the send-procs.
std::vector<int> sendProcs;
fei::mirrorProcs(comm_, recvProcs, sendProcs);
//declare arrays to send from, and corresponding sizes
std::vector<std::vector<int> > send_ints(sendProcs.size());
std::vector<std::vector<double> > send_doubles(sendProcs.size());
std::vector<int> send_sizes(sendProcs.size());
std::vector<MPI_Request> mpiReqs(sendProcs.size()+recvProcs.size());
std::vector<MPI_Status> mpiStatuses(sendProcs.size()+recvProcs.size());
int tag1 = 11111;
int tag2 = 11112;
//first, the procs we're going to send to, have to let us know
//how much data we're supposed to send. So we have to receive
//sizes and then indices from the "send"-procs.
for(unsigned i=0; i<sendProcs.size(); ++i) {
MPI_Irecv(&send_sizes[i], 1, MPI_INT, sendProcs[i],
tag1, comm_, &mpiReqs[i]);
}
//now we'll send the sizes of our remotely-owned data to the
//procs that we will be receiving the data from, and also the
//indices that we want to receive.
for(unsigned i=0; i<recvProcs.size(); ++i) {
int proc = recvProcs[i];
int size = remotelyOwned_[proc]->size();
MPI_Send(&size, 1, MPI_INT, proc, tag1, comm_);
}
MPI_Waitall(sendProcs.size(), &mpiReqs[0], &mpiStatuses[0]);
//now resize our send_ints and send_doubles arrays, and post the recvs
//for indices that we're supposed to pack.
for(unsigned i=0; i<sendProcs.size(); ++i) {
int proc = sendProcs[i];
int size = send_sizes[i];
send_ints[i].resize(size);
MPI_Irecv(&(send_ints[i][0]), size, MPI_INT, proc, tag1,
comm_, &mpiReqs[i]);
send_doubles[i].resize(size);
}
//now send the indices that we want to receive data for.
for(unsigned i=0; i<recvProcs.size(); ++i) {
int proc = recvProcs[i];
int size = remotelyOwned_[proc]->size();
int* indices = &(remotelyOwned_[proc]->indices())[0];
MPI_Send(indices, size, MPI_INT, proc, tag1, comm_);
}
MPI_Waitall(sendProcs.size(), &mpiReqs[0], &mpiStatuses[0]);
//now post our recvs.
for(unsigned i=0; i<recvProcs.size(); ++i) {
int proc = recvProcs[i];
int size = remotelyOwned_[proc]->size();
double* coefs = &(remotelyOwned_[proc]->coefs())[0];
MPI_Irecv(coefs, size, MPI_DOUBLE, proc, tag2, comm_, &mpiReqs[i]);
}
//now pack and send the coefs that the other procs need from us.
for(unsigned i=0; i<sendProcs.size(); ++i) {
int proc = sendProcs[i];
int num = send_sizes[i];
int err = copyOutOfUnderlyingVector(num, &(send_ints[i][0]),
&(send_doubles[i][0]), 0);
if (err != 0) {
FEI_COUT << "fei::Vector_core::scatterToOverlap ERROR getting data to send."<<FEI_ENDL;
return(err);
}
MPI_Send(&(send_doubles[i][0]), num, MPI_DOUBLE, proc,
tag2, comm_);
}
MPI_Waitall(recvProcs.size(), &mpiReqs[0], &mpiStatuses[0]);
#endif //#ifndef FEI_SER
return(0);
}