int fei::Vector_core::giveToVector(int numValues,
const int* indices,
const double* values,
bool sumInto,
int vectorIndex)
{
int prev_proc = -1;
fei::CSVec* prev_vec = NULL;
for(int i=0; i<numValues; ++i) {
int ind = indices[i];
double val = values[i];
if (ind < 0) {
// throw std::runtime_error("negative index not allowed");
//preservation of existing behavior: certain Sierra scenarios
//involve passing negative indices for positions that should be
//ignored... so we'll continue rather than throwing.
continue;
}
int local = ind - firstLocalOffset_;
if (local < 0 || local >= numLocal_) {
int proc = eqnComm_->getOwnerProc(ind);
if (output_level_ >= fei::BRIEF_LOGS && output_stream_ != NULL) {
FEI_OSTREAM& os = *output_stream_;
os << dbgprefix_<<"giveToVector remote["<<proc<<"]("
<<ind<<","<<val<<")"<<FEI_ENDL;
}
fei::CSVec* remoteVec = prev_vec;
if (proc != prev_proc) {
remoteVec = getRemotelyOwned(proc);
prev_vec = remoteVec;
prev_proc = proc;
}
if (sumInto) {
fei::add_entry( *remoteVec, ind, val);
}
else {
fei::put_entry( *remoteVec, ind, val);
}
}
else {
int err = giveToUnderlyingVector(1, &ind, &val, sumInto, vectorIndex);
if (err != 0) {
fei::console_out() << "giveToVector sumIn ERROR, ind: " << ind
<< ", val: " << val << FEI_ENDL;
ERReturn(-1);
}
}
}
return(0);
}
int fei::Vector_core::gatherFromOverlap(bool accumulate)
{
if (fei::numProcs(comm_) == 1 || haveFEVector()) {
return(0);
}
#ifndef FEI_SER
//first create the list of procs we'll be sending to.
std::vector<int> sendProcs;
for(unsigned i=0; i<remotelyOwned_.size(); ++i) {
if ((int)i == fei::localProc(comm_)) continue;
if (remotelyOwned_[i]->size() == 0) continue;
sendProcs.push_back(i);
}
std::vector<int> recvProcs;
fei::mirrorProcs(comm_, sendProcs, recvProcs);
//declare arrays to hold the indices and coefs we'll be receiving.
std::vector<std::vector<int> > recv_ints(recvProcs.size());
std::vector<std::vector<double> > recv_doubles(recvProcs.size());
std::vector<int> recv_sizes(recvProcs.size());
std::vector<MPI_Request> mpiReqs(recvProcs.size()*2);
std::vector<MPI_Status> mpiStatuses(recvProcs.size()*2);
int tag1 = 11111;
int tag2 = 11112;
//post the recvs for the sizes.
for(size_t i=0; i<recvProcs.size(); ++i) {
int proc = recvProcs[i];
MPI_Irecv(&recv_sizes[i], 1, MPI_INT, proc,
tag1, comm_, &mpiReqs[i]);
}
//send the sizes of data we'll be sending.
for(unsigned i=0; i<sendProcs.size(); ++i) {
int proc = sendProcs[i];
int size = remotelyOwned_[proc]->size();
MPI_Send(&size, 1, MPI_INT, proc, tag1, comm_);
}
if (recvProcs.size() > 0) {
MPI_Waitall(recvProcs.size(), &mpiReqs[0], &mpiStatuses[0]);
}
//now post the recvs for the data.
unsigned offset = 0;
for(size_t i=0; i<recvProcs.size(); ++i) {
int proc = recvProcs[i];
int size = recv_sizes[i];
std::vector<int>& recv_ints_i = recv_ints[i];
std::vector<double>& recv_doubles_i = recv_doubles[i];
recv_ints_i.resize(size);
recv_doubles_i.resize(size);
MPI_Irecv(&(recv_ints_i[0]), size, MPI_INT, proc,
tag1, comm_, &mpiReqs[offset++]);
MPI_Irecv(&(recv_doubles_i[0]), size, MPI_DOUBLE, proc,
tag2, comm_, &mpiReqs[offset++]);
}
//now send the outgoing data.
for(size_t i=0; i<sendProcs.size(); ++i) {
int proc = sendProcs[i];
int size = remotelyOwned_[proc]->size();
int* indices = &(remotelyOwned_[proc]->indices())[0];
MPI_Send(indices, size, MPI_INT, proc, tag1, comm_);
double* coefs = &(remotelyOwned_[proc]->coefs())[0];
MPI_Send(coefs, size, MPI_DOUBLE, proc, tag2, comm_);
fei::set_values(*remotelyOwned_[proc], 0.0);
}
if (recvProcs.size() > 0) {
MPI_Waitall(recvProcs.size()*2, &mpiReqs[0], &mpiStatuses[0]);
}
//now store the data we've received.
for(size_t i=0; i<recvProcs.size(); ++i) {
int num = recv_sizes[i];
std::vector<int>& recv_ints_i = recv_ints[i];
std::vector<double>& recv_doubles_i = recv_doubles[i];
int err = giveToUnderlyingVector(num, &(recv_ints_i[0]),
&(recv_doubles_i[0]), accumulate, 0);
if (err != 0) {
FEI_COUT << "fei::Vector_core::gatherFromOverlap ERROR storing recvd data" << FEI_ENDL;
return(err);
}
}
#endif //#ifndef FEI_SER
return(0);
}
int fei::Vector_core::gatherFromOverlap(bool accumulate)
{
if (fei::numProcs(comm_) == 1 || haveFEVector()) {
return(0);
}
#ifndef FEI_SER
std::vector<MPI_Request> mpiReqs;
int tag1 = 11111;
if (sendRecvProcsNeedUpdated_) {
setCommSizes();
}
mpiReqs.resize(recvProcs_.size());
//now post the recvs for the data.
for(size_t i=0; i<recvProcs_.size(); ++i) {
MPI_Irecv(&(recv_chars_[i][0]), recv_sizes_[i], MPI_CHAR, recvProcs_[i],
tag1, comm_, &mpiReqs[i]);
}
bool resize_buffer = false;
bool zero_remotely_owned_after_packing = true;
pack_send_buffers(sendProcs_, remotelyOwned_, send_chars_,
resize_buffer, zero_remotely_owned_after_packing);
//now send the outgoing data.
for(size_t i=0; i<sendProcs_.size(); ++i) {
int proc = sendProcs_[i];
int size = send_chars_[i].size();
MPI_Send(&(send_chars_[i][0]), size, MPI_CHAR, proc, tag1, comm_);
}
int numRecvProcs = recvProcs_.size();
for(size_t i=0; i<recvProcs_.size(); ++i) {
int index;
MPI_Status status;
MPI_Waitany(numRecvProcs, &mpiReqs[0], &index, &status);
}
std::vector<int> indices;
std::vector<double> coefs;
//now store the data we've received.
for(size_t i=0; i<recvProcs_.size(); ++i) {
fei::impl_utils::unpack_indices_coefs(recv_chars_[i], indices, coefs);
int num = indices.size();
if (num == 0) continue;
int err = giveToUnderlyingVector(num, &(indices[0]),
&(coefs[0]), accumulate, 0);
if (err != 0) {
// FEI_COUT << "fei::Vector_core::gatherFromOverlap ERROR storing recvd data" << FEI_ENDL;
return(err);
}
}
#endif //#ifndef FEI_SER
return(0);
}
