int fei::Vector_core::giveToVector(int numValues, const int* indices, const double* values, bool sumInto, int vectorIndex) { int prev_proc = -1; fei::CSVec* prev_vec = NULL; for(int i=0; i<numValues; ++i) { int ind = indices[i]; double val = values[i]; if (ind < 0) { // throw std::runtime_error("negative index not allowed"); //preservation of existing behavior: certain Sierra scenarios //involve passing negative indices for positions that should be //ignored... so we'll continue rather than throwing. continue; } int local = ind - firstLocalOffset_; if (local < 0 || local >= numLocal_) { int proc = eqnComm_->getOwnerProc(ind); if (output_level_ >= fei::BRIEF_LOGS && output_stream_ != NULL) { FEI_OSTREAM& os = *output_stream_; os << dbgprefix_<<"giveToVector remote["<<proc<<"](" <<ind<<","<<val<<")"<<FEI_ENDL; } fei::CSVec* remoteVec = prev_vec; if (proc != prev_proc) { remoteVec = getRemotelyOwned(proc); prev_vec = remoteVec; prev_proc = proc; } if (sumInto) { fei::add_entry( *remoteVec, ind, val); } else { fei::put_entry( *remoteVec, ind, val); } } else { int err = giveToUnderlyingVector(1, &ind, &val, sumInto, vectorIndex); if (err != 0) { fei::console_out() << "giveToVector sumIn ERROR, ind: " << ind << ", val: " << val << FEI_ENDL; ERReturn(-1); } } } return(0); }
int fei::Vector_core::gatherFromOverlap(bool accumulate) { if (fei::numProcs(comm_) == 1 || haveFEVector()) { return(0); } #ifndef FEI_SER //first create the list of procs we'll be sending to. std::vector<int> sendProcs; for(unsigned i=0; i<remotelyOwned_.size(); ++i) { if ((int)i == fei::localProc(comm_)) continue; if (remotelyOwned_[i]->size() == 0) continue; sendProcs.push_back(i); } std::vector<int> recvProcs; fei::mirrorProcs(comm_, sendProcs, recvProcs); //declare arrays to hold the indices and coefs we'll be receiving. std::vector<std::vector<int> > recv_ints(recvProcs.size()); std::vector<std::vector<double> > recv_doubles(recvProcs.size()); std::vector<int> recv_sizes(recvProcs.size()); std::vector<MPI_Request> mpiReqs(recvProcs.size()*2); std::vector<MPI_Status> mpiStatuses(recvProcs.size()*2); int tag1 = 11111; int tag2 = 11112; //post the recvs for the sizes. for(size_t i=0; i<recvProcs.size(); ++i) { int proc = recvProcs[i]; MPI_Irecv(&recv_sizes[i], 1, MPI_INT, proc, tag1, comm_, &mpiReqs[i]); } //send the sizes of data we'll be sending. for(unsigned i=0; i<sendProcs.size(); ++i) { int proc = sendProcs[i]; int size = remotelyOwned_[proc]->size(); MPI_Send(&size, 1, MPI_INT, proc, tag1, comm_); } if (recvProcs.size() > 0) { MPI_Waitall(recvProcs.size(), &mpiReqs[0], &mpiStatuses[0]); } //now post the recvs for the data. unsigned offset = 0; for(size_t i=0; i<recvProcs.size(); ++i) { int proc = recvProcs[i]; int size = recv_sizes[i]; std::vector<int>& recv_ints_i = recv_ints[i]; std::vector<double>& recv_doubles_i = recv_doubles[i]; recv_ints_i.resize(size); recv_doubles_i.resize(size); MPI_Irecv(&(recv_ints_i[0]), size, MPI_INT, proc, tag1, comm_, &mpiReqs[offset++]); MPI_Irecv(&(recv_doubles_i[0]), size, MPI_DOUBLE, proc, tag2, comm_, &mpiReqs[offset++]); } //now send the outgoing data. for(size_t i=0; i<sendProcs.size(); ++i) { int proc = sendProcs[i]; int size = remotelyOwned_[proc]->size(); int* indices = &(remotelyOwned_[proc]->indices())[0]; MPI_Send(indices, size, MPI_INT, proc, tag1, comm_); double* coefs = &(remotelyOwned_[proc]->coefs())[0]; MPI_Send(coefs, size, MPI_DOUBLE, proc, tag2, comm_); fei::set_values(*remotelyOwned_[proc], 0.0); } if (recvProcs.size() > 0) { MPI_Waitall(recvProcs.size()*2, &mpiReqs[0], &mpiStatuses[0]); } //now store the data we've received. for(size_t i=0; i<recvProcs.size(); ++i) { int num = recv_sizes[i]; std::vector<int>& recv_ints_i = recv_ints[i]; std::vector<double>& recv_doubles_i = recv_doubles[i]; int err = giveToUnderlyingVector(num, &(recv_ints_i[0]), &(recv_doubles_i[0]), accumulate, 0); if (err != 0) { FEI_COUT << "fei::Vector_core::gatherFromOverlap ERROR storing recvd data" << FEI_ENDL; return(err); } } #endif //#ifndef FEI_SER return(0); }
int fei::Vector_core::gatherFromOverlap(bool accumulate) { if (fei::numProcs(comm_) == 1 || haveFEVector()) { return(0); } #ifndef FEI_SER std::vector<MPI_Request> mpiReqs; int tag1 = 11111; if (sendRecvProcsNeedUpdated_) { setCommSizes(); } mpiReqs.resize(recvProcs_.size()); //now post the recvs for the data. for(size_t i=0; i<recvProcs_.size(); ++i) { MPI_Irecv(&(recv_chars_[i][0]), recv_sizes_[i], MPI_CHAR, recvProcs_[i], tag1, comm_, &mpiReqs[i]); } bool resize_buffer = false; bool zero_remotely_owned_after_packing = true; pack_send_buffers(sendProcs_, remotelyOwned_, send_chars_, resize_buffer, zero_remotely_owned_after_packing); //now send the outgoing data. for(size_t i=0; i<sendProcs_.size(); ++i) { int proc = sendProcs_[i]; int size = send_chars_[i].size(); MPI_Send(&(send_chars_[i][0]), size, MPI_CHAR, proc, tag1, comm_); } int numRecvProcs = recvProcs_.size(); for(size_t i=0; i<recvProcs_.size(); ++i) { int index; MPI_Status status; MPI_Waitany(numRecvProcs, &mpiReqs[0], &index, &status); } std::vector<int> indices; std::vector<double> coefs; //now store the data we've received. for(size_t i=0; i<recvProcs_.size(); ++i) { fei::impl_utils::unpack_indices_coefs(recv_chars_[i], indices, coefs); int num = indices.size(); if (num == 0) continue; int err = giveToUnderlyingVector(num, &(indices[0]), &(coefs[0]), accumulate, 0); if (err != 0) { // FEI_COUT << "fei::Vector_core::gatherFromOverlap ERROR storing recvd data" << FEI_ENDL; return(err); } } #endif //#ifndef FEI_SER return(0); }