void transfH_MPI(mblock<double>** Ap, mblock<float>** A, unsigned* seq_part)
{
unsigned rank = COMM_AHMED.Get_rank(),
nproc = COMM_AHMED.Get_size();
unsigned info[8];
if (rank==0) {
copyH(seq_part[1]-seq_part[0], Ap, A);
for (unsigned j=1; j<nproc; ++j) {
for (unsigned i=seq_part[j]; i<seq_part[j+1]; ++i) {
COMM_AHMED.Recv(info, 8, MPI::UNSIGNED, j, 7);
A[i] = new mblock<float>(info[0], info[1]);
float* tmp = NULL;
if (info[7]) {
tmp = new float[info[7]];
COMM_AHMED.Recv(tmp, info[7], MPI::FLOAT, j, 8);
}
A[i]->cpy_mbl(info+2, tmp);
delete [] tmp;
}
}
} else {
for (unsigned i=seq_part[rank]; i<seq_part[rank+1]; ++i) {
mblock<double>* p = Ap[i-seq_part[rank]];
info[0] = p->getn1();
info[1] = p->getn2();
p->get_prop(info+2);
info[7] = p->nvals();
COMM_AHMED.Send(info, 8, MPI::UNSIGNED, 0, 7);
if (info[7]) {
float* tmp = new float[info[7]];
for (unsigned i=0; i<info[7]; ++i) tmp[i] = (float) p->getdata()[i];
COMM_AHMED.Send(tmp, info[7], MPI::FLOAT, 0, 8);
delete [] tmp;
}
}
}
}
/*
* Send a block.
*/
void MemoryOArchive::send(MPI::Intracomm& comm, int dest)
{
int comm_size = comm.Get_size();
int myRank = comm.Get_rank();
// Preconditions
if (dest > comm_size - 1 || dest < 0) {
UTIL_THROW("Destination rank out of bounds");
}
if (dest == myRank) {
UTIL_THROW("Source and desination identical");
}
size_t sendBytes = cursor_ - buffer_;
size_t* sizePtr = (size_t*)buffer_;
*sizePtr = sendBytes;
comm.Send(buffer_, sendBytes, MPI::UNSIGNED_CHAR, dest, 5);
}
bool sendDataTo(const std::vector<double>& inData, int dest)
{
bool isDataSent = false;
if ( intraComm != MPI::COMM_NULL)
{
log.Info() << "SEND [ " << getRank()
<< " --> "
<< dest
<< " ] data : "
<< inData
<< std::endl;
intraComm.Send( &(inData[0]),
inData.size(),
MPI::DOUBLE,
dest, /*tag*/ 100);
isDataSent = true;
}else
{
log.Err() << "PID " << getProcessId()
<< " failed to SEND\n";
}
return isDataSent;
}
void mltaGeHVec_MPI(double d, mblock<double>** A, double* x, double* y,
unsigned p, unsigned *seq_part, blcluster** blList)
{
unsigned rank = COMM_AHMED.Get_rank();
unsigned info[3]; // nblcks, beg2, n2
if (rank == 0)
{
for (unsigned j=1; j<p; j++)
{
info[0] = seq_part[j+1] - seq_part[j];
dimsx_(blList+seq_part[j], info[0], info[1], info[2]);
COMM_AHMED.Send(info, 3, MPI::UNSIGNED, j, 1);
COMM_AHMED.Send(x+info[1], info[2], MPI::DOUBLE, j, 2);
}
// compute for processor 0
for (unsigned i=seq_part[0]; i<seq_part[1]; ++i)
{
unsigned idx = i-seq_part[0];
A[idx]->mltaVec(d, x+blList[i]->getb2(), y+blList[i]->getb1());
}
for (unsigned j=1; j<p; ++j)
{
unsigned dimsy[2];
COMM_AHMED.Recv(dimsy, 2, MPI::UNSIGNED, j, 3);
double* tmp = new double[dimsy[1]];
COMM_AHMED.Recv(tmp, dimsy[1], MPI::DOUBLE, j, 4);
blas::add(dimsy[1], tmp, y+dimsy[0]);
delete [] tmp;
}
}
else
{
COMM_AHMED.Recv(info, 3, MPI::UNSIGNED, 0, 1);
// receive required part of x
double* xp = new double[info[2]];
COMM_AHMED.Recv(xp, info[2], MPI::DOUBLE, 0, 2);
unsigned dimsy[2];
dimsy_(blList+seq_part[rank], info[0], dimsy[0], dimsy[1]);
double* tmp = new double[dimsy[1]];
blas::setzero(dimsy[1], tmp);
for (unsigned i=seq_part[rank]; i<seq_part[rank+1]; ++i)
{
unsigned idx = i-seq_part[rank];
A[idx]->mltaVec(d, xp+blList[i]->getb2()-info[1],
tmp+blList[i]->getb1()-dimsy[0]);
}
COMM_AHMED.Send(dimsy, 2, MPI::UNSIGNED, 0, 3);
COMM_AHMED.Send(tmp, dimsy[1], MPI::DOUBLE, 0, 4);
delete [] tmp;
delete [] xp;
}
}
void worker_process(const MPI::Intracomm &comm_world, const int manager_rank, const int rank, const Mat& Q, const vector<float> &lut) {
char flag = MANAGER_READY;
int inputSize;
int outputSize;
int maskSize;
char *input;
char *output;
char *mask;
comm_world.Barrier();
uint64_t t0, t1;
printf("worker %d ready\n", rank);
while (flag != MANAGER_FINISHED && flag != MANAGER_ERROR) {
t0 = cci::common::event::timestampInUS();
// tell the manager - ready
comm_world.Send(&WORKER_READY, 1, MPI::CHAR, manager_rank, TAG_CONTROL);
// printf("worker %d signal ready\n", rank);
// get the manager status
comm_world.Recv(&flag, 1, MPI::CHAR, manager_rank, TAG_CONTROL);
// printf("worker %d received manager status %d\n", rank, flag);
if (flag == MANAGER_READY) {
// get data from manager
comm_world.Recv(&inputSize, 1, MPI::INT, manager_rank, TAG_METADATA);
comm_world.Recv(&maskSize, 1, MPI::INT, manager_rank, TAG_METADATA);
comm_world.Recv(&outputSize, 1, MPI::INT, manager_rank, TAG_METADATA);
// allocate the buffers
input = new char[inputSize];
mask = new char[maskSize];
output = new char[outputSize];
memset(input, 0, inputSize * sizeof(char));
memset(mask, 0, maskSize * sizeof(char));
memset(output, 0, outputSize * sizeof(char));
// get the file names
comm_world.Recv(input, inputSize, MPI::CHAR, manager_rank, TAG_DATA);
comm_world.Recv(mask, maskSize, MPI::CHAR, manager_rank, TAG_DATA);
comm_world.Recv(output, outputSize, MPI::CHAR, manager_rank, TAG_DATA);
t0 = cci::common::event::timestampInUS();
// printf("comm time for worker %d is %lu us\n", rank, t1 -t0);
//printf("worker %d processing \"%s\"\n", rank, mask);
// now do some work
compute(input, mask, output, Q, lut);
t1 = cci::common::event::timestampInUS();
// printf("worker %d processed \"%s\" + \"%s\" -> \"%s\" in %lu us\n", rank, input, mask, output, t1 - t0);
printf("worker %d processed \"%s\" in %lu us\n", rank, mask, t1 - t0);
// clean up
delete [] input;
delete [] mask;
delete [] output;
}
}
}
void manager_process(const MPI::Intracomm &comm_world, const int manager_rank, const int worker_size, std::string &maskName, std::string &imgDir, std::string &outDir, bool overwrite) {
// first get the list of files to process
std::vector<std::string> filenames;
std::vector<std::string> seg_output;
std::vector<std::string> features_output;
uint64_t t1, t0;
t0 = cci::common::event::timestampInUS();
getFiles(maskName, imgDir, outDir, filenames, seg_output, features_output, overwrite);
t1 = cci::common::event::timestampInUS();
printf("Manager ready at %d, file read took %lu us\n", manager_rank, t1 - t0);
comm_world.Barrier();
// now start the loop to listen for messages
int curr = 0;
int total = filenames.size();
MPI::Status status;
int worker_id;
char ready;
char *input;
char *mask;
char *output;
int inputlen;
int masklen;
int outputlen;
while (curr < total) {
usleep(1000);
if (comm_world.Iprobe(MPI_ANY_SOURCE, TAG_CONTROL, status)) {
/* where is it coming from */
worker_id=status.Get_source();
comm_world.Recv(&ready, 1, MPI::CHAR, worker_id, TAG_CONTROL);
// printf("manager received request from worker %d\n",worker_id);
if (worker_id == manager_rank) continue;
if(ready == WORKER_READY) {
// tell worker that manager is ready
comm_world.Send(&MANAGER_READY, 1, MPI::CHAR, worker_id, TAG_CONTROL);
// printf("manager signal transfer\n");
/* send real data */
inputlen = filenames[curr].size() + 1; // add one to create the zero-terminated string
masklen = seg_output[curr].size() + 1;
outputlen = features_output[curr].size() + 1;
input = new char[inputlen];
memset(input, 0, sizeof(char) * inputlen);
strncpy(input, filenames[curr].c_str(), inputlen);
mask = new char[masklen];
memset(mask, 0, sizeof(char) * masklen);
strncpy(mask, seg_output[curr].c_str(), masklen);
output = new char[outputlen];
memset(output, 0, sizeof(char) * outputlen);
strncpy(output, features_output[curr].c_str(), outputlen);
comm_world.Send(&inputlen, 1, MPI::INT, worker_id, TAG_METADATA);
comm_world.Send(&masklen, 1, MPI::INT, worker_id, TAG_METADATA);
comm_world.Send(&outputlen, 1, MPI::INT, worker_id, TAG_METADATA);
// now send the actual string data
comm_world.Send(input, inputlen, MPI::CHAR, worker_id, TAG_DATA);
comm_world.Send(mask, masklen, MPI::CHAR, worker_id, TAG_DATA);
comm_world.Send(output, outputlen, MPI::CHAR, worker_id, TAG_DATA);
curr++;
delete [] input;
delete [] mask;
delete [] output;
}
if (curr % 100 == 1) {
printf("[ MANAGER STATUS ] %d tasks remaining.\n", total - curr);
}
}
}
/* tell everyone to quit */
int active_workers = worker_size;
while (active_workers > 0) {
usleep(1000);
if (comm_world.Iprobe(MPI_ANY_SOURCE, TAG_CONTROL, status)) {
/* where is it coming from */
worker_id=status.Get_source();
comm_world.Recv(&ready, 1, MPI::CHAR, worker_id, TAG_CONTROL);
// printf("manager received request from worker %d\n",worker_id);
if (worker_id == manager_rank) continue;
if(ready == WORKER_READY) {
comm_world.Send(&MANAGER_FINISHED, 1, MPI::CHAR, worker_id, TAG_CONTROL);
// printf("manager signal finished\n");
--active_workers;
}
}
}
}