//********************************************************************
//
// Chang Robert's algorithm function
//
// This function performs Chang and Robert's algrithom
//
// Return Value
// ------------
// void
//
//
// Reference Parameters
// --------------------
// rank INT MPI processor rank number
// id INT* MPI processor random id
// size INT MPI processor numbers
// right INT The right neighbor of processor
// left INT The left neighbor of processor
// state_status INT Define the participation of a processor
// count INT Count how many phase the processor has been through
// election INT* The election message array
// elected INT* The elected message array
// requests MPI_REQUEST The MPI requests in unblocking message
// status MPI_STATUS The MPI status
// flag INT The MPI flags used in MPI_Test
//
// Local Variables
// ---------------
// NONE
//*******************************************************************
void chang_robert(int rank, int size, int* id, int right, int left, int* count, int* flag, int* election, int* elected, int* state_status, MPI_Request requests, MPI_Status status)
{
MPI_Irecv(election, 2, MPI_INT, left, 100, MPI_COMM_WORLD, &requests);
MPI_Test(&requests, &flag[left], &status);
while (!flag[left])
MPI_Test(&requests, &flag[left], &status);
if (election[1] == -1)
{
*count++;
if (election[0] > *id)
{
printf("Phase: %d, Rank: %d, Identifier: %d, Status: Passive\n", *count+1, rank, *id);
fflush(stdout);
if (*state_status == NONPARTICIPANT)
*state_status = PARTICIPANT;
MPI_Send(election, 2, MPI_INT, right, 100, MPI_COMM_WORLD);
}
else if (election[0] < *id)
{
printf("Phase: %d, Rank: %d, Identifier: %d, Status: Active\n", *count+1, rank, *id);
fflush(stdout);
if (*state_status == NONPARTICIPANT)
{
*state_status = PARTICIPANT;
election[0] = *id;
MPI_Send(election, 2, MPI_INT, right, 100, MPI_COMM_WORLD);
}
}
else
{
printf("Phase: %d, Rank: %d, Identifier: %d, Status: Active\n", *count+1, rank, *id);
fflush(stdout);
*state_status = NONPARTICIPANT;
election[1] = rank;
MPI_Send(election, 2, MPI_INT, right, 100, MPI_COMM_WORLD);
fflush(stdout);
}
}
else
{
elected[0] = election[0];
elected[1] = election[1];
election = NULL;
if (elected[0] != *id)
{
*state_status = NONPARTICIPANT;
MPI_Send(elected, 2, MPI_INT, right, 100, MPI_COMM_WORLD);
}
else if(elected[0] == *id)
{
printf("I am the Leader and my Rank is: %d and my Identifier is: %d\n", rank, *id);
fflush(stdout);
}
}
}
void peano::applications::faxen::repositories::FaxenBatchJobRepositoryStatePacked::receive(int source, int tag) {
MPI_Request* sendRequestHandle = new MPI_Request();
MPI_Status status;
int flag = 0;
int result;
clock_t timeOutWarning = -1;
clock_t timeOutShutdown = -1;
bool triggeredTimeoutWarning = false;
result = MPI_Irecv(
this, 1, Datatype, source, tag,
tarch::parallel::Node::getInstance().getCommunicator(), sendRequestHandle
);
if ( result != MPI_SUCCESS ) {
std::ostringstream msg;
msg << "failed to start to receive peano::applications::faxen::repositories::FaxenBatchJobRepositoryStatePacked from node "
<< source << ": " << tarch::parallel::MPIReturnValueToString(result);
_log.error( "receive(int)", msg.str() );
}
result = MPI_Test( sendRequestHandle, &flag, &status );
while (!flag) {
if (timeOutWarning==-1) timeOutWarning = tarch::parallel::Node::getInstance().getDeadlockWarningTimeStamp();
if (timeOutShutdown==-1) timeOutShutdown = tarch::parallel::Node::getInstance().getDeadlockTimeOutTimeStamp();
result = MPI_Test( sendRequestHandle, &flag, &status );
if (result!=MPI_SUCCESS) {
std::ostringstream msg;
msg << "testing for finished receive task for peano::applications::faxen::repositories::FaxenBatchJobRepositoryStatePacked failed: "
<< tarch::parallel::MPIReturnValueToString(result);
_log.error("receive(int)", msg.str() );
}
// deadlock aspect
if (
tarch::parallel::Node::getInstance().isTimeOutWarningEnabled() &&
(clock()>timeOutWarning) &&
(!triggeredTimeoutWarning)
) {
tarch::parallel::Node::getInstance().writeTimeOutWarning(
"peano::applications::faxen::repositories::FaxenBatchJobRepositoryStatePacked",
"receive(int)", source
);
triggeredTimeoutWarning = true;
}
if (
tarch::parallel::Node::getInstance().isTimeOutDeadlockEnabled() &&
(clock()>timeOutShutdown)
) {
tarch::parallel::Node::getInstance().triggerDeadlockTimeOut(
"peano::applications::faxen::repositories::FaxenBatchJobRepositoryStatePacked",
"receive(int)", source
);
}
tarch::parallel::Node::getInstance().receiveDanglingMessages();
}
delete sendRequestHandle;
_senderRank = status.MPI_SOURCE;
#ifdef Debug
_log.debug("receive(int,int)", "received " + toString() );
#endif
}
inline bool test_impl(request_impl* r){
int f = 0; MPI_Test(&r->mpi_request, &f, MPI_STATUS_IGNORE); return f;
}
static PetscErrorCode PetscCommBuildTwoSided_Ibarrier(MPI_Comm comm,PetscMPIInt count,MPI_Datatype dtype,PetscMPIInt nto,const PetscMPIInt *toranks,const void *todata,PetscMPIInt *nfrom,PetscMPIInt **fromranks,void *fromdata)
{
PetscErrorCode ierr;
PetscMPIInt nrecvs,tag,done,i;
MPI_Aint lb,unitbytes;
char *tdata;
MPI_Request *sendreqs,barrier;
PetscSegBuffer segrank,segdata;
PetscBool barrier_started;
PetscFunctionBegin;
ierr = PetscCommDuplicate(comm,&comm,&tag);CHKERRQ(ierr);
ierr = MPI_Type_get_extent(dtype,&lb,&unitbytes);CHKERRQ(ierr);
if (lb != 0) SETERRQ1(comm,PETSC_ERR_SUP,"Datatype with nonzero lower bound %ld\n",(long)lb);
tdata = (char*)todata;
ierr = PetscMalloc1(nto,&sendreqs);CHKERRQ(ierr);
for (i=0; i<nto; i++) {
ierr = MPI_Issend((void*)(tdata+count*unitbytes*i),count,dtype,toranks[i],tag,comm,sendreqs+i);CHKERRQ(ierr);
}
ierr = PetscSegBufferCreate(sizeof(PetscMPIInt),4,&segrank);CHKERRQ(ierr);
ierr = PetscSegBufferCreate(unitbytes,4*count,&segdata);CHKERRQ(ierr);
nrecvs = 0;
barrier = MPI_REQUEST_NULL;
/* MPICH-3.2 sometimes does not create a request in some "optimized" cases. This is arguably a standard violation,
* but we need to work around it. */
barrier_started = PETSC_FALSE;
for (done=0; !done; ) {
PetscMPIInt flag;
MPI_Status status;
ierr = MPI_Iprobe(MPI_ANY_SOURCE,tag,comm,&flag,&status);CHKERRQ(ierr);
if (flag) { /* incoming message */
PetscMPIInt *recvrank;
void *buf;
ierr = PetscSegBufferGet(segrank,1,&recvrank);CHKERRQ(ierr);
ierr = PetscSegBufferGet(segdata,count,&buf);CHKERRQ(ierr);
*recvrank = status.MPI_SOURCE;
ierr = MPI_Recv(buf,count,dtype,status.MPI_SOURCE,tag,comm,MPI_STATUS_IGNORE);CHKERRQ(ierr);
nrecvs++;
}
if (!barrier_started) {
PetscMPIInt sent,nsends;
ierr = PetscMPIIntCast(nto,&nsends);CHKERRQ(ierr);
ierr = MPI_Testall(nsends,sendreqs,&sent,MPI_STATUSES_IGNORE);CHKERRQ(ierr);
if (sent) {
#if defined(PETSC_HAVE_MPI_IBARRIER)
ierr = MPI_Ibarrier(comm,&barrier);CHKERRQ(ierr);
#elif defined(PETSC_HAVE_MPIX_IBARRIER)
ierr = MPIX_Ibarrier(comm,&barrier);CHKERRQ(ierr);
#endif
barrier_started = PETSC_TRUE;
ierr = PetscFree(sendreqs);CHKERRQ(ierr);
}
} else {
ierr = MPI_Test(&barrier,&done,MPI_STATUS_IGNORE);CHKERRQ(ierr);
}
}
*nfrom = nrecvs;
ierr = PetscSegBufferExtractAlloc(segrank,fromranks);CHKERRQ(ierr);
ierr = PetscSegBufferDestroy(&segrank);CHKERRQ(ierr);
ierr = PetscSegBufferExtractAlloc(segdata,fromdata);CHKERRQ(ierr);
ierr = PetscSegBufferDestroy(&segdata);CHKERRQ(ierr);
ierr = PetscCommDestroy(&comm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static int ADIOI_GEN_irc_poll_fn(void *extra_state, MPI_Status *status)
{
ADIOI_NBC_Request *nbc_req;
ADIOI_GEN_IreadStridedColl_vars *rsc_vars = NULL;
ADIOI_Icalc_others_req_vars *cor_vars = NULL;
ADIOI_Iread_and_exch_vars *rae_vars = NULL;
ADIOI_R_Iexchange_data_vars *red_vars = NULL;
int errcode = MPI_SUCCESS;
int flag;
nbc_req = (ADIOI_NBC_Request *)extra_state;
switch (nbc_req->data.rd.state) {
case ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL:
rsc_vars = nbc_req->data.rd.rsc_vars;
errcode = MPI_Testall(2, rsc_vars->req_offset, &flag,
MPI_STATUSES_IGNORE);
if (errcode == MPI_SUCCESS && flag) {
ADIOI_GEN_IreadStridedColl_inter(nbc_req, &errcode);
}
break;
case ADIOI_IRC_STATE_GEN_IREADSTRIDEDCOLL_INDIO:
rsc_vars = nbc_req->data.rd.rsc_vars;
errcode = MPI_Test(&rsc_vars->req_ind_io, &flag, MPI_STATUS_IGNORE);
if (errcode == MPI_SUCCESS && flag) {
/* call the last function */
ADIOI_GEN_IreadStridedColl_fini(nbc_req, &errcode);
}
break;
case ADIOI_IRC_STATE_ICALC_OTHERS_REQ:
cor_vars = nbc_req->cor_vars;
errcode = MPI_Test(&cor_vars->req1, &flag, MPI_STATUS_IGNORE);
if (errcode == MPI_SUCCESS && flag) {
ADIOI_Icalc_others_req_main(nbc_req, &errcode);
}
break;
case ADIOI_IRC_STATE_ICALC_OTHERS_REQ_MAIN:
cor_vars = nbc_req->cor_vars;
if (cor_vars->num_req2) {
errcode = MPI_Testall(cor_vars->num_req2, cor_vars->req2,
&flag, MPI_STATUSES_IGNORE);
if (errcode == MPI_SUCCESS && flag) {
ADIOI_Icalc_others_req_fini(nbc_req, &errcode);
}
} else {
ADIOI_Icalc_others_req_fini(nbc_req, &errcode);
}
break;
case ADIOI_IRC_STATE_IREAD_AND_EXCH:
rae_vars = nbc_req->data.rd.rae_vars;
errcode = MPI_Test(&rae_vars->req1, &flag, MPI_STATUS_IGNORE);
if (errcode == MPI_SUCCESS && flag) {
rae_vars->m = 0;
ADIOI_Iread_and_exch_l1_begin(nbc_req, &errcode);
}
break;
case ADIOI_IRC_STATE_IREAD_AND_EXCH_L1_BEGIN:
rae_vars = nbc_req->data.rd.rae_vars;
errcode = MPI_Test(&rae_vars->req2, &flag, MPI_STATUS_IGNORE);
if (errcode == MPI_SUCCESS && flag) {
ADIOI_R_Iexchange_data(nbc_req, &errcode);
}
break;
case ADIOI_IRC_STATE_R_IEXCHANGE_DATA:
red_vars = nbc_req->data.rd.red_vars;
errcode = MPI_Test(&red_vars->req1, &flag, MPI_STATUS_IGNORE);
if (errcode == MPI_SUCCESS && flag) {
ADIOI_R_Iexchange_data_recv(nbc_req, &errcode);
}
break;
case ADIOI_IRC_STATE_R_IEXCHANGE_DATA_RECV:
red_vars = nbc_req->data.rd.red_vars;
errcode = MPI_Testall(red_vars->nprocs_recv, red_vars->req2, &flag,
MPI_STATUSES_IGNORE);
if (errcode == MPI_SUCCESS && flag) {
ADIOI_R_Iexchange_data_fill(nbc_req, &errcode);
}
break;
case ADIOI_IRC_STATE_R_IEXCHANGE_DATA_FILL:
red_vars = nbc_req->data.rd.red_vars;
errcode = MPI_Testall(red_vars->nprocs_send,
red_vars->req2 + red_vars->nprocs_recv,
&flag, MPI_STATUSES_IGNORE);
if (errcode == MPI_SUCCESS && flag) {
ADIOI_R_Iexchange_data_fini(nbc_req, &errcode);
}
break;
default:
break;
}
/* --BEGIN ERROR HANDLING-- */
if (errcode != MPI_SUCCESS) {
errcode = MPIO_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE,
"ADIOI_GEN_irc_poll_fn", __LINE__,
MPI_ERR_IO, "**mpi_grequest_complete",
0);
}
/* --END ERROR HANDLING-- */
return errcode;
}
double
do_compute_and_probe(double seconds, MPI_Request* request)
{
double t1 = 0.0, t2 = 0.0;
double test_time = 0.0;
int num_tests = 0;
double target_seconds_for_compute = 0.0;
int flag = 0;
MPI_Status status;
if (options.num_probes) {
target_seconds_for_compute = (double) seconds/options.num_probes;
if (DEBUG) fprintf(stderr, "setting target seconds to %f\n", (target_seconds_for_compute * 1e6 ));
}
else {
target_seconds_for_compute = seconds;
if (DEBUG) fprintf(stderr, "setting target seconds to %f\n", (target_seconds_for_compute * 1e6 ));
}
#ifdef _ENABLE_CUDA_KERNEL_
if (options.target == gpu) {
if (options.num_probes) {
/* Do the dummy compute on GPU only */
do_compute_gpu(target_seconds_for_compute);
num_tests = 0;
while (num_tests < options.num_probes) {
t1 = MPI_Wtime();
MPI_Test(request, &flag, &status);
t2 = MPI_Wtime();
test_time += (t2-t1);
num_tests++;
}
}
else {
do_compute_gpu(target_seconds_for_compute);
}
}
else if (options.target == both) {
if (options.num_probes) {
/* Do the dummy compute on GPU and CPU*/
do_compute_gpu(target_seconds_for_compute);
num_tests = 0;
while (num_tests < options.num_probes) {
t1 = MPI_Wtime();
MPI_Test(request, &flag, &status);
t2 = MPI_Wtime();
test_time += (t2-t1);
num_tests++;
do_compute_cpu(target_seconds_for_compute);
}
}
else {
do_compute_gpu(target_seconds_for_compute);
do_compute_cpu(target_seconds_for_compute);
}
}
else
#endif
if (options.target == cpu) {
if (options.num_probes) {
num_tests = 0;
while (num_tests < options.num_probes) {
do_compute_cpu(target_seconds_for_compute);
t1 = MPI_Wtime();
MPI_Test(request, &flag, &status);
t2 = MPI_Wtime();
test_time += (t2-t1);
num_tests++;
}
}
else {
do_compute_cpu(target_seconds_for_compute);
}
}
#ifdef _ENABLE_CUDA_KERNEL_
if (options.target == gpu || options.target == both) {
cudaDeviceSynchronize();
cudaStreamDestroy(stream);
}
#endif
return test_time;
}
void tarch::parallel::messages::NodePoolAnswerMessagePacked::send(int destination, int tag, bool exchangeOnlyAttributesMarkedWithParallelise) {
MPI_Request* sendRequestHandle = new MPI_Request();
MPI_Status status;
int flag = 0;
int result;
clock_t timeOutWarning = -1;
clock_t timeOutShutdown = -1;
bool triggeredTimeoutWarning = false;
#ifdef Asserts
_senderRank = -1;
#endif
if (exchangeOnlyAttributesMarkedWithParallelise) {
result = MPI_Isend(
this, 1, Datatype, destination,
tag, tarch::parallel::Node::getInstance().getCommunicator(),
sendRequestHandle
);
}
else {
result = MPI_Isend(
this, 1, FullDatatype, destination,
tag, tarch::parallel::Node::getInstance().getCommunicator(),
sendRequestHandle
);
}
if (result!=MPI_SUCCESS) {
std::ostringstream msg;
msg << "was not able to send message tarch::parallel::messages::NodePoolAnswerMessagePacked "
<< toString()
<< " to node " << destination
<< ": " << tarch::parallel::MPIReturnValueToString(result);
_log.error( "send(int)",msg.str() );
}
result = MPI_Test( sendRequestHandle, &flag, &status );
while (!flag) {
if (timeOutWarning==-1) timeOutWarning = tarch::parallel::Node::getInstance().getDeadlockWarningTimeStamp();
if (timeOutShutdown==-1) timeOutShutdown = tarch::parallel::Node::getInstance().getDeadlockTimeOutTimeStamp();
result = MPI_Test( sendRequestHandle, &flag, &status );
if (result!=MPI_SUCCESS) {
std::ostringstream msg;
msg << "testing for finished send task for tarch::parallel::messages::NodePoolAnswerMessagePacked "
<< toString()
<< " sent to node " << destination
<< " failed: " << tarch::parallel::MPIReturnValueToString(result);
_log.error("send(int)", msg.str() );
}
// deadlock aspect
if (
tarch::parallel::Node::getInstance().isTimeOutWarningEnabled() &&
(clock()>timeOutWarning) &&
(!triggeredTimeoutWarning)
) {
tarch::parallel::Node::getInstance().writeTimeOutWarning(
"tarch::parallel::messages::NodePoolAnswerMessagePacked",
"send(int)", destination,tag,1
);
triggeredTimeoutWarning = true;
}
if (
tarch::parallel::Node::getInstance().isTimeOutDeadlockEnabled() &&
(clock()>timeOutShutdown)
) {
tarch::parallel::Node::getInstance().triggerDeadlockTimeOut(
"tarch::parallel::messages::NodePoolAnswerMessagePacked",
"send(int)", destination,tag,1
);
}
tarch::parallel::Node::getInstance().receiveDanglingMessages();
}
delete sendRequestHandle;
#ifdef Debug
_log.debug("send(int,int)", "sent " + toString() );
#endif
}
/* slave 进程 */
void worker()
{
printf("\tProcessor %d at %s begin work..\n", myid, processor_name);
MPI_Status status;
MPI_Request handle;
int recv_flag = 0;
int count = 0;
int upload = 0;
// 非阻塞接收主进程消息
MPI_Irecv(selectedGenes, n, MPI_GENETYPE, 0, MPI_ANY_TAG, MPI_COMM_WORLD, &handle);
while(1)
{
// 独立繁衍count代
count = generations;
while(count--)
{
select();
crossover();
mutate();
evaluate();
prefer();
// 若满足终止条件，则向主进程发送最优路径，并结束进程
if(population[CARDINALITY].fitness <= optimal+margin)
{
printf("\tProcessor %d at %s Terminated\n", myid, processor_name);
MPI_Send(&population[CARDINALITY], 1, MPI_GENETYPE, 0, DONE_TAG, MPI_COMM_WORLD);
printf("\tProcessor %d at %s exit\n", myid, processor_name);
return;
}
// 探测是否收到主进程的消息
MPI_Test(&handle, &recv_flag, &status);
// 若收到主进程的消息
if(recv_flag)
{
printf("\tProcessor %d at %s recv %d\n", myid, processor_name, status.MPI_TAG);
// 状态重置
recv_flag = 0;
// 若接收到DONE_TAG则结束进程
if(status.MPI_TAG == DONE_TAG)
{
printf("\tProcessor %d at %s exit\n", myid, processor_name);
return;
}
// 否则，将接收到的优良个体替换种群中最差的个体
qsort(population, CARDINALITY, sizeof(GeneType), compare);
for(int i=1; i <= n; i++)
assign(&population[CARDINALITY-i], &selectedGenes[i-1]);
if(selectedGenes[0].fitness < population[CARDINALITY].fitness)
assign(&population[CARDINALITY], &selectedGenes[0]);
// 非阻塞接收主进程消息
MPI_Irecv(selectedGenes, n, MPI_GENETYPE, 0, MPI_ANY_TAG, MPI_COMM_WORLD, &handle);
}
}
// 繁衍count代后，若没有终止则向主进程发送最优个体
select_N_best(n);
MPI_Send(selectedGenes, n, MPI_GENETYPE, 0, PUT_BETTER_TAG, MPI_COMM_WORLD);
printf("\tProcessor %d at %s upload %d\n", myid, processor_name, upload++);
}
}
int main (int argc, char *argv[]){
MPI_Init (&argc,&argv);
MPI_Comm_size (MPI_COMM_WORLD,&numtasks);
MPI_Comm_rank (MPI_COMM_WORLD,&rank);
STEPS = 50;
NX = 800;
NY = 400;
NZ = 30;
x_partition_count = 2;
y_partition_count = 1;
z_partition_count = 1;
for(i=1; i<argc; ++i) {
if(!strcmp(argv[i], "s"))
STEPS = atoi(argv[i+1]);
else if(!strcmp(argv[i], "x"))
NX = atoi(argv[i+1]);
else if(!strcmp(argv[i],"y"))
NY = atoi(argv[i+1]);
else if(!strcmp(argv[i],"z"))
NZ = atoi(argv[i+1]);
else if(!strcmp(argv[i],"px"))
x_partition_count = atoi(argv[i+1]);
else if(!strcmp(argv[i],"py"))
y_partition_count = atoi(argv[i+1]);
else if(!strcmp(argv[i],"pz"))
z_partition_count = atoi(argv[i+1]);
}
rank_mass = total_mass/(double)numtasks;
if(rank == 0) {
printf("\n\n==================================\n==========SESSION START===========\n==================================\n"
"Program size (x/y/z/steps): %d x %d x %d x %d\n"
"Partition grid (x/y/z): %d x %d x %d\n",
NX, NY, NZ, STEPS, x_partition_count, y_partition_count, z_partition_count);
fp = fopen("log.txt", "a");
fprintf(fp,"%dx%dx%dx%d\n%dx%dx%d\n%d processes\n\n",
NX, NY, NZ, STEPS, x_partition_count, y_partition_count, z_partition_count,
x_partition_count * y_partition_count * z_partition_count);
if(!(NX && NY && NZ && x_partition_count && y_partition_count && z_partition_count)){
puts("Elements/Grid zero, cannot continue\n"\
"Use -x <number> to input x elements count\n"\
"Use -y <number> to input y elements count\n"\
"Use -z <number> to input z elements count\n"\
"Use -s <number> to input step count\n"\
"Use -px <number> to input x-dimension partition count\n"\
"Use -py <number> to input y-dimension partition count\n"\
"Use -pz <number> to input z-dimension partition count\n"\
);
//getchar();
return;
}
}
#pragma omp parallel
{
if((rank==0) && (omp_get_thread_num() == 0))
printf("Internal element processing threads (OpenMP parallelization): %d\n", omp_get_num_threads());
}
MPI_Barrier(MPI_COMM_WORLD); //for printf to apper in order
// ================================START Data partition assignment================================
x_length = NX/x_partition_count; //Divide elements uniformly among partitions
x_rank = rank % x_partition_count; //rank, as is "partition rank"
x_start = x_rank * x_length ; //min
x_end = (x_rank+1) * x_length - 1;
y_length = NY/y_partition_count;
y_rank = (rank / x_partition_count ) % y_partition_count; //rank, as is "partition rank"
y_start = y_rank * y_length; //min
y_end = (y_rank+1) * y_length - 1;
z_length = NZ/z_partition_count;
z_rank = rank / (x_partition_count*y_partition_count);
z_start = z_rank * z_length; //min
z_end = (z_rank+1) * z_length - 1;
printf("Rank %d range: x(%d-%d) of %d, y(%d-%d) of %d, z(%d-%d) of %d\n",
rank, x_start, x_end, NX, y_start, y_end, NY, z_start, z_end, NZ);
//================================END Data partition assignment================================
//=====================================START Initialization====================================
duration_sendrecv = 0.0;
duration_internal = 0.0;
duration_busywait = 0.0;
duration_external = 0.0;
duration_mass_reduce= 0.0;
duration_waitsend = 0.0;
//Each of the arrays needs to have a size of (x+4)*(y +4)* z elements.
//The size must be identical for all so that MPI datatype column will work correctly.
//The +4 is the Halo zone for receives.
u[0] = (struct element*) malloc((x_length+4)*(y_length+4)*z_length*sizeof(struct element));
u[1] = (struct element*) malloc((x_length+4)*(y_length+4)*z_length*sizeof(struct element));
for (iz=0; iz<z_length; ++iz)
for (iy=0; iy<y_length+4; ++iy)
for (ix=0; ix<x_length+4; ++ix){
(u[0]+c(ix,iy,iz))->mass = total_mass/numtasks/NX/NY/NZ;
(u[0]+c(ix,iy,iz))->xy_value = (double)(rand() % 100);
(u[0]+c(ix,iy,iz))->z_value = a*pow((u[0]+c(ix,iy,iz))->xy_value,10.0);
(u[1]+c(ix,iy,iz))->mass = total_mass/numtasks/NX/NY/NZ;
(u[1]+c(ix,iy,iz))->xy_value = 0.0;
}
//iz: Track which of the u arrays is the "old"
iu = 0;
//sprintf(filename,"atm%ds%d.txt", rank, it);
//prtdat(filename);
//printf("Rank %d saving in %s\n", rank, filename);
// for printf to apper in order
MPI_Barrier(MPI_COMM_WORLD);
// DATATYPE: Notice how column size(1st arg) depends on partition size
// an element consists of 3 doubles
MPI_Type_vector (2, 3*x_length, 3*(x_length+4), MPI_DOUBLE, &xmargindata);
MPI_Type_commit (&xmargindata);
MPI_Type_vector (y_length, 6, 3*(x_length+4), MPI_DOUBLE, &ymargindata);
MPI_Type_commit (&ymargindata);
sizes[2] = 3*(x_length+4);
sizes[1] = y_length+4;
sizes[0] = z_length;
subsizes_right_left[2] = 3*2;
subsizes_right_left[1] = y_length;
subsizes_right_left[0] = z_length;
starts_right_left[0] = 0;
starts_right_left[1] = 0;
starts_right_left[2] = 0;
MPI_Type_create_subarray(3, sizes, subsizes_right_left, starts_right_left, MPI_ORDER_C, MPI_DOUBLE, &right_left_type);
MPI_Type_commit (&right_left_type);
subsizes_down_up[2] = 3*x_length;
subsizes_down_up[1] = 2;
subsizes_down_up[0] = z_length;
MPI_Type_create_subarray(3, sizes, subsizes_down_up, starts_right_left, MPI_ORDER_C, MPI_DOUBLE, &down_up_type);
MPI_Type_commit (&down_up_type);
printf("Rank %d has finished initialisation\n", rank);
//==============================================END Initialization==============================================
//Main Computation
for (it = 1; it <= STEPS; ++it) {
if(rank == 0) printf("Step %d\n", it);
time_start_sendrecv = MPI_Wtime();
//printf("Rank %d starts iteration %d\n",rank,it);
/* if(STEPS==1) printf("Rank %d neighbours: U %d D %d L %d R %d\n",rank,
((rank+x_partition_count) % (x_partition_count*y_partition_count)
+ (x_partition_count*y_partition_count)*(rank/x_partition_count/y_partition_count)),
((rank-x_partition_count+x_partition_count*y_partition_count) % (x_partition_count*y_partition_count)
+ (x_partition_count*y_partition_count)*(rank/x_partition_count/y_partition_count)),
(rank+(rank % x_partition_count ? 0 : x_partition_count)-1),
(rank+((rank+1) % x_partition_count ? 0 : -x_partition_count)+1));*/
MPI_Isend(u[iu]+c(2, y_length, 0), 1, down_up_type,\
(rank+x_partition_count) % (x_partition_count*y_partition_count) +
(x_partition_count*y_partition_count)*(rank/x_partition_count/y_partition_count),
DTUTAG, MPI_COMM_WORLD, req_send + 0);
MPI_Irecv(u[iu]+c(2, y_length+2, 0), 1, down_up_type,\
(rank+x_partition_count) % (x_partition_count*y_partition_count) +
(x_partition_count*y_partition_count)*(rank/x_partition_count/y_partition_count),
UTDTAG, MPI_COMM_WORLD, req_recv + 0);
MPI_Isend(u[iu]+c(2,2,0), 1, down_up_type,\
(rank-x_partition_count+x_partition_count*y_partition_count) % (x_partition_count*y_partition_count) +
(x_partition_count*y_partition_count)*(rank/x_partition_count/y_partition_count),
UTDTAG, MPI_COMM_WORLD, req_send + 1);
MPI_Irecv(u[iu]+c(2,0,0), 1, down_up_type,\
(rank-x_partition_count+x_partition_count*y_partition_count) % (x_partition_count*y_partition_count) +
(x_partition_count*y_partition_count)*(rank/x_partition_count/y_partition_count),
DTUTAG, MPI_COMM_WORLD, req_recv+1);
// use % due to spatial wraparound
MPI_Isend(u[iu]+c(2,2,0), 1, right_left_type,\
rank+(rank % x_partition_count ? 0 : x_partition_count)-1, RTLTAG, MPI_COMM_WORLD, req_send+2);
MPI_Irecv(u[iu]+c(0,2,0), 1, right_left_type,\
rank+(rank % x_partition_count ? 0 : x_partition_count)-1, LTRTAG, MPI_COMM_WORLD, req_recv+2);
MPI_Isend(u[iu]+c(x_length,2,0), 1, right_left_type,\
rank+((rank+1) % x_partition_count ? 0 : -x_partition_count)+1, LTRTAG, MPI_COMM_WORLD, req_send+3);
MPI_Irecv(u[iu]+c(x_length+2,2,0), 1, right_left_type,\
rank+((rank+1) % x_partition_count ? 0 : -x_partition_count)+1, RTLTAG, MPI_COMM_WORLD, req_recv+3);
//printf("Rank %d has finished nonblocking sendrecvs\n", rank);
duration_sendrecv += MPI_Wtime() - time_start_sendrecv;
//begin update of internal elements
time_start_internal = MPI_Wtime();
#pragma omp parallel
{
#pragma omp for
for(iz=0; iz<z_length; ++iz){ //full z range
//printf("Iteration %d is assigned to thread %d\n", iz, omp_get_thread_num());
//disregard both the data waiting to be received (width 2 perimeter) and the ones
//who need them to be calculated (another 2 width perimeter)(central elements)
for(iy=4; iy<y_length; ++iy)
for(ix=4; ix<x_length; ++ix)
update(ix, iy, iz, u[iu], u[1-iu]);
}
}
duration_internal += MPI_Wtime() - time_start_internal;
// printf("Rank %d has finished internal elements\n", rank);
// finished update of internal elements
time_start_busywait = MPI_Wtime();
done_count = 0;
memset(done, 0, 4*sizeof(int));
while(done_count<4){
for(i=0; i<4; ++i)
if(!done[i]){
MPI_Test(req_recv+i, done+i, MPI_STATUS_IGNORE);
if(done[i]){
switch(i){
case 0:
for(iz=0; iz<z_length; ++iz) //full z range
for(iy=y_length; iy<y_length+2; ++iy)
for(ix=2; ix<x_length+2; ++ix)
update(ix,iy,iz,u[iu],u[1-iu]);//update top row except corners
break;
case 1:
for(iz=0; iz<z_length; ++iz) //full z range
for(iy=2; iy<4; ++iy)
for(ix=2; ix<x_length+2; ++ix)
update(ix,iy,iz,u[iu],u[1-iu]);//update bottom row except corners
break;
case 2:
for(iz=0; iz<z_length; ++iz) //full z range
for(ix=2; ix<4; ++ix)
for(iy=2; iy<y_length+2; ++iy)
update(ix,iy,iz,u[iu],u[1-iu]);//update left column except corners
break;
case 3:
for(iz=0;iz<z_length;iz++) //full z range
for(ix=x_length;ix<x_length+2;ix++)
for(iy=2;iy<y_length+2;iy++)
update(ix,iy,iz,u[iu],u[1-iu]);//update right column except corners
}
++done_count;
}//end if(done[i])
}//end if(!done[i]).
}//end while(done_count<4)
//printf("Rank %d has finished busywait phase\n", rank);
duration_busywait += MPI_Wtime() - time_start_busywait;
time_start_external = MPI_Wtime();
for(iz=0; iz<z_length; ++iz) //full z range
for(iy=2*y_length-2; iy<2*y_length+2; ++iy)
for(ix=2*x_length-2; ix<2*x_length+2; ++ix)
update(ix%x_length+2,iy%y_length+2,iz,u[iu],u[1-iu]);//update the four corners
duration_external += MPI_Wtime() - time_start_external;
time_start_mass_reduce = MPI_Wtime();
if(it % reduce_frequency == 0){
MPI_Reduce(&rank_mass, &mass_reduced, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
if(rank == 0)
printf("Step %d: Rank %d reduced total mass of %f\n", it, rank, mass_reduced);
}
duration_mass_reduce += MPI_Wtime() - time_start_mass_reduce;
time_start_waitsend = MPI_Wtime();
for(i=0; i<4; ++i)
MPI_Wait(req_send+i, MPI_STATUS_IGNORE);//Wait for the sends
//MPI_Barrier(MPI_COMM_WORLD);
//printf("rank %d finished MPI_Waits at step %d\n", rank, it);
//Revert arrays
iu = 1-iu;
duration_waitsend += MPI_Wtime() - time_start_waitsend;
//sprintf(filename,"atm%ds%d.txt", rank, it);
//prtdat(filename);
}//end STEPS iteration
MPI_Reduce(&duration_sendrecv ,&total_sendrecv ,1, MPI_DOUBLE,MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&duration_busywait ,&total_busywait ,1, MPI_DOUBLE,MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&duration_internal ,&total_internal ,1, MPI_DOUBLE,MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&duration_external ,&total_external ,1, MPI_DOUBLE,MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&duration_mass_reduce,&total_mass_reduce,1, MPI_DOUBLE,MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&duration_waitsend ,&total_waitsend ,1, MPI_DOUBLE,MPI_SUM, 0, MPI_COMM_WORLD);
total_accounted_for = total_sendrecv + total_internal + total_external + total_busywait + total_mass_reduce + total_waitsend;
if(!rank) printf("Time elapsed: %f seconds\n", total_accounted_for);
if(!rank) printf("Durations:\nSend/Recv = %f\nInternal = %f\nBusywait = %f\nExternal = %f\nReduce mass = %f\nWait sends = %f\n\n"
"Respective percentages (based on accounted for):\nSend/Recv = %f\nInternal = %f\nBusywait = %f\nExternal = %f\nReduce mass = %f\nWait sends = %f\n\n",
total_sendrecv,total_internal,total_busywait,total_external,total_mass_reduce,total_waitsend,
100.0 * total_sendrecv /total_accounted_for,
100.0 * total_internal /total_accounted_for,
100.0 * total_busywait /total_accounted_for,
100.0 * total_external /total_accounted_for,
100.0 * total_mass_reduce/total_accounted_for,
100.0 * total_waitsend /total_accounted_for);
if(rank==0) {
fprintf(fp,"Total/Sendrecv/internal/busywait/external/mass reduce/waitsend durations:\n%f\t%f\t%f\t%f\t%f\t%f\t%f\n\n\n",
total_accounted_for, total_sendrecv, total_internal, total_busywait, total_external, total_mass_reduce, total_waitsend);
fclose(fp);
}
if(!rank) printf("\n\n==================================\n===========SESSION END============\n==================================\n\n\n");
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
int errs = 0;
int found, completed;
int rank, size;
int *sendbuf = NULL, *recvbuf = NULL;
int count, i;
MPI_Message msg;
MPI_Request rreq;
MPI_Status s1, s2;
MPI_Datatype vectype;
MTest_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
if (size < 2) {
printf("this test requires at least 2 processes\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* all processes besides ranks 0 & 1 aren't used by this test */
if (rank >= 2) {
goto epilogue;
}
sendbuf = (int *) malloc(LARGE_SZ * sizeof(int));
recvbuf = (int *) malloc(LARGE_SZ * sizeof(int));
if (sendbuf == NULL || recvbuf == NULL) {
printf("Error in memory allocation\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* test 0: simple send & mprobe+mrecv */
if (rank == 0) {
sendbuf[0] = 0xdeadbeef;
sendbuf[1] = 0xfeedface;
MPI_Send(sendbuf, 2, MPI_INT, 1, 5, MPI_COMM_WORLD);
} else {
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(0, 5, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 5);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg != MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 2);
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Mrecv(recvbuf, count, MPI_INT, &msg, &s2);
check(recvbuf[0] == 0xdeadbeef);
check(recvbuf[1] == 0xfeedface);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 5);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
}
/* test 1: simple send & mprobe+imrecv */
if (rank == 0) {
sendbuf[0] = 0xdeadbeef;
sendbuf[1] = 0xfeedface;
MPI_Send(sendbuf, 2, MPI_INT, 1, 5, MPI_COMM_WORLD);
} else {
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(0, 5, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 5);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg != MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 2);
rreq = MPI_REQUEST_NULL;
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Imrecv(recvbuf, count, MPI_INT, &msg, &rreq);
check(rreq != MPI_REQUEST_NULL);
MPI_Wait(&rreq, &s2);
check(recvbuf[0] == 0xdeadbeef);
check(recvbuf[1] == 0xfeedface);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 5);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
}
/* test 2: simple send & improbe+mrecv */
if (rank == 0) {
sendbuf[0] = 0xdeadbeef;
sendbuf[1] = 0xfeedface;
MPI_Send(sendbuf, 2, MPI_INT, 1, 5, MPI_COMM_WORLD);
} else {
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
do {
check(msg == MPI_MESSAGE_NULL);
MPI_Improbe(0, 5, MPI_COMM_WORLD, &found, &msg, &s1);
} while (!found);
check(msg != MPI_MESSAGE_NULL);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 5);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 2);
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Mrecv(recvbuf, count, MPI_INT, &msg, &s2);
check(recvbuf[0] == 0xdeadbeef);
check(recvbuf[1] == 0xfeedface);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 5);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
}
/* test 3: simple send & improbe+imrecv */
if (rank == 0) {
sendbuf[0] = 0xdeadbeef;
sendbuf[1] = 0xfeedface;
MPI_Send(sendbuf, 2, MPI_INT, 1, 5, MPI_COMM_WORLD);
} else {
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
do {
check(msg == MPI_MESSAGE_NULL);
MPI_Improbe(0, 5, MPI_COMM_WORLD, &found, &msg, &s1);
} while (!found);
check(msg != MPI_MESSAGE_NULL);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 5);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 2);
rreq = MPI_REQUEST_NULL;
MPI_Imrecv(recvbuf, count, MPI_INT, &msg, &rreq);
check(rreq != MPI_REQUEST_NULL);
MPI_Wait(&rreq, &s2);
check(recvbuf[0] == 0xdeadbeef);
check(recvbuf[1] == 0xfeedface);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 5);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
}
/* test 4: mprobe+mrecv with MPI_PROC_NULL */
{
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(MPI_PROC_NULL, 5, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == MPI_PROC_NULL);
check(s1.MPI_TAG == MPI_ANY_TAG);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg == MPI_MESSAGE_NO_PROC);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 0);
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Mrecv(recvbuf, count, MPI_INT, &msg, &s2);
/* recvbuf should remain unmodified */
check(recvbuf[0] == 0x01234567);
check(recvbuf[1] == 0x89abcdef);
/* should get back "proc null status" */
check(s2.MPI_SOURCE == MPI_PROC_NULL);
check(s2.MPI_TAG == MPI_ANY_TAG);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s2, MPI_INT, &count);
check(count == 0);
}
/* test 5: mprobe+imrecv with MPI_PROC_NULL */
{
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(MPI_PROC_NULL, 5, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == MPI_PROC_NULL);
check(s1.MPI_TAG == MPI_ANY_TAG);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg == MPI_MESSAGE_NO_PROC);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 0);
rreq = MPI_REQUEST_NULL;
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Imrecv(recvbuf, count, MPI_INT, &msg, &rreq);
check(rreq != MPI_REQUEST_NULL);
completed = 0;
MPI_Test(&rreq, &completed, &s2); /* single test should always succeed */
check(completed);
/* recvbuf should remain unmodified */
check(recvbuf[0] == 0x01234567);
check(recvbuf[1] == 0x89abcdef);
/* should get back "proc null status" */
check(s2.MPI_SOURCE == MPI_PROC_NULL);
check(s2.MPI_TAG == MPI_ANY_TAG);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s2, MPI_INT, &count);
check(count == 0);
}
/* test 6: improbe+mrecv with MPI_PROC_NULL */
{
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
found = 0;
MPI_Improbe(MPI_PROC_NULL, 5, MPI_COMM_WORLD, &found, &msg, &s1);
check(found);
check(msg == MPI_MESSAGE_NO_PROC);
check(s1.MPI_SOURCE == MPI_PROC_NULL);
check(s1.MPI_TAG == MPI_ANY_TAG);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 0);
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Mrecv(recvbuf, count, MPI_INT, &msg, &s2);
/* recvbuf should remain unmodified */
check(recvbuf[0] == 0x01234567);
check(recvbuf[1] == 0x89abcdef);
/* should get back "proc null status" */
check(s2.MPI_SOURCE == MPI_PROC_NULL);
check(s2.MPI_TAG == MPI_ANY_TAG);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s2, MPI_INT, &count);
check(count == 0);
}
/* test 7: improbe+imrecv */
{
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Improbe(MPI_PROC_NULL, 5, MPI_COMM_WORLD, &found, &msg, &s1);
check(found);
check(msg == MPI_MESSAGE_NO_PROC);
check(s1.MPI_SOURCE == MPI_PROC_NULL);
check(s1.MPI_TAG == MPI_ANY_TAG);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 0);
rreq = MPI_REQUEST_NULL;
MPI_Imrecv(recvbuf, count, MPI_INT, &msg, &rreq);
check(rreq != MPI_REQUEST_NULL);
completed = 0;
MPI_Test(&rreq, &completed, &s2); /* single test should always succeed */
check(completed);
/* recvbuf should remain unmodified */
check(recvbuf[0] == 0x01234567);
check(recvbuf[1] == 0x89abcdef);
/* should get back "proc null status" */
check(s2.MPI_SOURCE == MPI_PROC_NULL);
check(s2.MPI_TAG == MPI_ANY_TAG);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s2, MPI_INT, &count);
check(count == 0);
}
/* test 8: simple ssend & mprobe+mrecv */
if (rank == 0) {
sendbuf[0] = 0xdeadbeef;
sendbuf[1] = 0xfeedface;
MPI_Ssend(sendbuf, 2, MPI_INT, 1, 5, MPI_COMM_WORLD);
} else {
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(0, 5, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 5);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg != MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 2);
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Mrecv(recvbuf, count, MPI_INT, &msg, &s2);
check(recvbuf[0] == 0xdeadbeef);
check(recvbuf[1] == 0xfeedface);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 5);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
}
/* test 9: mprobe+mrecv LARGE */
if (rank == 0) {
for (i = 0; i < LARGE_SZ; i++)
sendbuf[i] = i;
MPI_Send(sendbuf, LARGE_SZ, MPI_INT, 1, 5, MPI_COMM_WORLD);
} else {
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(0, 5, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 5);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg != MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == LARGE_SZ);
memset(recvbuf, 0xFF, LARGE_SZ * sizeof(int));
MPI_Mrecv(recvbuf, count, MPI_INT, &msg, &s2);
for (i = 0; i < LARGE_SZ; i++)
check(recvbuf[i] == i);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 5);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
}
/* test 10: mprobe+mrecv noncontiguous datatype */
MPI_Type_vector(2, 1, 4, MPI_INT, &vectype);
MPI_Type_commit(&vectype);
if (rank == 0) {
memset(sendbuf, 0, 8 * sizeof(int));
sendbuf[0] = 0xdeadbeef;
sendbuf[4] = 0xfeedface;
MPI_Send(sendbuf, 1, vectype, 1, 5, MPI_COMM_WORLD);
} else {
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(0, 5, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 5);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg != MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s1, vectype, &count);
check(count == 1);
memset(recvbuf, 0, 8 * sizeof(int));
MPI_Mrecv(recvbuf, 1, vectype, &msg, &s2);
check(recvbuf[0] == 0xdeadbeef);
for (i = 1; i < 4; i++)
check(recvbuf[i] == 0);
check(recvbuf[4] = 0xfeedface);
for (i = 5; i < 8; i++)
check(recvbuf[i] == 0);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 5);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
}
MPI_Type_free(&vectype);
/* test 11: mprobe+mrecv noncontiguous datatype LARGE */
MPI_Type_vector(LARGE_DIM, LARGE_DIM - 1, LARGE_DIM, MPI_INT, &vectype);
MPI_Type_commit(&vectype);
if (rank == 0) {
for (i = 0; i < LARGE_SZ; i++)
sendbuf[i] = i;
MPI_Send(sendbuf, 1, vectype, 1, 5, MPI_COMM_WORLD);
} else {
int idx = 0;
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(0, 5, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 5);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg != MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s1, vectype, &count);
check(count == 1);
memset(recvbuf, 0, LARGE_SZ * sizeof(int));
MPI_Mrecv(recvbuf, 1, vectype, &msg, &s2);
for (i = 0; i < LARGE_DIM; i++) {
int j;
for (j = 0; j < LARGE_DIM - 1; j++) {
check(recvbuf[idx] == idx);
++idx;
}
check(recvbuf[idx++] == 0);
}
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 5);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
}
MPI_Type_free(&vectype);
/* test 12: order test */
if (rank == 0) {
MPI_Request lrequest[2];
sendbuf[0] = 0xdeadbeef;
sendbuf[1] = 0xfeedface;
sendbuf[2] = 0xdeadbeef;
sendbuf[3] = 0xfeedface;
sendbuf[4] = 0xdeadbeef;
sendbuf[5] = 0xfeedface;
MPI_Isend(&sendbuf[0], 4, MPI_INT, 1, 6, MPI_COMM_WORLD, &lrequest[0]);
MPI_Isend(&sendbuf[4], 2, MPI_INT, 1, 6, MPI_COMM_WORLD, &lrequest[1]);
MPI_Waitall(2, &lrequest[0], MPI_STATUSES_IGNORE);
} else {
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(0, 6, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 6);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg != MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 4);
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Recv(recvbuf, 2, MPI_INT, 0, 6, MPI_COMM_WORLD, &s2);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 6);
check(recvbuf[0] == 0xdeadbeef);
check(recvbuf[1] == 0xfeedface);
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
recvbuf[2] = 0x01234567;
recvbuf[3] = 0x89abcdef;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
MPI_Mrecv(recvbuf, count, MPI_INT, &msg, &s2);
check(recvbuf[0] == 0xdeadbeef);
check(recvbuf[1] == 0xfeedface);
check(recvbuf[2] == 0xdeadbeef);
check(recvbuf[3] == 0xfeedface);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 6);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
}
/* test 13: simple send & mprobe+mrecv with zero count */
if (rank == 0) {
MPI_Send(sendbuf, 0, MPI_INT, 1, 13, MPI_COMM_WORLD);
} else {
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(0, 13, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 13);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg != MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s1, MPI_INT, &count);
check(count == 0);
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Mrecv(recvbuf, 0, MPI_INT, &msg, &s2);
/* recvbuf should remain unmodified */
check(recvbuf[0] == 0x01234567);
check(recvbuf[1] == 0x89abcdef);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 13);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s2, MPI_INT, &count);
check(count == 0);
}
/* test 14: simple send & mprobe+mrecv with zero-size datatype */
if (rank == 0) {
MPI_Send(sendbuf, 0, MPI_BYTE, 1, 14, MPI_COMM_WORLD);
} else {
MPI_Datatype zero_dtype;
MPI_Type_contiguous(0, MPI_INT, &zero_dtype);
MPI_Type_commit(&zero_dtype);
memset(&s1, 0xab, sizeof(MPI_Status));
memset(&s2, 0xab, sizeof(MPI_Status));
/* the error field should remain unmodified */
s1.MPI_ERROR = MPI_ERR_DIMS;
s2.MPI_ERROR = MPI_ERR_TOPOLOGY;
msg = MPI_MESSAGE_NULL;
MPI_Mprobe(0, 14, MPI_COMM_WORLD, &msg, &s1);
check(s1.MPI_SOURCE == 0);
check(s1.MPI_TAG == 14);
check(s1.MPI_ERROR == MPI_ERR_DIMS);
check(msg != MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s1, zero_dtype, &count);
check(count == 0);
recvbuf[0] = 0x01234567;
recvbuf[1] = 0x89abcdef;
MPI_Mrecv(recvbuf, 1, zero_dtype, &msg, &s2);
/* recvbuf should remain unmodified */
check(recvbuf[0] == 0x01234567);
check(recvbuf[1] == 0x89abcdef);
check(s2.MPI_SOURCE == 0);
check(s2.MPI_TAG == 14);
check(s2.MPI_ERROR == MPI_ERR_TOPOLOGY);
check(msg == MPI_MESSAGE_NULL);
count = -1;
MPI_Get_count(&s2, zero_dtype, &count);
check(count == 0);
MPI_Type_free(&zero_dtype);
}
free(sendbuf);
free(recvbuf);
/* TODO MPI_ANY_SOURCE and MPI_ANY_TAG should be tested as well */
/* TODO a full range of message sizes should be tested too */
/* TODO threaded tests are also needed, but they should go in a separate
* program */
/* simple test to ensure that c2f/f2c routines are present (initially missed
* in MPICH impl) */
{
MPI_Fint f_handle = 0xdeadbeef;
f_handle = MPI_Message_c2f(MPI_MESSAGE_NULL);
msg = MPI_Message_f2c(f_handle);
check(f_handle != 0xdeadbeef);
check(msg == MPI_MESSAGE_NULL);
/* PMPI_ versions should also exists */
f_handle = 0xdeadbeef;
f_handle = PMPI_Message_c2f(MPI_MESSAGE_NULL);
msg = PMPI_Message_f2c(f_handle);
check(f_handle != 0xdeadbeef);
check(msg == MPI_MESSAGE_NULL);
}
epilogue:
MTest_Finalize(errs);
return MTestReturnValue(errs);
}
int master_main(const char *host, int port, const char *addr) {
time_t idle_stoptime;
struct link *master = NULL;
int num_workers, i;
struct mpi_queue_job **workers;
struct itable *active_jobs = itable_create(0);
struct itable *waiting_jobs = itable_create(0);
struct list *complete_jobs = list_create();
MPI_Comm_size(MPI_COMM_WORLD, &num_workers);
workers = malloc(num_workers * sizeof(*workers));
memset(workers, 0, num_workers * sizeof(*workers));
idle_stoptime = time(0) + idle_timeout;
while(!abort_flag) {
char line[MPI_QUEUE_LINE_MAX];
if(time(0) > idle_stoptime) {
if(master) {
printf("mpi master: gave up after waiting %ds to receive a task.\n", idle_timeout);
} else {
printf("mpi master: gave up after waiting %ds to connect to %s port %d.\n", idle_timeout, host, port);
}
break;
}
if(!master) {
char working_dir[MPI_QUEUE_LINE_MAX];
master = link_connect(addr, port, idle_stoptime);
if(!master) {
sleep(5);
continue;
}
link_tune(master, LINK_TUNE_INTERACTIVE);
link_readline(master, line, sizeof(line), time(0) + active_timeout);
memset(working_dir, 0, MPI_QUEUE_LINE_MAX);
if(sscanf(line, "workdir %s", working_dir) == 1) {
MPI_Bcast(working_dir, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
master = NULL;
continue;
}
}
if(link_readline(master, line, sizeof(line), time(0) + short_timeout)) {
struct mpi_queue_operation *op;
int jobid, mode;
INT64_T length;
char path[MPI_QUEUE_LINE_MAX];
op = NULL;
debug(D_MPI, "received: %s\n", line);
if(!strcmp(line, "get results")) {
struct mpi_queue_job *job;
debug(D_MPI, "results requested: %d available\n", list_size(complete_jobs));
link_putfstring(master, "num results %d\n", time(0) + active_timeout, list_size(complete_jobs));
while(list_size(complete_jobs)) {
job = list_pop_head(complete_jobs);
link_putfstring(master, "result %d %d %d %lld\n", time(0) + active_timeout, job->jobid, job->status, job->result, job->output_length);
if(job->output_length) {
link_write(master, job->output, job->output_length, time(0)+active_timeout);
}
mpi_queue_job_delete(job);
}
} else if(sscanf(line, "work %d %lld", &jobid, &length)) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_WORK;
op->buffer_length = length+1;
op->buffer = malloc(length+1);
op->buffer[op->buffer_length] = 0;
link_read(master, op->buffer, length, time(0) + active_timeout);
op->result = -1;
} else if(sscanf(line, "stat %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_STAT;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "unlink %d %s", &jobid, path) == 2) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_UNLINK;
sprintf(op->args, "%s", path);
op->result = -1;
} else if(sscanf(line, "mkdir %d %s %o", &jobid, path, &mode) == 3) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_MKDIR;
sprintf(op->args, "%s %o", path, mode);
op->result = -1;
} else if(sscanf(line, "close %d", &jobid) == 1) {
op = malloc(sizeof(*op));
memset(op, 0, sizeof(*op));
op->type = MPI_QUEUE_OP_CLOSE;
op->result = -1;
// } else if(sscanf(line, "symlink %d %s %s", &jobid, path, filename) == 3) {
// } else if(sscanf(line, "put %d %s %lld %o", &jobid, filename, &length, &mode) == 4) {
// } else if(sscanf(line, "rget %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "get %d %s", &jobid, filename) == 2) {
// } else if(sscanf(line, "thirdget %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
// } else if(sscanf(line, "thirdput %d %d %s %[^\n]", &jobid, &mode, filename, path) == 4) {
} else if(!strcmp(line, "exit")) {
break;
} else {
abort_flag = 1;
continue;
}
if(op) {
struct mpi_queue_job *job;
job = itable_lookup(active_jobs, jobid);
if(!job) {
job = itable_lookup(waiting_jobs, jobid);
}
if(!job) {
job = malloc(sizeof(*job));
memset(job, 0, sizeof(*job));
job->jobid = jobid;
job->operations = list_create();
job->status = MPI_QUEUE_JOB_WAITING;
job->worker_rank = -1;
itable_insert(waiting_jobs, jobid, job);
}
list_push_tail(job->operations, op);
}
idle_stoptime = time(0) + idle_timeout;
} else {
link_close(master);
master = 0;
sleep(5);
}
int num_waiting_jobs = itable_size(waiting_jobs);
int num_unvisited_jobs = itable_size(active_jobs);
for(i = 1; i < num_workers && (num_unvisited_jobs > 0 || num_waiting_jobs > 0); i++) {
struct mpi_queue_job *job;
struct mpi_queue_operation *op;
int flag = 0;
UINT64_T jobid;
if(!workers[i]) {
if(num_waiting_jobs) {
itable_firstkey(waiting_jobs);
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
itable_insert(active_jobs, jobid, job);
workers[i] = job;
num_waiting_jobs--;
job->worker_rank = i;
job->status = MPI_QUEUE_JOB_READY;
} else {
continue;
}
} else {
num_unvisited_jobs--;
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Test(&workers[i]->request, &flag, &workers[i]->mpi_status);
if(flag) {
op = list_pop_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
workers[i]->status = MPI_QUEUE_JOB_READY;
if(op->type == MPI_QUEUE_OP_WORK || op->result < 0) {
if(workers[i]->output)
free(workers[i]->output);
workers[i]->output = op->output_buffer;
op->output_buffer = NULL;
workers[i]->output_length = op->output_length;
workers[i]->result = op->result;
if(op->result < 0) {
workers[i]->status = MPI_QUEUE_JOB_FAILED | op->type;
op->type = MPI_QUEUE_OP_CLOSE;
list_push_head(workers[i]->operations, op);
op = NULL;
}
}
if(op) {
if(op->buffer)
free(op->buffer);
if(op->output_buffer)
free(op->output_buffer);
free(op);
}
}
}
}
if( workers[i]->status != MPI_QUEUE_JOB_BUSY && list_size(workers[i]->operations)) {
op = list_peek_head(workers[i]->operations);
if(op->type == MPI_QUEUE_OP_CLOSE) {
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
if(!(workers[i]->status & MPI_QUEUE_JOB_FAILED))
workers[i]->status = MPI_QUEUE_JOB_COMPLETE;
workers[i] = NULL;
i--;
continue;
}
MPI_Send(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
if(op->buffer_length) {
MPI_Send(op->buffer, op->buffer_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD);
free(op->buffer);
op->buffer_length = 0;
op->buffer = NULL;
}
MPI_Irecv(op, sizeof(*op), MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->request);
workers[i]->status = MPI_QUEUE_JOB_BUSY;
}
}
}
/** Clean up waiting & complete jobs, send Exit commands to each worker */
if(!master) {
// If the master link hasn't been set up yet
// the workers will be waiting for the working directory
char line[MPI_QUEUE_LINE_MAX];
memset(line, 0, MPI_QUEUE_LINE_MAX);
MPI_Bcast(line, MPI_QUEUE_LINE_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
} else {
link_close(master);
}
for(i = 1; i < num_workers; i++) {
struct mpi_queue_operation *op, close;
memset(&close, 0, sizeof(close));
close.type = MPI_QUEUE_OP_EXIT;
if(workers[i]) {
if(workers[i]->status == MPI_QUEUE_JOB_BUSY) {
MPI_Wait(&workers[i]->request, &workers[i]->mpi_status);
op = list_peek_head(workers[i]->operations);
if(op->output_length) {
op->output_buffer = malloc(op->output_length);
MPI_Recv(op->output_buffer, op->output_length, MPI_BYTE, workers[i]->worker_rank, 0, MPI_COMM_WORLD, &workers[i]->mpi_status);
}
}
itable_remove(active_jobs, workers[i]->jobid);
list_push_tail(complete_jobs, workers[i]);
}
MPI_Send(&close, sizeof(close), MPI_BYTE, i, 0, MPI_COMM_WORLD);
}
itable_firstkey(waiting_jobs);
while(itable_size(waiting_jobs)) {
struct mpi_queue_job *job;
UINT64_T jobid;
itable_nextkey(waiting_jobs, &jobid, (void **)&job);
itable_remove(waiting_jobs, jobid);
list_push_tail(complete_jobs, job);
}
while(list_size(complete_jobs)) {
mpi_queue_job_delete(list_pop_head(complete_jobs));
}
MPI_Finalize();
return abort_flag;
}
void do_master_stuff(int argc, char ** argv, struct mw_api_spec *f)
{
DEBUG_PRINT(("master starting"));
int number_of_slaves;
MPI_Comm_size(MPI_COMM_WORLD, &number_of_slaves);
LinkedList * work_list;
double start, end, start_create, end_create, start_results, end_results;
start = MPI_Wtime();
DEBUG_PRINT(("creating work list..."));
start_create = MPI_Wtime();
work_list = listFromArray(f->create(argc, argv));
end_create = MPI_Wtime();
DEBUG_PRINT(("created work in %f seconds!", end_create - start_create));
int slave=1, num_work_units=0;
num_work_units = list_length(work_list);
mw_result_t * received_results = malloc(f->res_sz * num_work_units);
if (received_results == NULL)
{
fprintf(stderr, "ERROR: insufficient memory to allocate received_results\n");
exit(0);
}
int num_results_received = 0;
// make array keeping track of pointers for work that's active
LinkedList* assignment_ptrs[number_of_slaves-2];
// make array of binary indicators for inactive workers
// initially all workers are active and 0
//unsigned int inactive_workers[number_of_slaves-2];
// create array of start times
double assignment_time[number_of_slaves-2];
int are_you_down[number_of_slaves-2];
// current pointer
LinkedList
* next_work_node = work_list,
* list_end = NULL;
// have supervisor so starting at 2
for(slave=2; slave<number_of_slaves; ++slave)
{
are_you_down[slave-2] = 0; //slaves are all working in the beginning
DEBUG_PRINT(("assigning work to slave"));
if(next_work_node == NULL)
{
DEBUG_PRINT(("reached the end of the work, breaking!"));
break;
}
mw_work_t * work_unit = next_work_node->data;
send_to_slave(work_unit, f->work_sz, MPI_CHAR, slave, WORK_TAG, MPI_COMM_WORLD);
// save next_work_node to assigned work
assignment_ptrs[slave-2] = next_work_node;
assert(assignment_ptrs[slave-2] != NULL);
// save start time
assignment_time[slave-2] = MPI_Wtime();
// update next_work_node
if(next_work_node->next == NULL)
{
list_end = next_work_node;
}
next_work_node=next_work_node->next;
DEBUG_PRINT(("work sent to slave"));
}
// send time array to supervisor
DEBUG_PRINT(("Sending supervisor first time update"));
MPI_Send(assignment_time, number_of_slaves-2, MPI_DOUBLE, 1, SUPERVISOR_TAG, MPI_COMM_WORLD);
// failure id
int failure_id;
MPI_Status status_fail, status_res;
MPI_Request request_fail, request_res;
int flag_fail = 0, flag_res = 0;
// receive failure from supervisor as non-blocking recv
MPI_Irecv(&failure_id, 1, MPI_INT, 1, FAIL_TAG, MPI_COMM_WORLD, &request_fail);
// receive result from workers as non-blocking recv
MPI_Irecv(&received_results[num_results_received], f->res_sz, MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &request_res);
// send units of work while haven't received all results
while(num_results_received < num_work_units)
{
// check for flag_fail again
MPI_Test(&request_fail, &flag_fail, &status_fail);
// check for flag_res again
MPI_Test(&request_res, &flag_res, &status_res);
// send work if have failures or got results
if (flag_fail)
{
// change inactive workers array
//inactive_workers[status_fail.MPI_SOURCE-2] = 1;
DEBUG_PRINT(("received failure from supervisor, process %d", failure_id));
// get work_unit that needs to be reassigned
LinkedList * work_unit = assignment_ptrs[failure_id];
if(work_unit != NULL)
{
DEBUG_PRINT(("Moving assignment at %p to end of the queue", work_unit));
move_node_to_end(work_unit);
if(next_work_node == NULL)
{
next_work_node = work_unit;
}
assert(next_work_node != NULL);
}
if(assignment_time[failure_id] == 0.0)
{
DEBUG_PRINT(("Failure on idle process %d. WTF??", failure_id));
}
if(are_you_down[failure_id] == 1)
{
DEBUG_PRINT(("Failure on a process which is already failed. WTF??"));
}
are_you_down[failure_id] = 1; //this slave is considered dead :(
assignment_ptrs[failure_id] = NULL;
assignment_time[failure_id] = 0.0;
MPI_Send(assignment_time, number_of_slaves-2, MPI_DOUBLE, 1, SUPERVISOR_TAG, MPI_COMM_WORLD);
flag_fail = 0;
// continue to receive failures from supervisor as non-blocking recv
MPI_Irecv(&failure_id, 1, MPI_INT, 1, FAIL_TAG, MPI_COMM_WORLD, &request_fail);
}
int idle_process = -1, i;
for(i=0; i<number_of_slaves-2; ++i)
{
if(assignment_time[i] == 0.0 && !are_you_down[i])
{
idle_process = i;
break;
}
}
if(next_work_node != NULL && idle_process > -1)
{
send_to_slave(next_work_node->data, f->work_sz, MPI_CHAR, idle_process+2, WORK_TAG, MPI_COMM_WORLD);
assignment_ptrs[idle_process] = next_work_node;
assignment_time[idle_process] = MPI_Wtime();
MPI_Send(assignment_time, number_of_slaves-2, MPI_DOUBLE, 1, SUPERVISOR_TAG, MPI_COMM_WORLD);
DEBUG_PRINT(("Gave an assignment to previously idle process %d, assignment at %p", idle_process, next_work_node));
if(next_work_node->next == NULL)
{
list_end = next_work_node;
}
next_work_node = next_work_node->next;
}
if (flag_res)
{
int worker_number = status_res.MPI_SOURCE-2;
if(!are_you_down[worker_number]) //If this slave is marked dead, just ignore him
{
// update number of results received
num_results_received++;
if(next_work_node == NULL && list_end != NULL && list_end->next != NULL)
{
DEBUG_PRINT(("Found more work to do, now an idle process can get an assignment"));
next_work_node = list_end->next;
list_end = NULL;
}
if(next_work_node != NULL)
{
// get work_unit
mw_work_t* work_unit = next_work_node->data;
// send new unit of work
send_to_slave(work_unit, f->work_sz, MPI_CHAR, status_res.MPI_SOURCE, WORK_TAG, MPI_COMM_WORLD);
// update pointer
if(next_work_node->next == NULL)
{
list_end = next_work_node;
}
// update work index for new_pid
assignment_ptrs[status_res.MPI_SOURCE-2] = next_work_node;
assert(assignment_ptrs[status_res.MPI_SOURCE-2] != NULL);
assignment_time[status_res.MPI_SOURCE-2] = MPI_Wtime();
// send updated array of times to supervisor
MPI_Send(assignment_time, number_of_slaves-2, MPI_DOUBLE, 1, SUPERVISOR_TAG, MPI_COMM_WORLD);
DEBUG_PRINT(("SENT TIME TO SUP"));
next_work_node = next_work_node->next;
if(next_work_node == NULL)
{
DEBUG_PRINT(("Reached the end of the work list, should get idle processors after this"));
}
}
else
{
DEBUG_PRINT(("Worker %d is now idle, I ain't got shit for him to do", worker_number));
assignment_time[worker_number] = 0.0;
assignment_ptrs[worker_number] = NULL;
assert(!are_you_down[worker_number]);
MPI_Send(assignment_time, number_of_slaves-2, MPI_DOUBLE, 1, SUPERVISOR_TAG, MPI_COMM_WORLD);
}
}
// continue to receive results from workers as non-blocking recv
MPI_Irecv(&received_results[num_results_received], f->res_sz, MPI_CHAR, MPI_ANY_SOURCE, WORK_TAG, MPI_COMM_WORLD, &request_res);
}
}
// send kill signal to other processes, including supervisor
for(slave=1; slave<number_of_slaves; ++slave)
{
DEBUG_PRINT(("Murdering slave"));
kill_slave(slave);
}
start_results = MPI_Wtime();
int err_code = f->result(num_results_received, received_results);
end_results = MPI_Wtime();
end = MPI_Wtime();
DEBUG_PRINT(("all %f s\n", end-start));
DEBUG_PRINT(("create %f s\n", end_create-start_create));
DEBUG_PRINT(("process %f s\n", end_results-start_results));
}
int main(int argc, char *argv[])
{
int rank, numprocs, i;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
//Tests:
//A: 0(isend -> wait) with 1(recv)
int A = 1;
//B: 0(send) with 1(irecv -> wait)
int B = 1;
//C: 0(N * isend -> N * wait) with 1(N * recv)
int C = 1;
//D: 0(N * isend -> N * waitany) with 1(N * recv)
int D = 1;
//E: 0(N*send) with 1(N*irecv, N*wait)
int E = 1;
//F: 0(N*send) with 1(N*irecv, N*waitany)
int F = 1;
//G: 0(N* isend -> waitall) with 1(N*recv)
int G = 1;
//H: 0(N*send) with 1(N*irecv, waitall)
int H = 1;
//I: 0(2*N*send, 2*N*Irecv, Waitall) with
// 1(N*irecv, waitall, N*isend, N*waitany) with
// 2(N*irecv, N*waitany, N*isend, waitall)
int I = 1;
//J: 0(N*isend, N*test, N*wait) with (N*irecv, N*test, N*wait)
int J = 1;
int N = 13;
int tag = 12345;
/////////////////////////////////////////
////////////////// RANK 0
///////////////////////////////////
if (rank == 0) {
MPI_Request request;
MPI_Status status;
MPI_Request req[2 * N];
MPI_Status sta[2 * N];
int *r = (int *) malloc(sizeof(int) * DATATOSENT);
if (A) {
TRACE_smpi_set_category("A");
MPI_Isend(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD, &request);
MPI_Wait(&request, &status);
}
MPI_Barrier(MPI_COMM_WORLD);
if (B) {
TRACE_smpi_set_category("B");
MPI_Send(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD);
}
MPI_Barrier(MPI_COMM_WORLD);
if (C) {
TRACE_smpi_set_category("C");
for (i = 0; i < N; i++) {
MPI_Isend(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD, &req[i]);
}
for (i = 0; i < N; i++) {
MPI_Wait(&req[i], &sta[i]);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (D) {
TRACE_smpi_set_category("D");
for (i = 0; i < N; i++) {
MPI_Isend(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD, &req[i]);
}
for (i = 0; i < N; i++) {
int completed;
MPI_Waitany(N, req, &completed, sta);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (E) {
TRACE_smpi_set_category("E");
for (i = 0; i < N; i++) {
MPI_Send(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (F) {
TRACE_smpi_set_category("F");
for (i = 0; i < N; i++) {
MPI_Send(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (G) {
TRACE_smpi_set_category("G");
for (i = 0; i < N; i++) {
MPI_Isend(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD, &req[i]);
}
MPI_Waitall(N, req, sta);
}
MPI_Barrier(MPI_COMM_WORLD);
if (H) {
TRACE_smpi_set_category("H");
for (i = 0; i < N; i++) {
MPI_Send(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (I) {
TRACE_smpi_set_category("I");
for (i = 0; i < 2 * N; i++) {
if (i < N) {
MPI_Send(r, DATATOSENT, MPI_INT, 2, tag, MPI_COMM_WORLD);
} else {
MPI_Send(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD);
}
}
MPI_Barrier(MPI_COMM_WORLD);
for (i = 0; i < 2 * N; i++) {
if (i < N) {
MPI_Irecv(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD,
&req[i]);
} else {
MPI_Irecv(r, DATATOSENT, MPI_INT, 2, tag, MPI_COMM_WORLD,
&req[i]);
}
}
MPI_Waitall(2 * N, req, sta);
}
MPI_Barrier(MPI_COMM_WORLD);
if (J) {
TRACE_smpi_set_category("J");
for (i = 0; i < N; i++) {
MPI_Isend(r, DATATOSENT, MPI_INT, 1, tag, MPI_COMM_WORLD, &req[i]);
}
for (i = 0; i < N; i++) {
int flag;
MPI_Test(&req[i], &flag, &sta[i]);
}
for (i = 0; i < N; i++) {
MPI_Wait(&req[i], &sta[i]);
}
}
free(r);
/////////////////////////////////////////
////////////////// RANK 1
///////////////////////////////////
} else if (rank == 1) {
MPI_Request request;
MPI_Status status;
MPI_Request req[N];
MPI_Status sta[N];
int *r = (int *) malloc(sizeof(int) * DATATOSENT);
if (A) {
TRACE_smpi_set_category("A");
MPI_Recv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &status);
}
MPI_Barrier(MPI_COMM_WORLD);
if (B) {
TRACE_smpi_set_category("B");
MPI_Irecv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &request);
MPI_Wait(&request, &status);
}
MPI_Barrier(MPI_COMM_WORLD);
if (C) {
TRACE_smpi_set_category("C");
for (i = 0; i < N; i++) {
MPI_Recv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &sta[i]);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (D) {
TRACE_smpi_set_category("D");
for (i = 0; i < N; i++) {
MPI_Recv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &sta[i]);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (E) {
TRACE_smpi_set_category("E");
for (i = 0; i < N; i++) {
MPI_Irecv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &req[i]);
}
for (i = 0; i < N; i++) {
MPI_Wait(&req[i], &sta[i]);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (F) {
TRACE_smpi_set_category("F");
for (i = 0; i < N; i++) {
MPI_Irecv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &req[i]);
}
for (i = 0; i < N; i++) {
int completed;
MPI_Waitany(N, req, &completed, sta);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (G) {
TRACE_smpi_set_category("G");
for (i = 0; i < N; i++) {
MPI_Recv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &sta[i]);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (H) {
TRACE_smpi_set_category("H");
for (i = 0; i < N; i++) {
MPI_Irecv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &req[i]);
}
MPI_Waitall(N, req, sta);
}
MPI_Barrier(MPI_COMM_WORLD);
if (I) {
TRACE_smpi_set_category("I");
for (i = 0; i < N; i++) {
MPI_Irecv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &req[i]);
}
MPI_Waitall(N, req, sta);
MPI_Barrier(MPI_COMM_WORLD);
for (i = 0; i < N; i++) {
MPI_Isend(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &req[i]);
}
MPI_Waitall(N, req, sta);
// for (i = 0; i < N; i++){
// MPI_Wait (&req[i], &sta[i]);
// }
}
MPI_Barrier(MPI_COMM_WORLD);
if (J) {
TRACE_smpi_set_category("J");
for (i = 0; i < N; i++) {
MPI_Irecv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &req[i]);
}
for (i = 0; i < N; i++) {
int flag;
MPI_Test(&req[i], &flag, &sta[i]);
}
for (i = 0; i < N; i++) {
MPI_Wait(&req[i], &sta[i]);
}
}
free(r);
/////////////////////////////////////////
////////////////// RANK 2
///////////////////////////////////
} else if (rank == 2) {
// MPI_Request request;
// MPI_Status status;
MPI_Request req[N];
MPI_Status sta[N];
int *r = (int *) malloc(sizeof(int) * DATATOSENT);
if (A) {
}
MPI_Barrier(MPI_COMM_WORLD);
if (B) {
}
MPI_Barrier(MPI_COMM_WORLD);
if (C) {
}
MPI_Barrier(MPI_COMM_WORLD);
if (D) {
}
MPI_Barrier(MPI_COMM_WORLD);
if (E) {
}
MPI_Barrier(MPI_COMM_WORLD);
if (F) {
}
MPI_Barrier(MPI_COMM_WORLD);
if (G) {
}
MPI_Barrier(MPI_COMM_WORLD);
if (H) {
}
MPI_Barrier(MPI_COMM_WORLD);
if (I) {
TRACE_smpi_set_category("I");
for (i = 0; i < N; i++) {
MPI_Irecv(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD, &req[i]);
}
for (i = 0; i < N; i++) {
int completed;
MPI_Waitany(N, req, &completed, sta);
}
MPI_Barrier(MPI_COMM_WORLD);
for (i = 0; i < N; i++) {
MPI_Send(r, DATATOSENT, MPI_INT, 0, tag, MPI_COMM_WORLD);
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (J) {
}
free(r);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return 0;
}
static void complete_something_somehow(unsigned int rndnum, int numreqs, MPI_Request reqs[], int *outcount, int indices[])
{
int i, idx, flag;
#define COMPLETION_CASES (8)
switch (rand_range(rndnum, 0, COMPLETION_CASES)) {
case 0:
MPI_Waitall(numreqs, reqs, MPI_STATUSES_IGNORE);
*outcount = numreqs;
for (i = 0; i < numreqs; ++i) {
indices[i] = i;
}
break;
case 1:
MPI_Testsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);
if (*outcount == MPI_UNDEFINED) {
*outcount = 0;
}
break;
case 2:
MPI_Waitsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);
if (*outcount == MPI_UNDEFINED) {
*outcount = 0;
}
break;
case 3:
MPI_Waitany(numreqs, reqs, &idx, MPI_STATUS_IGNORE);
if (idx == MPI_UNDEFINED) {
*outcount = 0;
}
else {
*outcount = 1;
indices[0] = idx;
}
break;
case 4:
MPI_Testany(numreqs, reqs, &idx, &flag, MPI_STATUS_IGNORE);
if (idx == MPI_UNDEFINED) {
*outcount = 0;
}
else {
*outcount = 1;
indices[0] = idx;
}
break;
case 5:
MPI_Testall(numreqs, reqs, &flag, MPI_STATUSES_IGNORE);
if (flag) {
*outcount = numreqs;
for (i = 0; i < numreqs; ++i) {
indices[i] = i;
}
}
else {
*outcount = 0;
}
break;
case 6:
/* select a new random index and wait on it */
rndnum = gen_prn(rndnum);
idx = rand_range(rndnum, 0, numreqs);
MPI_Wait(&reqs[idx], MPI_STATUS_IGNORE);
*outcount = 1;
indices[0] = idx;
break;
case 7:
/* select a new random index and wait on it */
rndnum = gen_prn(rndnum);
idx = rand_range(rndnum, 0, numreqs);
MPI_Test(&reqs[idx], &flag, MPI_STATUS_IGNORE);
*outcount = (flag ? 1 : 0);
indices[0] = idx;
break;
default:
assert(0);
break;
}
#undef COMPLETION_CASES
}
enum async_status async_mpi(void* session) {
async_mpi_session* ses = (async_mpi_session*) session;
switch(ses->state) {
case ASYNC_MPI_STATE_SEND_OR_RECV: {
PRINT_SES(ses);
if((ses->flags & ASYNC_MPI_FLAG_SHOULD_SEND_COUNT) != 0) {
int r;
if((ses->flags & ASYNC_MPI_FLAG_IS_SENDING) != 0) {
//printf("MPI_Isend(%p[%i], %i, MPI_INT, %i, %i, MPI_COMM_WORLD, %p)\n", &(ses->count), ses->count, 1, ses->peer, 0, ses->request);
// fprintf(stderr, "Isend(1) to %d\n", ses->peer);
//r = MPI_Isend(&(ses->count), 2, MPI_INT, ses->peer, 0, ses->comm, ses->request);
ses->tmp[0] = ses->count;
ses->tmp[1] = ses->tag;
r = MPI_Isend(ses->tmp, 2, MPI_INT, ses->peer, 0, ses->comm, ses->request);
} else {
//printf("MPI_Irecv(%p[%i], %i, MPI_INT, %i, %i, MPI_COMM_WORLD, %p)\n", &(ses->count), ses->count, 1, ses->peer, 0, ses->request);
//r = MPI_Irecv(&(ses->count), 2, MPI_INT, ses->peer, 0, ses->comm, ses->request);
r = MPI_Irecv(ses->tmp, 2, MPI_INT, ses->peer, 0, ses->comm, ses->request);
}
if(r != MPI_SUCCESS) {
ses->state = ASYNC_MPI_STATE_FAILURE;
return ASYNC_FAILURE;
}
} else {
ses->state = ASYNC_MPI_STATE_SEND_OR_RECV2;
return async_mpi(ses);
}
ses->state = ASYNC_MPI_STATE_TEST;
// fall-through
}
case ASYNC_MPI_STATE_TEST: {
PRINT_SES(ses);
int flag;
MPI_Status status;
MPI_Test(ses->request, &flag, &status);
if(!flag) {
return ASYNC_PENDING;
}
if((ses->flags & ASYNC_MPI_FLAG_IS_SENDING) == 0) {
ses->count = ses->tmp[0];
ses->tag = ses->tmp[1];
//printf("count=%i source=%i tag=%i error=%i\n", ses->count, status.MPI_SOURCE, status.MPI_TAG, status.MPI_ERROR);
ses->peer = status.MPI_SOURCE;
//ses->tag = status.MPI_TAG;
if(safe_realloc(&(ses->buf), ses->buf, ses->count) <= 0) {
ses->state = ASYNC_MPI_STATE_FAILURE;
return ASYNC_FAILURE;
}
// fprintf(stderr, "%s:%u: recv message of size %u\n", __FILE__, __LINE__, ses->count);
}
ses->state = ASYNC_MPI_STATE_SEND_OR_RECV2;
// fall-through
}
case ASYNC_MPI_STATE_SEND_OR_RECV2: {
PRINT_SES(ses);
int r;
if((ses->flags & ASYNC_MPI_FLAG_IS_SENDING) != 0) {
//fprintf(stderr, "MPI_Isend(%p[%i,%i], %i, MPI_BYTE, %i, %i, MPI_COMM_WORLD, %p)\n", ses->buf, ((int*)ses->buf)[0], ((int*)ses->buf)[1], ses->count, ses->peer, ses->tag, ses->request);
// fprintf(stderr, "Isend(2) to %d\n", ses->peer);
r = MPI_Isend(ses->buf, ses->count, ses->datatype, ses->peer, ses->tag, ses->comm, ses->request);
} else {
//fprintf(stderr, "MPI_Irecv(%p[%i,%i], %i, MPI_BYTE, %i, %i, MPI_COMM_WORLD, %p)\n", ses->buf, ((int*)ses->buf)[0], ((int*)ses->buf)[1], ses->count, ses->peer, ses->tag, ses->request);
r = MPI_Irecv(ses->buf, ses->count, ses->datatype, ses->peer, ses->tag, ses->comm, ses->request);
}
if(r != MPI_SUCCESS) {
//printf("FAILURE! (from async_mpi)\n");
ses->state = ASYNC_MPI_STATE_FAILURE;
return ASYNC_FAILURE;
}
ses->state = ASYNC_MPI_STATE_TEST2;
// fall-through
}
case ASYNC_MPI_STATE_TEST2: {
PRINT_SES(ses);
int flag = 1;
MPI_Status status;
MPI_Test(ses->request, &flag, &status);
//fprintf(stderr, "MPI_Test(%p[%i,%i], %i, MPI_BYTE, %i, %i, MPI_COMM_WORLD, %p)\n", ses->buf, ((int*)ses->buf)[0], ((int*)ses->buf)[1], ses->count, ses->peer, ses->tag, ses->request);
if(!flag) {
return ASYNC_PENDING;
}
ses->peer = status.MPI_SOURCE;
//printf("flag = %i\tSOURCE = %i\tTAG = %i\tERROR = %i\n", flag, status.MPI_SOURCE, status.MPI_TAG, status.MPI_ERROR);
ses->state = ASYNC_MPI_STATE_SUCCESS;
// fall-through
}
case ASYNC_MPI_STATE_SUCCESS: {
PRINT_SES(ses);
return ASYNC_SUCCESS;
}
case ASYNC_MPI_STATE_FAILURE: {
PRINT_SES(ses);
return ASYNC_FAILURE;
}
default: {
printf("UNHANDLED CASE!\n");
return ASYNC_FAILURE;
}
}
}
int main( int argc, char **argv )
{
MPI_Request r1;
int size, rank;
int err = 0;
int partner, buf[10], flag, idx, index;
MPI_Status status;
MPI_Init( &argc, &argv );
MPI_Comm_size( MPI_COMM_WORLD, &size );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
if (size < 2) {
printf( "Cancel test requires at least 2 processes\n" );
MPI_Abort( MPI_COMM_WORLD, 1 );
}
/*
* Here is the test. First, we ensure an unsatisfied Irecv:
* process 0 process size-1
* Sendrecv Sendrecv
* Irecv ----
* Cancel ----
* Sendrecv Sendrecv
* Next, we confirm receipt before canceling
* Irecv Send
* Sendrecv Sendrecv
* Cancel
*/
if (rank == 0) {
partner = size - 1;
/* Cancel succeeds for wait/waitall */
MPI_Send_init( buf, 10, MPI_INT, partner, 0, MPI_COMM_WORLD, &r1 );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
MPI_Start( &r1 );
MPI_Cancel( &r1 );
MPI_Wait( &r1, &status );
MPI_Test_cancelled( &status, &flag );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
if (!flag) {
err++;
printf( "Cancel of a send failed where it should succeed (Wait).\n" );
}
MPI_Request_free( &r1 );
/* Cancel fails for test/testall */
buf[0] = 3;
MPI_Send_init( buf, 3, MPI_INT, partner, 2, MPI_COMM_WORLD, &r1 );
MPI_Start( &r1 );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
MPI_Cancel( &r1 );
MPI_Test( &r1, &flag, &status );
MPI_Test_cancelled( &status, &flag );
if (flag) {
err++;
printf( "Cancel of a send succeeded where it shouldn't (Test).\n" );
}
MPI_Request_free( &r1 );
/* Cancel succeeds for waitany */
MPI_Send_init( buf, 10, MPI_INT, partner, 0, MPI_COMM_WORLD, &r1 );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
MPI_Start( &r1 );
MPI_Cancel( &r1 );
MPI_Waitany( 1, &r1, &idx, &status );
MPI_Test_cancelled( &status, &flag );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
if (!flag) {
err++;
printf( "Cancel of a send failed where it should succeed (Waitany).\n" );
}
MPI_Request_free( &r1 );
/* Cancel fails for testany */
buf[0] = 3;
MPI_Send_init( buf, 3, MPI_INT, partner, 2, MPI_COMM_WORLD, &r1 );
MPI_Start( &r1 );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
MPI_Cancel( &r1 );
MPI_Testany( 1, &r1, &idx, &flag, &status );
MPI_Test_cancelled( &status, &flag );
if (flag) {
err++;
printf( "Cancel of a send succeeded where it shouldn't (Testany).\n" );
}
MPI_Request_free( &r1 );
/* Cancel succeeds for waitsome */
MPI_Send_init( buf, 10, MPI_INT, partner, 0, MPI_COMM_WORLD, &r1 );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
MPI_Start( &r1 );
MPI_Cancel( &r1 );
MPI_Waitsome( 1, &r1, &idx, &index, &status );
MPI_Test_cancelled( &status, &flag );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
if (!flag) {
err++;
printf( "Cancel of a send failed where it should succeed (Waitsome).\n" );
}
MPI_Request_free( &r1 );
/* Cancel fails for testsome*/
buf[0] = 3;
MPI_Send_init( buf, 3, MPI_INT, partner, 2, MPI_COMM_WORLD, &r1 );
MPI_Start( &r1 );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
MPI_Cancel( &r1 );
MPI_Testsome( 1, &r1, &idx, &index, &status );
MPI_Test_cancelled( &status, &flag );
if (flag) {
err++;
printf( "Cancel of a send succeeded where it shouldn't (Testsome).\n" );
}
MPI_Request_free( &r1 );
if (err) {
printf( "Test failed with %d errors.\n", err );
}
else {
printf( "Test passed\n" );
}
}
else if (rank == size - 1) {
partner = 0;
/* Cancel succeeds for wait/waitall */
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
/* Cancel fails for test/testall */
buf[0] = -1;
MPI_Recv( buf, 3, MPI_INT, partner, 2, MPI_COMM_WORLD, &status );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
if (buf[0] == -1) {
printf( "Receive buffer did not change even though cancel should not have suceeded! (Test).\n" );
}
/* Cancel succeeds for waitany */
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
/* Cancel fails for testany */
buf[0] = -1;
MPI_Recv( buf, 3, MPI_INT, partner, 2, MPI_COMM_WORLD, &status );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
if (buf[0] == -1) {
printf( "Receive buffer did not change even though cancel should not have suceeded! (Testany).\n" );
}
/* Cancel succeeds for waitsome */
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
/* Cancel fails for testsome */
buf[0] = -1;
MPI_Recv( buf, 3, MPI_INT, partner, 2, MPI_COMM_WORLD, &status );
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_BOTTOM, 0, MPI_INT, partner, 1,
MPI_COMM_WORLD, &status );
if (buf[0] == -1) {
printf( "Receive buffer did not change even though cancel should not have suceeded! (Test).\n" );
}
}
MPI_Finalize();
return 0;
}
FORT_NAME( mpi_test , MPI_TEST)(int *request, int *flag, int *status,
int *ierror)
{
*ierror=MPI_Test( (void *)request ,flag,(MPI_Status *)status);
}
void do_supervisor_stuff(int argc, char ** argv, struct mw_api_spec *f)
{
//DEBUG_PRINT(("supervisor starting"));
int number_of_slaves;
MPI_Comm_size(MPI_COMM_WORLD, &number_of_slaves);
number_of_slaves = number_of_slaves - 2;
MPI_Status status, status1, status2;
MPI_Request request1, request2;
// keep track of start times
double * assignment_time1 = malloc(sizeof(double)*number_of_slaves);
double * assignment_time2 = malloc(sizeof(double)*number_of_slaves);
// determine how long each worker took
double * complete_time = malloc(sizeof(double)*number_of_slaves);
// booleans for failure
int * failed_worker = calloc(sizeof(int), number_of_slaves);
// supervisor does blocking receive to get list of workers and their start times
MPI_Recv(assignment_time1, number_of_slaves, MPI_DOUBLE, 0, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
//assignment_time2 = assignment_time1;
// calc approximate time diff between sup and master
double master_time = assignment_time1[number_of_slaves-1];
double mytime_off_by = MPI_Wtime() - master_time;
//DEBUG_PRINT(("supervisor knows when the workers started"));
int units_received = 0;
int i;
int kill_msg, flag1, received_update;
double tot_time=0, sq_err=0, mean=0, stddev=0, threshold=0;
//set up receivers for kill and master updates
MPI_Irecv(&kill_msg, 1, MPI_INT, 0, KILL_TAG, MPI_COMM_WORLD, &request1);
MPI_Irecv(assignment_time2, number_of_slaves, MPI_DOUBLE, 0, SUPERVISOR_TAG, MPI_COMM_WORLD, &request2);
//waiting for updates on start times from master
while(1)
{
//kill myself if the master says so
MPI_Test(&request1, &flag1, &status1);
if(flag1)
{
//DEBUG_PRINT(("SUPERVISOR SUICIDE!"));
return;
}
//get a new start time array from master
MPI_Test(&request2, &received_update, &status2);
if(received_update)
{
//DEBUG_PRINT(("got an update from master"));
}
//DEBUG_PRINT(("supervisor is about to check if the slaves are different"));
int found_change = 0;
//check for differences in working slaves
for(i=0; i<number_of_slaves; i++)
{
if(failed_worker[i] != 0)
{
continue;
}
if(assignment_time1[i] == 0.0 && assignment_time2[i] != 0.0)
{
//a previously idle worked got assigned something
assignment_time1[i] = assignment_time2[i];
found_change = 1;
DEBUG_PRINT(("Worker %d just got off his lazy ass", i));
continue;
}
if(assignment_time2[i] == 0.0)
{
//worker is currently idle, not dead
if(assignment_time1[i] != 0.0)
{
DEBUG_PRINT(("Just found out %d is idle", i));
assignment_time1[i] = 0.0;
found_change = 1;
}
continue;
}
//we have a good worker!
if(assignment_time1[i] != assignment_time2[i] && received_update)
{
//DEBUG_PRINT(("supervisor is impressed by his good worker %d", i));
complete_time[i] = assignment_time2[i] - assignment_time1[i];
units_received++;
tot_time += complete_time[i];
mean = tot_time/units_received;
sq_err += pow(complete_time[i] - mean, 2);
stddev = sqrt(sq_err/units_received);
//we have enough data to update threshold
if(units_received >= number_of_slaves/2)
{
//DEBUG_PRINT(("the stddev is %f", stddev));
threshold = mean + 10*stddev + 0.1;
//DEBUG_PRINT(("the threshold is %f", threshold));
}
assignment_time1[i] = assignment_time2[i];
found_change = 1;
}
//if we have enough data, we can tell if we have a bad worker :(
else if(threshold>0 && assignment_time1[i]==assignment_time2[i] && MPI_Wtime() - mytime_off_by - assignment_time1[i] > threshold)
{
assert(assignment_time1[i] != 0.0);
DEBUG_PRINT(("methinks someone is slacking %d", i));
MPI_Send(&i, 1, MPI_INT, 0, FAIL_TAG, MPI_COMM_WORLD);
failed_worker[i] = 1;
}
}
if(received_update)
{
if(found_change == 0)
{
DEBUG_PRINT(("Received an update without any change :("));
}
MPI_Irecv(assignment_time2, number_of_slaves, MPI_DOUBLE, 0, SUPERVISOR_TAG, MPI_COMM_WORLD, &request2);
}
}
}
int
main (int argc, char **argv)
{
int nprocs = -1;
int rank = -1;
int comm = MPI_COMM_WORLD;
char processor_name[128];
int namelen = 128;
int bbuf[(BUF_SIZE + MPI_BSEND_OVERHEAD) * 2 * NUM_BSEND_TYPES];
int buf[BUF_SIZE * 2 * NUM_SEND_TYPES + SLOP];
int i, j, k, l, m, at_size, send_t_number, index, outcount, total, flag;
int num_errors, error_count, indices[2 * NUM_SEND_TYPES];
MPI_Request aReq[2 * NUM_SEND_TYPES];
MPI_Status aStatus[2 * NUM_SEND_TYPES];
/* init */
MPI_Init (&argc, &argv);
MPI_Comm_size (comm, &nprocs);
MPI_Comm_rank (comm, &rank);
MPI_Get_processor_name (processor_name, &namelen);
printf ("(%d) is alive on %s\n", rank, processor_name);
fflush (stdout);
MPI_Buffer_attach (bbuf, sizeof(int) *
(BUF_SIZE + MPI_BSEND_OVERHEAD) * 2 * NUM_BSEND_TYPES);
if (rank == 0) {
/* set up persistent sends... */
send_t_number = NUM_SEND_TYPES - NUM_PERSISTENT_SEND_TYPES;
MPI_Send_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Send_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Bsend_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Bsend_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Rsend_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Rsend_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Ssend_init (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Ssend_init (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
}
for (m = 0; m < NUM_RECV_TYPES; m++) {
if ((m == 1) && (rank == 1)) {
/* set up the persistent receives... */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Recv_init (&buf[j * BUF_SIZE],
BUF_SIZE + (j % 2) * SLOP,
MPI_INT, 0, j, comm, &aReq[j]);
}
}
for (l = 0; l < (NUM_COMPLETION_MECHANISMS * 2); l++) {
for (k = 0; k < (NUM_COMPLETION_MECHANISMS * 2); k++) {
if (rank == 0) {
/* initialize all of the send buffers */
for (j = 0; j < NUM_SEND_TYPES; j++) {
for (i = 0; i < BUF_SIZE; i++) {
buf[2 * j * BUF_SIZE + i] = i;
buf[((2 * j + 1) * BUF_SIZE) + i] = BUF_SIZE - 1 - i;
}
}
}
else if (rank == 1) {
/* zero out all of the receive buffers */
bzero (buf, sizeof(int) * BUF_SIZE * 2 * NUM_SEND_TYPES);
}
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
/* set up transient sends... */
send_t_number = 0;
MPI_Isend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Isend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Ibsend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Ibsend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
/* Barrier to ensure receives are posted for rsends... */
MPI_Barrier(MPI_COMM_WORLD);
MPI_Irsend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Irsend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
send_t_number++;
MPI_Issend (&buf[send_t_number * 2 * BUF_SIZE], BUF_SIZE, MPI_INT,
1, send_t_number * 2, comm, &aReq[send_t_number * 2]);
MPI_Issend (&buf[(send_t_number * 2 + 1) * BUF_SIZE],
BUF_SIZE, MPI_INT, 1, send_t_number * 2 + 1,
comm, &aReq[send_t_number * 2 + 1]);
/* just to be paranoid */
send_t_number++;
assert (send_t_number ==
NUM_SEND_TYPES - NUM_PERSISTENT_SEND_TYPES);
/* start the persistent sends... */
if (k % 2) {
MPI_Startall (NUM_PERSISTENT_SEND_TYPES * 2,
&aReq[2 * send_t_number]);
}
else {
for (j = 0; j < NUM_PERSISTENT_SEND_TYPES * 2; j++) {
MPI_Start (&aReq[2 * send_t_number + j]);
}
}
/* complete the sends */
switch (k/2) {
case 0:
/* use MPI_Wait */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Wait (&aReq[j], &aStatus[j]);
}
break;
case 1:
/* use MPI_Waitall */
MPI_Waitall (NUM_SEND_TYPES * 2, aReq, aStatus);
break;
case 2:
/* use MPI_Waitany */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Waitany (NUM_SEND_TYPES * 2, aReq, &index, aStatus);
}
break;
case 3:
/* use MPI_Waitsome */
total = 0;
while (total < NUM_SEND_TYPES * 2) {
MPI_Waitsome (NUM_SEND_TYPES * 2, aReq,
&outcount, indices, aStatus);
total += outcount;
}
break;
case 4:
/* use MPI_Test */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
flag = 0;
while (!flag) {
MPI_Test (&aReq[j], &flag, &aStatus[j]);
}
}
break;
case 5:
/* use MPI_Testall */
flag = 0;
while (!flag) {
MPI_Testall (NUM_SEND_TYPES * 2, aReq, &flag, aStatus);
}
break;
case 6:
/* use MPI_Testany */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
flag = 0;
while (!flag) {
MPI_Testany (NUM_SEND_TYPES * 2, aReq,
&index, &flag, aStatus);
}
}
break;
case 7:
/* use MPI_Testsome */
total = 0;
while (total < NUM_SEND_TYPES * 2) {
outcount = 0;
while (!outcount) {
MPI_Testsome (NUM_SEND_TYPES * 2, aReq,
&outcount, indices, aStatus);
}
total += outcount;
}
break;
default:
assert (0);
break;
}
}
else if (rank == 1) {
/* start receives for all of the sends */
if (m == 0) {
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Irecv (&buf[j * BUF_SIZE],
BUF_SIZE + (j % 2) * SLOP,
MPI_INT, 0, j, comm, &aReq[j]);
}
}
else {
/* start the persistent receives... */
if (l % 2) {
MPI_Startall (NUM_SEND_TYPES * 2, aReq);
}
else {
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Start (&aReq[j]);
}
}
}
/* Barrier to ensure receives are posted for rsends... */
MPI_Barrier(MPI_COMM_WORLD);
/* complete all of the receives... */
switch (l/2) {
case 0:
/* use MPI_Wait */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Wait (&aReq[j], &aStatus[j]);
}
break;
case 1:
/* use MPI_Waitall */
MPI_Waitall (NUM_SEND_TYPES * 2, aReq, aStatus);
break;
case 2:
/* use MPI_Waitany */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
MPI_Waitany (NUM_SEND_TYPES * 2, aReq, &index, aStatus);
}
break;
case 3:
/* use MPI_Waitsome */
total = 0;
while (total < NUM_SEND_TYPES * 2) {
MPI_Waitsome (NUM_SEND_TYPES * 2, aReq,
&outcount, indices, aStatus);
total += outcount;
}
break;
case 4:
/* use MPI_Test */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
flag = 0;
while (!flag) {
MPI_Test (&aReq[j], &flag, &aStatus[j]);
}
}
break;
case 5:
/* use MPI_Testall */
flag = 0;
while (!flag) {
MPI_Testall (NUM_SEND_TYPES * 2, aReq, &flag, aStatus);
}
break;
case 6:
/* use MPI_Testany */
for (j = 0; j < NUM_SEND_TYPES * 2; j++) {
flag = 0;
while (!flag) {
MPI_Testany (NUM_SEND_TYPES * 2, aReq,
&index, &flag, aStatus);
}
}
break;
case 7:
/* use MPI_Testsome */
total = 0;
while (total < NUM_SEND_TYPES * 2) {
outcount = 0;
while (!outcount) {
MPI_Testsome (NUM_SEND_TYPES * 2, aReq,
&outcount, indices, aStatus);
}
total += outcount;
}
break;
default:
assert (0);
break;
}
}
else {
/* Barrier to ensure receives are posted for rsends... */
MPI_Barrier(MPI_COMM_WORLD);
}
}
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
/* free the persistent send requests */
for (i = 2* (NUM_SEND_TYPES - NUM_PERSISTENT_SEND_TYPES);
i < 2 * NUM_SEND_TYPES; i++) {
MPI_Request_free (&aReq[i]);
}
}
else if (rank == 1) {
/* free the persistent receive requests */
for (i = 0; i < 2 * NUM_SEND_TYPES; i++) {
MPI_Request_free (&aReq[i]);
}
}
MPI_Buffer_detach (bbuf, &at_size);
assert (at_size ==
sizeof(int) * (BUF_SIZE + MPI_BSEND_OVERHEAD) * 2 * NUM_BSEND_TYPES);
MPI_Finalize ();
printf ("(%d) Finished normally\n", rank);
}
void declareBindings (void)
{
/* === Point-to-point === */
void* buf;
int count;
MPI_Datatype datatype;
int dest;
int tag;
MPI_Comm comm;
MPI_Send (buf, count, datatype, dest, tag, comm); // L12
int source;
MPI_Status status;
MPI_Recv (buf, count, datatype, source, tag, comm, &status); // L15
MPI_Get_count (&status, datatype, &count);
MPI_Bsend (buf, count, datatype, dest, tag, comm);
MPI_Ssend (buf, count, datatype, dest, tag, comm);
MPI_Rsend (buf, count, datatype, dest, tag, comm);
void* buffer;
int size;
MPI_Buffer_attach (buffer, size); // L22
MPI_Buffer_detach (buffer, &size);
MPI_Request request;
MPI_Isend (buf, count, datatype, dest, tag, comm, &request); // L25
MPI_Ibsend (buf, count, datatype, dest, tag, comm, &request);
MPI_Issend (buf, count, datatype, dest, tag, comm, &request);
MPI_Irsend (buf, count, datatype, dest, tag, comm, &request);
MPI_Irecv (buf, count, datatype, source, tag, comm, &request);
MPI_Wait (&request, &status);
int flag;
MPI_Test (&request, &flag, &status); // L32
MPI_Request_free (&request);
MPI_Request* array_of_requests;
int index;
MPI_Waitany (count, array_of_requests, &index, &status); // L36
MPI_Testany (count, array_of_requests, &index, &flag, &status);
MPI_Status* array_of_statuses;
MPI_Waitall (count, array_of_requests, array_of_statuses); // L39
MPI_Testall (count, array_of_requests, &flag, array_of_statuses);
int incount;
int outcount;
int* array_of_indices;
MPI_Waitsome (incount, array_of_requests, &outcount, array_of_indices,
array_of_statuses); // L44--45
MPI_Testsome (incount, array_of_requests, &outcount, array_of_indices,
array_of_statuses); // L46--47
MPI_Iprobe (source, tag, comm, &flag, &status); // L48
MPI_Probe (source, tag, comm, &status);
MPI_Cancel (&request);
MPI_Test_cancelled (&status, &flag);
MPI_Send_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Bsend_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Ssend_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Rsend_init (buf, count, datatype, dest, tag, comm, &request);
MPI_Recv_init (buf, count, datatype, source, tag, comm, &request);
MPI_Start (&request);
MPI_Startall (count, array_of_requests);
void* sendbuf;
int sendcount;
MPI_Datatype sendtype;
int sendtag;
void* recvbuf;
int recvcount;
MPI_Datatype recvtype;
MPI_Datatype recvtag;
MPI_Sendrecv (sendbuf, sendcount, sendtype, dest, sendtag,
recvbuf, recvcount, recvtype, source, recvtag,
comm, &status); // L67--69
MPI_Sendrecv_replace (buf, count, datatype, dest, sendtag, source, recvtag,
comm, &status); // L70--71
MPI_Datatype oldtype;
MPI_Datatype newtype;
MPI_Type_contiguous (count, oldtype, &newtype); // L74
int blocklength;
{
int stride;
MPI_Type_vector (count, blocklength, stride, oldtype, &newtype); // L78
}
{
MPI_Aint stride;
MPI_Type_hvector (count, blocklength, stride, oldtype, &newtype); // L82
}
int* array_of_blocklengths;
{
int* array_of_displacements;
MPI_Type_indexed (count, array_of_blocklengths, array_of_displacements,
oldtype, &newtype); // L87--88
}
{
MPI_Aint* array_of_displacements;
MPI_Type_hindexed (count, array_of_blocklengths, array_of_displacements,
oldtype, &newtype); // L92--93
MPI_Datatype* array_of_types;
MPI_Type_struct (count, array_of_blocklengths, array_of_displacements,
array_of_types, &newtype); // L95--96
}
void* location;
MPI_Aint address;
MPI_Address (location, &address); // L100
MPI_Aint extent;
MPI_Type_extent (datatype, &extent); // L102
MPI_Type_size (datatype, &size);
MPI_Aint displacement;
MPI_Type_lb (datatype, &displacement); // L105
MPI_Type_ub (datatype, &displacement);
MPI_Type_commit (&datatype);
MPI_Type_free (&datatype);
MPI_Get_elements (&status, datatype, &count);
void* inbuf;
void* outbuf;
int outsize;
int position;
MPI_Pack (inbuf, incount, datatype, outbuf, outsize, &position, comm); // L114
int insize;
MPI_Unpack (inbuf, insize, &position, outbuf, outcount, datatype,
comm); // L116--117
MPI_Pack_size (incount, datatype, comm, &size);
/* === Collectives === */
MPI_Barrier (comm); // L121
int root;
MPI_Bcast (buffer, count, datatype, root, comm); // L123
MPI_Gather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
root, comm); // L124--125
int* recvcounts;
int* displs;
MPI_Gatherv (sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
root, comm); // L128--130
MPI_Scatter (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
root, comm); // L131--132
int* sendcounts;
MPI_Scatterv (sendbuf, sendcounts, displs, sendtype,
recvbuf, recvcount, recvtype, root, comm); // L134--135
MPI_Allgather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
comm); // L136--137
MPI_Allgatherv (sendbuf, sendcount, sendtype,
recvbuf, recvcounts, displs, recvtype,
comm); // L138--140
MPI_Alltoall (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
comm); // L141--142
int* sdispls;
int* rdispls;
MPI_Alltoallv (sendbuf, sendcounts, sdispls, sendtype,
recvbuf, recvcounts, rdispls, recvtype,
comm); // L145--147
MPI_Op op;
MPI_Reduce (sendbuf, recvbuf, count, datatype, op, root, comm); // L149
#if 0
MPI_User_function function;
int commute;
MPI_Op_create (function, commute, &op); // L153
#endif
MPI_Op_free (&op); // L155
MPI_Allreduce (sendbuf, recvbuf, count, datatype, op, comm);
MPI_Reduce_scatter (sendbuf, recvbuf, recvcounts, datatype, op, comm);
MPI_Scan (sendbuf, recvbuf, count, datatype, op, comm);
/* === Groups, contexts, and communicators === */
MPI_Group group;
MPI_Group_size (group, &size); // L162
int rank;
MPI_Group_rank (group, &rank); // L164
MPI_Group group1;
int n;
int* ranks1;
MPI_Group group2;
int* ranks2;
MPI_Group_translate_ranks (group1, n, ranks1, group2, ranks2); // L170
int result;
MPI_Group_compare (group1, group2, &result); // L172
MPI_Group newgroup;
MPI_Group_union (group1, group2, &newgroup); // L174
MPI_Group_intersection (group1, group2, &newgroup);
MPI_Group_difference (group1, group2, &newgroup);
int* ranks;
MPI_Group_incl (group, n, ranks, &newgroup); // L178
MPI_Group_excl (group, n, ranks, &newgroup);
extern int ranges[][3];
MPI_Group_range_incl (group, n, ranges, &newgroup); // L181
MPI_Group_range_excl (group, n, ranges, &newgroup);
MPI_Group_free (&group);
MPI_Comm_size (comm, &size);
MPI_Comm_rank (comm, &rank);
MPI_Comm comm1;
MPI_Comm comm2;
MPI_Comm_compare (comm1, comm2, &result);
MPI_Comm newcomm;
MPI_Comm_dup (comm, &newcomm);
MPI_Comm_create (comm, group, &newcomm);
int color;
int key;
MPI_Comm_split (comm, color, key, &newcomm); // L194
MPI_Comm_free (&comm);
MPI_Comm_test_inter (comm, &flag);
MPI_Comm_remote_size (comm, &size);
MPI_Comm_remote_group (comm, &group);
MPI_Comm local_comm;
int local_leader;
MPI_Comm peer_comm;
int remote_leader;
MPI_Comm newintercomm;
MPI_Intercomm_create (local_comm, local_leader, peer_comm, remote_leader, tag,
&newintercomm); // L204--205
MPI_Comm intercomm;
MPI_Comm newintracomm;
int high;
MPI_Intercomm_merge (intercomm, high, &newintracomm); // L209
int keyval;
#if 0
MPI_Copy_function copy_fn;
MPI_Delete_function delete_fn;
void* extra_state;
MPI_Keyval_create (copy_fn, delete_fn, &keyval, extra_state); // L215
#endif
MPI_Keyval_free (&keyval); // L217
void* attribute_val;
MPI_Attr_put (comm, keyval, attribute_val); // L219
MPI_Attr_get (comm, keyval, attribute_val, &flag);
MPI_Attr_delete (comm, keyval);
/* === Environmental inquiry === */
char* name;
int resultlen;
MPI_Get_processor_name (name, &resultlen); // L226
MPI_Errhandler errhandler;
#if 0
MPI_Handler_function function;
MPI_Errhandler_create (function, &errhandler); // L230
#endif
MPI_Errhandler_set (comm, errhandler); // L232
MPI_Errhandler_get (comm, &errhandler);
MPI_Errhandler_free (&errhandler);
int errorcode;
char* string;
MPI_Error_string (errorcode, string, &resultlen); // L237
int errorclass;
MPI_Error_class (errorcode, &errorclass); // L239
MPI_Wtime ();
MPI_Wtick ();
int argc;
char** argv;
MPI_Init (&argc, &argv); // L244
MPI_Finalize ();
MPI_Initialized (&flag);
MPI_Abort (comm, errorcode);
}
static PetscErrorCode PetscCommBuildTwoSidedFReq_Ibarrier(MPI_Comm comm,PetscMPIInt count,MPI_Datatype dtype,PetscMPIInt nto,const PetscMPIInt *toranks,const void *todata,
PetscMPIInt *nfrom,PetscMPIInt **fromranks,void *fromdata,PetscMPIInt ntags,MPI_Request **toreqs,MPI_Request **fromreqs,
PetscErrorCode (*send)(MPI_Comm,const PetscMPIInt[],PetscMPIInt,PetscMPIInt,void*,MPI_Request[],void*),
PetscErrorCode (*recv)(MPI_Comm,const PetscMPIInt[],PetscMPIInt,void*,MPI_Request[],void*),void *ctx)
{
PetscErrorCode ierr;
PetscMPIInt nrecvs,tag,*tags,done,i;
MPI_Aint lb,unitbytes;
char *tdata;
MPI_Request *sendreqs,*usendreqs,*req,barrier;
PetscSegBuffer segrank,segdata,segreq;
PetscBool barrier_started;
PetscFunctionBegin;
ierr = PetscCommDuplicate(comm,&comm,&tag);CHKERRQ(ierr);
ierr = PetscMalloc1(ntags,&tags);CHKERRQ(ierr);
for (i=0; i<ntags; i++) {
ierr = PetscCommGetNewTag(comm,&tags[i]);CHKERRQ(ierr);
}
ierr = MPI_Type_get_extent(dtype,&lb,&unitbytes);CHKERRQ(ierr);
if (lb != 0) SETERRQ1(comm,PETSC_ERR_SUP,"Datatype with nonzero lower bound %ld\n",(long)lb);
tdata = (char*)todata;
ierr = PetscMalloc1(nto,&sendreqs);CHKERRQ(ierr);
ierr = PetscMalloc1(nto*ntags,&usendreqs);CHKERRQ(ierr);
/* Post synchronous sends */
for (i=0; i<nto; i++) {
ierr = MPI_Issend((void*)(tdata+count*unitbytes*i),count,dtype,toranks[i],tag,comm,sendreqs+i);CHKERRQ(ierr);
}
/* Post actual payloads. These are typically larger messages. Hopefully sending these later does not slow down the
* synchronous messages above. */
for (i=0; i<nto; i++) {
PetscMPIInt k;
for (k=0; k<ntags; k++) usendreqs[i*ntags+k] = MPI_REQUEST_NULL;
ierr = (*send)(comm,tags,i,toranks[i],tdata+count*unitbytes*i,usendreqs+i*ntags,ctx);CHKERRQ(ierr);
}
ierr = PetscSegBufferCreate(sizeof(PetscMPIInt),4,&segrank);CHKERRQ(ierr);
ierr = PetscSegBufferCreate(unitbytes,4*count,&segdata);CHKERRQ(ierr);
ierr = PetscSegBufferCreate(sizeof(MPI_Request),4,&segreq);CHKERRQ(ierr);
nrecvs = 0;
barrier = MPI_REQUEST_NULL;
/* MPICH-3.2 sometimes does not create a request in some "optimized" cases. This is arguably a standard violation,
* but we need to work around it. */
barrier_started = PETSC_FALSE;
for (done=0; !done; ) {
PetscMPIInt flag;
MPI_Status status;
ierr = MPI_Iprobe(MPI_ANY_SOURCE,tag,comm,&flag,&status);CHKERRQ(ierr);
if (flag) { /* incoming message */
PetscMPIInt *recvrank,k;
void *buf;
ierr = PetscSegBufferGet(segrank,1,&recvrank);CHKERRQ(ierr);
ierr = PetscSegBufferGet(segdata,count,&buf);CHKERRQ(ierr);
*recvrank = status.MPI_SOURCE;
ierr = MPI_Recv(buf,count,dtype,status.MPI_SOURCE,tag,comm,MPI_STATUS_IGNORE);CHKERRQ(ierr);
ierr = PetscSegBufferGet(segreq,ntags,&req);CHKERRQ(ierr);
for (k=0; k<ntags; k++) req[k] = MPI_REQUEST_NULL;
ierr = (*recv)(comm,tags,status.MPI_SOURCE,buf,req,ctx);CHKERRQ(ierr);
nrecvs++;
}
if (!barrier_started) {
PetscMPIInt sent,nsends;
ierr = PetscMPIIntCast(nto,&nsends);CHKERRQ(ierr);
ierr = MPI_Testall(nsends,sendreqs,&sent,MPI_STATUSES_IGNORE);CHKERRQ(ierr);
if (sent) {
#if defined(PETSC_HAVE_MPI_IBARRIER)
ierr = MPI_Ibarrier(comm,&barrier);CHKERRQ(ierr);
#elif defined(PETSC_HAVE_MPIX_IBARRIER)
ierr = MPIX_Ibarrier(comm,&barrier);CHKERRQ(ierr);
#endif
barrier_started = PETSC_TRUE;
}
} else {
ierr = MPI_Test(&barrier,&done,MPI_STATUS_IGNORE);CHKERRQ(ierr);
}
}
*nfrom = nrecvs;
ierr = PetscSegBufferExtractAlloc(segrank,fromranks);CHKERRQ(ierr);
ierr = PetscSegBufferDestroy(&segrank);CHKERRQ(ierr);
ierr = PetscSegBufferExtractAlloc(segdata,fromdata);CHKERRQ(ierr);
ierr = PetscSegBufferDestroy(&segdata);CHKERRQ(ierr);
*toreqs = usendreqs;
ierr = PetscSegBufferExtractAlloc(segreq,fromreqs);CHKERRQ(ierr);
ierr = PetscSegBufferDestroy(&segreq);CHKERRQ(ierr);
ierr = PetscFree(sendreqs);CHKERRQ(ierr);
ierr = PetscFree(tags);CHKERRQ(ierr);
ierr = PetscCommDestroy(&comm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char *argv[])
{
// declare mpz structs
mpz_t n;
mpz_t zero;
mpz_t r;
mpz_t d;
mpz_t start;
mpz_t end;
mpz_t temp;
mpz_t my_rank_mpz;
mpz_t local_end;
mpz_t current_prime;
mpz_t global_end;
mpz_init_set_ui(zero, 0);
mpz_init(d);
double t1, t2, totaltime;
int tag = 0;
int my_rank, p;
int source, dest, i;
mpz_init_set_str(n, argv[1], 10); // initialize the key (n)
mpz_init(global_end);
mpz_sqrt(global_end, n);
// MPI boilerplate
MPI_Status status;
MPI_Request request;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &p);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
t1 = MPI_Wtime();
// initialize mpz structs
mpz_init(r);
mpz_init(start);
mpz_init(local_end);
mpz_init(end);
mpz_init(temp);
mpz_init(current_prime);
mpz_init(my_rank_mpz);
mpz_init_set_ui(start, 2);
mpz_set(temp, start);
size_t temp_window_size = mpz_sizeinbase(temp, 2);
mpz_add_ui(temp, temp, temp_window_size);
for (i = 0; i < my_rank; i++){
temp_window_size = mpz_sizeinbase(temp, 2);
mpz_add_ui(temp, temp, temp_window_size);
}
mpz_set(start, temp);
size_t window_size = mpz_sizeinbase(start, 2);
mpz_add_ui(local_end, start, window_size);
mpz_init_set_ui(my_rank_mpz, my_rank);
mpz_init_set(end, global_end);
mpz_nextprime(current_prime, start);
mpz_init_set_str(n, argv[1], 10);
int check = 0;
int flag = 0;
int found = 0;
MPI_Irecv(&found, 1, MPI_INTEGER, MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &request);
MPI_Barrier(MPI_COMM_WORLD);
while (mpz_cmp(start, end) <= 0){
while(mpz_cmp(current_prime, local_end) <= 0 && mpz_cmp(current_prime, end) <= 0){
mpz_mod(r, n, current_prime);
if (mpz_cmp(zero, r) == 0){
mpz_divexact(d, n, current_prime);
gmp_printf("%Zd %Zd\n", d, current_prime);
found = 1;
for (dest = 0; dest < p; dest++){
MPI_Isend(&found, 1, MPI_INTEGER, dest, tag, MPI_COMM_WORLD, &request);
}
break;
}
check++;
if (check == 500){
check = 0;
MPI_Test(&request, &flag, &status);
if (flag){
found = 1;
break;
}
}
mpz_nextprime(current_prime, current_prime);
}
if( found == 1){
break;
}
// calculate new start index
mpz_set(temp, start);
size_t temp_window_size = mpz_sizeinbase(start, 2);
mpz_add_ui(temp, temp, temp_window_size);
mpz_set(start, local_end);
for (i = my_rank + 1; i < p; i++){
temp_window_size = mpz_sizeinbase(temp, 2);
mpz_add_ui(temp, temp, temp_window_size);
}
for (i = 0; i < my_rank; i++){
temp_window_size = mpz_sizeinbase(temp, 2);
mpz_add_ui(temp, temp, temp_window_size);
}
mpz_set(start, temp);
temp_window_size = mpz_sizeinbase(start, 2);
mpz_add_ui(local_end, start, temp_window_size);
mpz_nextprime(current_prime, start);
}
t2 = MPI_Wtime();
totaltime = t2 - t1;
MPI_Barrier(MPI_COMM_WORLD);
if (my_rank == 0){
char buf[0x1000];
snprintf(buf, sizeof(buf), "time_%s", argv[1]);
FILE *f = fopen(buf, "w");
double other_totaltime = 0;
fprintf(f, "0\t%1.2e\n", totaltime);
for (source = 1; source < p; source++){
MPI_Recv(&other_totaltime, 1, MPI_DOUBLE, source, tag, MPI_COMM_WORLD, &status);
fprintf(f, "%d\t%1.2e\n", source, other_totaltime);
}
fclose(f);
}
else{
MPI_Send(&totaltime, 1, MPI_DOUBLE, 0, tag, MPI_COMM_WORLD);
}
MPI_Finalize();
return 0;
}
void connection_handler::handle_messages()
{
detail::handling_messages hm(handling_messages_); // reset on exit
bool bootstrapping = hpx::is_starting();
bool has_work = true;
std::size_t k = 0;
hpx::util::high_resolution_timer t;
std::list<std::pair<int, MPI_Request> > close_requests;
// We let the message handling loop spin for another 2 seconds to avoid the
// costs involved with posting it to asio
while(bootstrapping || (!stopped_ && has_work) || (!has_work && t.elapsed() < 2.0))
{
// break the loop if someone requested to pause the parcelport
if(!enable_parcel_handling_) break;
// handle all send requests
{
hpx::lcos::local::spinlock::scoped_lock l(senders_mtx_);
for(
senders_type::iterator it = senders_.begin();
!stopped_ && enable_parcel_handling_ && it != senders_.end();
/**/)
{
if((*it)->done())
{
it = senders_.erase(it);
}
else
{
++it;
}
}
has_work = !senders_.empty();
}
// Send the pending close requests
{
hpx::lcos::local::spinlock::scoped_lock l(close_mtx_);
typedef std::pair<int, int> pair_type;
BOOST_FOREACH(pair_type p, pending_close_requests_)
{
header close_request = header::close(p.first, p.second);
close_requests.push_back(std::make_pair(p.first, MPI_Request()));
MPI_Isend(
close_request.data(), // Data pointer
close_request.data_size_, // Size
close_request.type(), // MPI Datatype
close_request.rank(), // Destination
0, // Tag
communicator_, // Communicator
&close_requests.back().second
);
}
pending_close_requests_.clear();
}
// add new receive requests
std::pair<bool, header> next(acceptor_.next_header());
if(next.first)
{
boost::shared_ptr<receiver> rcv;
header h = next.second;
receivers_tag_map_type & tag_map = receivers_map_[h.rank()];
receivers_tag_map_type::iterator jt = tag_map.find(h.tag());
if(jt != tag_map.end())
{
rcv = jt->second;
}
else
{
rcv = boost::make_shared<receiver>(
communicator_
, get_next_tag()
, h.tag()
, h.rank()
, *this);
tag_map.insert(std::make_pair(h.tag(), rcv));
}
if(h.close_request())
{
rcv->close();
}
else
{
h.assert_valid();
if (static_cast<std::size_t>(h.size()) > this->get_max_message_size())
{
// report this problem ...
HPX_THROW_EXCEPTION(boost::asio::error::operation_not_supported,
"mpi::connection_handler::handle_messages",
"The size of this message exceeds the maximum inbound data size");
return;
}
if(rcv->async_read(h))
{
#ifdef HPX_DEBUG
receivers_type::iterator it = std::find(receivers_.begin(), receivers_.end(), rcv);
HPX_ASSERT(it == receivers_.end());
#endif
receivers_.push_back(rcv);
}
}
}
// handle all receive requests
for(receivers_type::iterator it = receivers_.begin();
it != receivers_.end(); /**/)
{
boost::shared_ptr<receiver> rcv = *it;
if(rcv->done())
{
HPX_ASSERT(rcv->sender_tag() != -1);
if(rcv->closing())
{
receivers_tag_map_type & tag_map = receivers_map_[rcv->rank()];
receivers_tag_map_type::iterator jt = tag_map.find(rcv->sender_tag());
HPX_ASSERT(jt != tag_map.end());
tag_map.erase(jt);
{
hpx::lcos::local::spinlock::scoped_lock l(tag_mtx_);
free_tags_.push_back(rcv->tag());
}
}
it = receivers_.erase(it);
}
else
{
++it;
}
}
if(!has_work) has_work = !receivers_.empty();
// handle completed close requests
for(
std::list<std::pair<int, MPI_Request> >::iterator it = close_requests.begin();
!stopped_ && enable_parcel_handling_ && it != close_requests.end();
)
{
int completed = 0;
MPI_Status status;
int ret = 0;
ret = MPI_Test(&it->second, &completed, &status);
HPX_ASSERT(ret == MPI_SUCCESS);
if(completed && status.MPI_ERROR != MPI_ERR_PENDING)
{
hpx::lcos::local::spinlock::scoped_lock l(tag_mtx_);
free_tags_.push_back(it->first);
it = close_requests.erase(it);
}
else
{
++it;
}
}
if(!has_work)
has_work = !close_requests.empty();
if (bootstrapping)
bootstrapping = hpx::is_starting();
if(has_work)
{
t.restart();
k = 0;
}
else
{
if(enable_parcel_handling_)
{
hpx::lcos::local::spinlock::yield(k);
++k;
}
}
}
int client_thread(void *ptr) {
int data_in, data_out;
struct client_data_t client_data = *(struct client_data_t *)ptr;
// struct timeval tv;
int percent_done = 0;
int chunks_lim = 0;
int need_to_ask = 0;
int last_chunk = 0;
int req_was_issued = 0;
int req_was_sent = 0;
int isend_in_fly = 1;
int wait_for_answer = 0;
int got_answer = 0;
off_t data_writen = 0;
int chunks_writen = 0;
int fd, ret, i;
bool infly[256];
struct aiocb my_aiocb[client_data.maxreqs];
MPI_Status status;
MPI_Request request;
data_out = M_IFNEWCHUNK;
fd = open( client_data.fname, O_WRONLY|O_CREAT, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
if (fd < 0) {
printf("Rank %05d | Error open() for \n", client_data.rank, strerror(errno), client_data.fname);
}
for (i=0; i < client_data.maxreqs; i++) {
bzero( (char *)&my_aiocb[i], sizeof(struct aiocb) );
// Allocate a data buffer for the aiocb request
my_aiocb[i].aio_buf = calloc(client_data.bsize+1, 1);
if (!my_aiocb[i].aio_buf) {
printf("Rank %05d | Error calloc()[%d]: %s\n", client_data.rank, strerror(errno), i);
exit(errno);
}
// Initialize the necessary fields in the aiocb
my_aiocb[i].aio_fildes = fd;
my_aiocb[i].aio_nbytes = client_data.bsize;
infly[i] = 0;
}
data_in = -1;
while(1) {
/*if (client_data.rank == 0) {
printf("%05d | Percent done %d %d %d %lli\n", client_data.rank, \
percent_done, chunks_lim, client_data.csize, data_writen);
} */
// what hve be done so far?
percent_done = (int) ((100*(data_writen - client_data.csize*(chunks_lim-1)))/client_data.csize);
// triggers only on the beginning. Ugly.
if ((data_writen == 0) && (chunks_lim == 0) && (req_was_issued == 0)) {
percent_done = 100;
}
// we are close to idle. need to ask for another portion of the data
if ((percent_done > PERCENT_LIM) && (last_chunk == 0) && (req_was_sent == 0) ) {
need_to_ask = 1;
got_answer = 0;
data_in = 0;
}
// sending requests
if ( (need_to_ask == 1) && (req_was_issued == 0 ) && (req_was_sent == 0)) {
MPI_Isend(&data_out, 1, MPI_INT, 0, M_TAG_CLIENT, *client_data.thread_comm, &request);
// printf("%05d | Asked for a new chunk\n", client_data.rank);
req_was_issued = 1;
}
// checking if send is finished
if (req_was_issued == 1) {
MPI_Test(&request, &isend_in_fly, &status);
if (isend_in_fly != 0 ) {
// printf("%05d req_was_issued \n",client_data.rank);
req_was_sent = 1;
req_was_issued = 0;
}
}
// if send request gone, run async receive
if ((req_was_sent == 1) && (wait_for_answer == 0)) {
MPI_Irecv(&data_in, 1, MPI_INT, 0, M_TAG_SERVER, *client_data.thread_comm, &request);
// printf("%05d wait_for_answer \n",client_data.rank);
wait_for_answer = 1;
}
// check the satatus of the irecv
if (wait_for_answer == 1) {
MPI_Test(&request, &isend_in_fly, &status);
if (isend_in_fly != 0 ) {
// printf("%05d got_answer \n",client_data.rank);
wait_for_answer = 0;
got_answer = 1;
}
}
// finaly got an answer
if (got_answer == 1) {
// printf("%05d got_answer %d \n",client_data.rank, data_in);
if (data_in == -1) {
break;
}
if (data_in > 0) {
chunks_lim += data_in;
} else {
last_chunk = 1;
// printf("%05d last_chunk \n",client_data.rank);
}
req_was_sent = 0;
need_to_ask = 0;
got_answer = 0;
}
// we are here and not the last chunk? that is bad. Probably needs to decrease PERCENT_LIM
if (data_writen >= chunks_lim * client_data.csize) {
// check if we done what needed
if (last_chunk == 1) {
break;
}
// printf("Rank %05d | OOPS. Wrote %d chunks, %lli \n",client_data.rank,chunks_lim,data_writen);
continue;
}
// writing data to file
for (i=0; i < client_data.maxreqs; i++) {
if ( aio_error( &my_aiocb[i] ) == EINPROGRESS ) {
continue;
}
if ((ret = aio_return( &my_aiocb[i] )) < 0) {
exit(ret);
}
if (data_writen >= chunks_lim * client_data.csize) {
break;
}
my_aiocb[i].aio_offset = data_writen;
if ((ret = aio_write( &my_aiocb[i] )) < 0) {
exit(errno);
}
data_writen += client_data.bsize;
}
}
// waiting for the last bytes are in fly
for (i=0; i < client_data.maxreqs; i++) {
while (aio_error( &my_aiocb[i] ) == EINPROGRESS)
if ((ret = aio_return( &my_aiocb[i] )) < 0) {
exit(ret);
}
}
// send 'bye' to manager thread.
data_out = M_BYE;
MPI_Send(&data_out, 1, MPI_INT, status.MPI_SOURCE, M_TAG_CLIENT, *client_data.thread_comm );
printf("Rank %05d | Wrote %d chunks, %lli \n",client_data.rank,chunks_lim,data_writen);
return 0;
}
double gradientDescent(double **X, double *Y, double *theta, double **meanAndRange, int features, int examples, int numProcs, int procId){
char iters[5];
int iterations;
double alpha = (1.0 / pow(10, (numProcs / 2) + 1)) * pow(10, (numProcs - procId) / 2);
double alphaUpdate = .01 / pow(10, procId % 2);
double hypothesis[examples];
double runningSum;
double gradients[examples];
double intermediateCost, absCost;
double previousCost = 0;
int itsOver = 0;
int flag = 0;
MPI_Status status;
MPI_Request request;
MPI_Irecv(&itsOver, 1, MPI_INT, MPI_ANY_SOURCE, 0, MPI_COMM_WORLD, &request);
/*printf("Gradient descent iterations(1-4 digits): ");
scanf("%s", iters);
iterations = atoi(iters);
*/
iterations = 9999;
/*Iterates gradient descent iterations times*/
for(int i = 0; i < iterations; i++){
/*initialize all the gradients to zero*/
for(int i = 0; i < features; i++){
gradients[i] = 0;
}
/*Sets the values of the hypothesis, based on the current values of theta*/
for(int godDamn = 0; godDamn < examples; godDamn++){
runningSum = 0;
for(int f**k = 0; f**k < features; f**k++){
runningSum += theta[f**k] * X[godDamn][f**k];
}
hypothesis[godDamn] = runningSum;
}
/*Actual gradient descent step- adjusts the values of theta by descending the gradient*/
for(int j = 0; j < examples; j++){
intermediateCost = (hypothesis[j] - Y[j]);
for(int godDamn = 0; godDamn < features; godDamn++){
gradients[godDamn] += intermediateCost * X[j][godDamn];
}
for(int k = 0; k < features;k++){
theta[k] -= (alpha * gradients[k])/examples;
}
absCost = fabs(intermediateCost);
if(absCost > previousCost){
alpha /= 2;
}
else{
alpha += alphaUpdate;
}
previousCost = absCost;
}
if(absCost < 0.0000001){
return 0;
}
MPI_Test(&request, &flag, &status);
if(flag){
return -1;
}
}
return absCost;
}
FC_FUNC( mpi_test , MPI_TEST)(int *request, int *flag, int *status,
int *ierror)
{
*ierror=MPI_Test( (void *)request ,flag,mpi_c_status(status));
}
int wc_mpi_test(wc_mpirequest_t *req, int *flag)
{
int rc = MPI_Test(req, flag, MPI_STATUS_IGNORE);
WC_HANDLE_MPI_ERROR(MPI_Test, rc);
return (rc == MPI_SUCCESS) ? 0 : -1;
}
static void test_pair (void)
{
int prev, next, count, tag, index, i, outcount, indices[2];
int rank, size, flag, ierr, reqcount;
double send_buf[TEST_SIZE], recv_buf[TEST_SIZE];
double buffered_send_buf[TEST_SIZE * 2 + MPI_BSEND_OVERHEAD]; /* factor of two is based on guessing - only dynamic allocation would be safe */
void *buffer;
MPI_Status statuses[2];
MPI_Status status;
MPI_Request requests[2];
MPI_Comm dupcom, intercom;
#ifdef V_T
struct _VT_FuncFrameHandle {
char *name;
int func;
int frame;
};
typedef struct _VT_FuncFrameHandle VT_FuncFrameHandle_t;
VT_FuncFrameHandle_t normal_sends,
buffered_sends,
buffered_persistent_sends,
ready_sends,
sync_sends,
nblock_sends,
nblock_rsends,
nblock_ssends,
pers_sends,
pers_rsends,
pers_ssends,
sendrecv,
sendrecv_repl,
intercomm;
int classid;
VT_classdef( "Application:test_pair", &classid );
#define VT_REGION_DEF( _name, _nameframe, _class ) \
(_nameframe).name=_name; \
VT_funcdef( (_nameframe).name, _class, &((_nameframe).func) );
#define VT_BEGIN_REGION( _nameframe ) \
LOCDEF(); \
VT_begin( (_nameframe).func )
#define VT_END_REGION( _nameframe ) \
LOCDEF(); VT_end( (_nameframe).func )
#else
#define VT_REGION_DEF( _name, _nameframe, _class )
#define VT_BEGIN_REGION( _nameframe )
#define VT_END_REGION( _nameframe )
#endif
ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
ierr = MPI_Comm_size(MPI_COMM_WORLD, &size);
if ( size < 2 ) {
if ( rank == 0 ) {
printf("Program needs to be run on at least 2 processes.\n");
}
ierr = MPI_Abort( MPI_COMM_WORLD, 66 );
}
ierr = MPI_Comm_dup(MPI_COMM_WORLD, &dupcom);
if ( rank >= 2 ) {
/* printf( "%d Calling finalize.\n", rank ); */
ierr = MPI_Finalize( );
exit(0);
}
next = rank + 1;
if (next >= 2)
next = 0;
prev = rank - 1;
if (prev < 0)
prev = 1;
VT_REGION_DEF( "Normal_Sends", normal_sends, classid );
VT_REGION_DEF( "Buffered_Sends", buffered_sends, classid );
VT_REGION_DEF( "Buffered_Persistent_Sends", buffered_persistent_sends, classid );
VT_REGION_DEF( "Ready_Sends", ready_sends, classid );
VT_REGION_DEF( "Sync_Sends", sync_sends, classid );
VT_REGION_DEF( "nblock_Sends", nblock_sends, classid );
VT_REGION_DEF( "nblock_RSends", nblock_rsends, classid );
VT_REGION_DEF( "nblock_SSends", nblock_ssends, classid );
VT_REGION_DEF( "Pers_Sends", pers_sends, classid );
VT_REGION_DEF( "Pers_RSends", pers_rsends, classid );
VT_REGION_DEF( "Pers_SSends", pers_ssends, classid );
VT_REGION_DEF( "SendRecv", sendrecv, classid );
VT_REGION_DEF( "SendRevc_Repl", sendrecv_repl, classid );
VT_REGION_DEF( "InterComm", intercomm, classid );
/*
* Normal sends
*/
VT_BEGIN_REGION( normal_sends );
if (rank == 0)
printf ("Send\n");
tag = 0x100;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Send(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check(recv_buf, prev, tag, count, &status, TEST_SIZE, "send and recv");
}
else {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,"send and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( normal_sends );
/*
* Buffered sends
*/
VT_BEGIN_REGION( buffered_sends );
if (rank == 0)
printf ("Buffered Send\n");
tag = 138;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Buffer_attach(buffered_send_buf, sizeof(buffered_send_buf));
MPI_Bsend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
MPI_Buffer_detach(&buffer, &size);
if(buffer != buffered_send_buf || size != sizeof(buffered_send_buf)) {
printf ("[%d] Unexpected buffer returned by MPI_Buffer_detach(): %p/%d != %p/%d\n", rank, buffer, size, buffered_send_buf, (int)sizeof(buffered_send_buf));
MPI_Abort(MPI_COMM_WORLD, 201);
}
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check(recv_buf, prev, tag, count, &status, TEST_SIZE, "send and recv");
}
else {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,"send and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( buffered_sends );
/*
* Buffered sends
*/
VT_BEGIN_REGION( buffered_persistent_sends );
if (rank == 0)
printf ("Buffered Persistent Send\n");
tag = 238;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Buffer_attach(buffered_send_buf, sizeof(buffered_send_buf));
MPI_Bsend_init(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD, requests);
MPI_Start(requests);
MPI_Wait(requests, statuses);
MPI_Request_free(requests);
MPI_Buffer_detach(&buffer, &size);
if(buffer != buffered_send_buf || size != sizeof(buffered_send_buf)) {
printf ("[%d] Unexpected buffer returned by MPI_Buffer_detach(): %p/%d != %p/%d\n", rank, buffer, size, buffered_send_buf, (int)sizeof(buffered_send_buf));
MPI_Abort(MPI_COMM_WORLD, 201);
}
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check(recv_buf, prev, tag, count, &status, TEST_SIZE, "send and recv");
}
else {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,"send and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( buffered_persistent_sends );
/*
* Ready sends. Note that we must insure that the receive is posted
* before the rsend; this requires using Irecv.
*/
VT_BEGIN_REGION( ready_sends );
if (rank == 0)
printf ("Rsend\n");
tag = 1456;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Recv(MPI_BOTTOM, 0, MPI_INT, next, tag, MPI_COMM_WORLD, &status);
MPI_Rsend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
MPI_Probe(MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &status);
if (status.MPI_SOURCE != prev)
printf ("Incorrect src, expected %d, got %d\n",prev, status.MPI_SOURCE);
if (status.MPI_TAG != tag)
printf ("Incorrect tag, expected %d, got %d\n",tag, status.MPI_TAG);
MPI_Get_count(&status, MPI_DOUBLE, &i);
if (i != count)
printf ("Incorrect count, expected %d, got %d\n",count,i);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"rsend and recv");
}
else {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
MPI_Send( MPI_BOTTOM, 0, MPI_INT, next, tag, MPI_COMM_WORLD);
MPI_Wait(requests, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"rsend and recv");
init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( ready_sends );
/*
* Synchronous sends
*/
VT_BEGIN_REGION( sync_sends );
if (rank == 0)
printf ("Ssend\n");
tag = 1789;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Iprobe(MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &flag, &status);
if (flag)
printf ("Iprobe succeeded! source %d, tag %d\n",status.MPI_SOURCE,
status.MPI_TAG);
MPI_Ssend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
while (!flag)
MPI_Iprobe(MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &flag, &status);
if (status.MPI_SOURCE != prev)
printf ("Incorrect src, expected %d, got %d\n",prev, status.MPI_SOURCE);
if (status.MPI_TAG != tag)
printf ("Incorrect tag, expected %d, got %d\n",tag, status.MPI_TAG);
MPI_Get_count(&status, MPI_DOUBLE, &i);
if (i != count)
printf ("Incorrect count, expected %d, got %d\n",count,i);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE, "ssend and recv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE, "ssend and recv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Ssend(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( sync_sends );
/*
* Nonblocking normal sends
*/
VT_BEGIN_REGION( nblock_sends );
if (rank == 0)
printf ("Isend\n");
tag = 2123;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
init_test_data(send_buf,TEST_SIZE,0);
MPI_Isend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,
(requests+1));
MPI_Waitall(2, requests, statuses);
rq_check( requests, 2, "isend and irecv" );
msg_check(recv_buf,prev,tag,count,statuses, TEST_SIZE,"isend and irecv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check(recv_buf,prev,tag,count,&status, TEST_SIZE,"isend and irecv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Isend(recv_buf, count, MPI_DOUBLE, next, tag,MPI_COMM_WORLD,
(requests));
MPI_Wait((requests), &status);
rq_check(requests, 1, "isend (and recv)");
}
VT_END_REGION( nblock_sends );
/*
* Nonblocking ready sends
*/
VT_BEGIN_REGION( nblock_rsends );
if (rank == 0)
printf ("Irsend\n");
tag = 2456;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
init_test_data(send_buf,TEST_SIZE,0);
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, next, 0,
MPI_BOTTOM, 0, MPI_INT, next, 0,
dupcom, &status);
MPI_Irsend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,
(requests+1));
reqcount = 0;
while (reqcount != 2) {
MPI_Waitany( 2, requests, &index, statuses);
if( index == 0 ) {
memcpy( &status, statuses, sizeof(status) );
}
reqcount++;
}
rq_check( requests, 1, "irsend and irecv");
msg_check(recv_buf,prev,tag,count,&status, TEST_SIZE,"irsend and irecv");
}
else {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests);
MPI_Sendrecv( MPI_BOTTOM, 0, MPI_INT, next, 0,
MPI_BOTTOM, 0, MPI_INT, next, 0,
dupcom, &status);
flag = 0;
while (!flag)
MPI_Test(requests, &flag, &status);
rq_check( requests, 1, "irsend and irecv (test)");
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"irsend and irecv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Irsend(recv_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, requests);
MPI_Waitall(1, requests, statuses);
rq_check( requests, 1, "irsend and irecv");
}
VT_END_REGION( nblock_rsends );
/*
* Nonblocking synchronous sends
*/
VT_BEGIN_REGION( nblock_ssends );
if (rank == 0)
printf ("Issend\n");
tag = 2789;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
MPI_Irecv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, requests );
init_test_data(send_buf,TEST_SIZE,0);
MPI_Issend(send_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,
(requests+1));
flag = 0;
while (!flag)
MPI_Testall(2, requests, &flag, statuses);
rq_check( requests, 2, "issend and irecv (testall)");
msg_check( recv_buf, prev, tag, count, statuses, TEST_SIZE,
"issend and recv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"issend and recv"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Issend(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD,requests);
flag = 0;
while (!flag)
MPI_Testany(1, requests, &index, &flag, statuses);
rq_check( requests, 1, "issend and recv (testany)");
}
VT_END_REGION( nblock_ssends );
/*
* Persistent normal sends
*/
VT_BEGIN_REGION( pers_sends );
if (rank == 0)
printf ("Send_init\n");
tag = 3123;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
MPI_Send_init(send_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, requests);
MPI_Recv_init(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
MPI_COMM_WORLD, (requests+1));
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Startall(2, requests);
MPI_Waitall(2, requests, statuses);
msg_check( recv_buf, prev, tag, count, (statuses+1),
TEST_SIZE, "persistent send/recv");
}
else {
MPI_Start((requests+1));
MPI_Wait((requests+1), &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"persistent send/recv");
init_test_data(send_buf,TEST_SIZE,1);
MPI_Start(requests);
MPI_Wait(requests, &status);
}
MPI_Request_free(requests);
MPI_Request_free((requests+1));
VT_END_REGION( pers_sends );
/*
* Persistent ready sends
*/
VT_BEGIN_REGION( pers_rsends );
if (rank == 0)
printf ("Rsend_init\n");
tag = 3456;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
MPI_Rsend_init(send_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, requests);
MPI_Recv_init(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, (requests+1));
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0); MPI_Barrier( MPI_COMM_WORLD );
MPI_Startall(2, requests);
reqcount = 0;
while (reqcount != 2) {
MPI_Waitsome(2, requests, &outcount, indices, statuses);
for (i=0; i<outcount; i++) {
if (indices[i] == 1) {
msg_check( recv_buf, prev, tag, count, (statuses+i),
TEST_SIZE, "waitsome");
}
reqcount++;
}
}
}
else {
MPI_Start((requests+1)); MPI_Barrier( MPI_COMM_WORLD );
flag = 0;
while (!flag)
MPI_Test((requests+1), &flag, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE, "test");
init_test_data(send_buf,TEST_SIZE,1);
MPI_Start(requests);
MPI_Wait(requests, &status);
}
MPI_Request_free(requests);
MPI_Request_free((requests+1));
VT_END_REGION( pers_rsends );
/*
* Persistent synchronous sends
*/
VT_BEGIN_REGION( pers_ssends );
if (rank == 0)
printf ("Ssend_init\n");
tag = 3789;
count = TEST_SIZE / 3;
clear_test_data(recv_buf,TEST_SIZE);
MPI_Ssend_init(send_buf, count, MPI_DOUBLE, next, tag,
MPI_COMM_WORLD, (requests+1));
MPI_Recv_init(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, requests);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Startall(2, requests);
reqcount = 0;
while (reqcount != 2) {
MPI_Testsome(2, requests, &outcount, indices, statuses);
for (i=0; i<outcount; i++) {
if (indices[i] == 0) {
msg_check( recv_buf, prev, tag, count, (statuses+i),
TEST_SIZE, "testsome");
}
reqcount++;
}
}
}
else {
MPI_Start(requests);
flag = 0;
while (!flag)
MPI_Testany(1, requests, &index, &flag, statuses);
msg_check( recv_buf, prev, tag, count, statuses, TEST_SIZE, "testany" );
init_test_data(send_buf,TEST_SIZE,1);
MPI_Start((requests+1));
MPI_Wait((requests+1), &status);
}
MPI_Request_free(requests);
MPI_Request_free((requests+1));
VT_END_REGION( pers_ssends );
/*
* Send/receive.
*/
VT_BEGIN_REGION( sendrecv );
if (rank == 0)
printf ("Sendrecv\n");
tag = 4123;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
MPI_Sendrecv(send_buf, count, MPI_DOUBLE, next, tag,
recv_buf, count, MPI_DOUBLE, prev, tag,
MPI_COMM_WORLD, &status );
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"sendrecv");
}
else {
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"recv/send"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( sendrecv );
#ifdef V_T
VT_flush();
#endif
/*
* Send/receive replace.
*/
VT_BEGIN_REGION( sendrecv_repl );
if (rank == 0)
printf ("Sendrecv_replace\n");
tag = 4456;
count = TEST_SIZE / 3;
if (rank == 0) {
init_test_data(recv_buf, TEST_SIZE,0);
for (i=count; i< TEST_SIZE; i++)
recv_buf[i] = 0.0;
MPI_Sendrecv_replace(recv_buf, count, MPI_DOUBLE,
next, tag, prev, tag, MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"sendrecvreplace");
}
else {
clear_test_data(recv_buf,TEST_SIZE);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, MPI_COMM_WORLD, &status);
msg_check( recv_buf, prev, tag, count, &status, TEST_SIZE,
"recv/send for replace"); init_test_data(recv_buf,TEST_SIZE,1);
MPI_Send(recv_buf, count, MPI_DOUBLE, next, tag, MPI_COMM_WORLD);
}
VT_END_REGION( sendrecv_repl );
/*
* Send/Receive via inter-communicator
*/
VT_BEGIN_REGION( intercomm );
MPI_Intercomm_create(MPI_COMM_SELF, 0, MPI_COMM_WORLD, next, 1, &intercom);
if (rank == 0)
printf ("Send via inter-communicator\n");
tag = 4018;
count = TEST_SIZE / 5;
clear_test_data(recv_buf,TEST_SIZE);
if (rank == 0) {
init_test_data(send_buf,TEST_SIZE,0);
LOCDEF();
MPI_Send(send_buf, count, MPI_DOUBLE, 0, tag, intercom);
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE, MPI_ANY_SOURCE,
MPI_ANY_TAG, intercom, &status);
msg_check(recv_buf, 0, tag, count, &status, TEST_SIZE, "send and recv via inter-communicator");
}
else if (rank == 1) {
LOCDEF();
MPI_Recv(recv_buf, TEST_SIZE, MPI_DOUBLE,MPI_ANY_SOURCE, MPI_ANY_TAG,
intercom, &status);
msg_check( recv_buf, 0, tag, count, &status, TEST_SIZE,"send and recv via inter-communicator");
init_test_data(recv_buf,TEST_SIZE,0);
MPI_Send(recv_buf, count, MPI_DOUBLE, 0, tag, intercom);
}
VT_END_REGION( normal_sends );
MPI_Comm_free(&intercom);
MPI_Comm_free(&dupcom);
}
void peano::applications::puregrid::repositories::GridConstructionMovieBatchJobRepositoryState::send(int destination, int tag) {
MPI_Request* sendRequestHandle = new MPI_Request();
MPI_Status status;
int flag = 0;
int result;
clock_t timeOutWarning = -1;
clock_t timeOutShutdown = -1;
bool triggeredTimeoutWarning = false;
#ifdef Asserts
_senderRank = -1;
#endif
result = MPI_Isend(
this, 1, Datatype, destination,
tag, tarch::parallel::Node::getInstance().getCommunicator(),
sendRequestHandle
);
if (result!=MPI_SUCCESS) {
std::ostringstream msg;
msg << "was not able to send message peano::applications::puregrid::repositories::GridConstructionMovieBatchJobRepositoryState "
<< toString()
<< " to node " << destination
<< ": " << tarch::parallel::MPIReturnValueToString(result);
_log.error( "send(int)",msg.str() );
}
result = MPI_Test( sendRequestHandle, &flag, &status );
while (!flag) {
if (timeOutWarning==-1) timeOutWarning = tarch::parallel::Node::getInstance().getDeadlockWarningTimeStamp();
if (timeOutShutdown==-1) timeOutShutdown = tarch::parallel::Node::getInstance().getDeadlockTimeOutTimeStamp();
result = MPI_Test( sendRequestHandle, &flag, &status );
if (result!=MPI_SUCCESS) {
std::ostringstream msg;
msg << "testing for finished send task for peano::applications::puregrid::repositories::GridConstructionMovieBatchJobRepositoryState "
<< toString()
<< " sent to node " << destination
<< " failed: " << tarch::parallel::MPIReturnValueToString(result);
_log.error("send(int)", msg.str() );
}
// deadlock aspect
if (
tarch::parallel::Node::getInstance().isTimeOutWarningEnabled() &&
(clock()>timeOutWarning) &&
(!triggeredTimeoutWarning)
) {
tarch::parallel::Node::getInstance().writeTimeOutWarning(
"peano::applications::puregrid::repositories::GridConstructionMovieBatchJobRepositoryState",
"send(int)", destination
);
triggeredTimeoutWarning = true;
}
if (
tarch::parallel::Node::getInstance().isTimeOutDeadlockEnabled() &&
(clock()>timeOutShutdown)
) {
tarch::parallel::Node::getInstance().triggerDeadlockTimeOut(
"peano::applications::puregrid::repositories::GridConstructionMovieBatchJobRepositoryState",
"send(int)", destination
);
}
tarch::parallel::Node::getInstance().receiveDanglingMessages();
}
delete sendRequestHandle;
#ifdef Debug
_log.debug("send(int,int)", "sent " + toString() );
#endif
}
/*
* Class: mpjdev_natmpjdev_NativeRecvRequest
* Method: Test
* Signature: (Lmpjdev/Status;)Lmpjdev/Status;
*/
JNIEXPORT jobject JNICALL Java_mpjdev_natmpjdev_NativeRecvRequest_Test(
JNIEnv *env, jobject thisObject, jobject stat) {
/*
* Conventionaly our itest()
* provide iprobe like functionality.
*
* So we have 3 options:
* 1: change our mpi.Request Test() and don't call Wait() then
* we have to change implementations of itest() in all the devices
*
* 2: Put if-else in mpi.Request Test() to distinguish between native
* and pure. This is messy. And provide the real MPI_Test() functionality
* here.
*
* 3: Provide iprobe functionality here and don't change anything in
* the upper layers. This requires introducing source and tag in
* NativeRecvRequest
*
* We are opting for option 2 right now.
*
*/
int flag;
MPI_Request request = (MPI_Request)(
(*env)->GetLongField(env, thisObject, reqhandleID));
MPI_Status mpi_status;
MPI_Test(&request, &flag, &mpi_status);
if (flag) {
int elements;
MPI_Get_count(&mpi_status, MPI_BYTE, &elements);
(*env)->SetIntField(env, stat, mpjdev_Status_sourceID,
mpi_status.MPI_SOURCE);
(*env)->SetIntField(env, stat, mpjdev_Status_tagID, mpi_status.MPI_TAG);
//seting the number of elements as size of the buffer and in the iwait() method
// we pass it to setSize() method of buffer
// bufferHandle.setSize(status.numEls);
// later numEls is again initialized with getSectionSize() of the buffer
(*env)->SetIntField(env, stat, mpjdev_Status_numEls, elements);
// I am also setting this because we need it in mpi/Status Get_count
// method : TODO remove magic numbers
(*env)->SetIntField(env, stat, mpjdev_Status_countInBytes,
(jint)(elements - 8 - 8)); // 8 is the size of section header
// and 8 recv overhead
//check if dynamic buffer?
jobject mpjbuf;
jobject staticBuffer;
jbyteArray directBuffer;
char *buffer_address = NULL;
/* Get the static Buffer Related things.. */
mpjbuf = (*env)->GetObjectField(env, thisObject,
mpjdev_natmpjdev_NativeRequest_mpjbuf);
staticBuffer = (*env)->GetObjectField(env, mpjbuf,
FID_mpjbuf_Buffer_staticBuffer);
directBuffer = (*env)->GetObjectField(env, staticBuffer,
FID_mpjbuf_NIOBuffer_buffer);
buffer_address = (char *) (*env)->GetDirectBufferAddress(env,
(jobject) directBuffer);
char encoding = 0;
int dbuf_len = -1;
encoding = buffer_address[0];
if (encoding == 1) {
dbuf_len = (((int) (unsigned char) buffer_address[4]) << 24)
| (((int) (unsigned char) buffer_address[5]) << 16)
| (((int) (unsigned char) buffer_address[6]) << 8)
| (((int) (unsigned char) buffer_address[7]));
/* Declarations of Dynamic Buffer */
jboolean isCopy = JNI_TRUE;
jbyteArray darr;
jbyte * dBuffer;
MPI_Status mpi_status_dyn;
MPI_Comm mpi_comm = (MPI_Comm)(*env)->GetLongField(env, thisObject,
mpjdev_natmpjdev_NativeRequest_comHandle);
darr = (*env)->NewByteArray(env, dbuf_len);
// perhaps no need for this - use malloc instead
dBuffer = (*env)->GetByteArrayElements(env, darr, &isCopy);
// This is wrong here Why blocking? we need to set the NativeSendRequest
// wait to not call wait on dynamic buffer, else deadlock arises
MPI_Recv(dBuffer, dbuf_len, MPI_BYTE, mpi_status.MPI_SOURCE,
mpi_status.MPI_TAG + 10001, mpi_comm, &mpi_status_dyn);
(*env)->SetByteArrayRegion(env, darr, 0, dbuf_len, dBuffer);
jmethodID setdbuf = (*env)->GetMethodID(env, CL_mpjbuf_Buffer,
"setDynamicBuffer", "([B)V");
(*env)->CallVoidMethod(env, mpjbuf, setdbuf, darr);
}
return stat;
} else
return NULL;
}
