void *sender_bsend(void *ptr)
{
char buffer[MSGSIZE];
MPI_Bsend(buffer, MSGSIZE, MPI_CHAR, (rank + 1) % size, 0, MPI_COMM_WORLD);
return NULL;
}
void echange_buffer(struct graphe_t * graphe,
char **msg_snd, int *taille_msg_snd,
char **msg_rcv, int *taille_msg_rcv)
{
int rang, nbv, iv, vois, tag, buff_size;
char *buff_mpi;
MPI_Status sta;
MPI_Comm_rank(MPI_COMM_WORLD, &rang);
tag = 1000;
nbv = graphe->nb_voisins[rang];
buff_size = 0;
for( iv = 0 ; iv < nbv ; iv++ )
buff_size += MPI_BSEND_OVERHEAD + taille_msg_snd[iv];
buff_mpi = malloc(buff_size);
MPI_Buffer_attach(buff_mpi, buff_size);
//remaruqe: il faut séparer les 2 boucles
for( iv = 0 ; iv < nbv ; iv++ ) {
vois = graphe->voisins[rang][iv];
MPI_Bsend(msg_snd[iv], taille_msg_snd[iv], MPI_CHAR, vois, tag, MPI_COMM_WORLD);
}
for( iv = 0 ; iv < nbv ; iv++ ) {
vois = graphe->voisins[rang][iv];
MPI_Recv(msg_rcv[iv], taille_msg_rcv[iv], MPI_CHAR, vois, tag, MPI_COMM_WORLD, &sta);
}
//si on appl buffer detach juste après la boucle de bsend ça peut posera des pbs
MPI_Buffer_detach(&buff_mpi, &buff_size);
free(buff_mpi);
}
void Master::updateExchangeLimits(){
int mLength = so.num_threads + 1;
int message[mLength];
MPI_Recv(message, mLength, MPI_INT, s.MPI_SOURCE, IMPORT_LIMITS, MPI_COMM_WORLD, &s);
// Update importLimits
int maxImport = 0;
int sender = message[0];
for(int i = 0 ; i < so.num_threads ; i++){
if(i != sender){
maxImport = std::max(maxImport, message[i+1]);
}
importLimits[sender*so.num_threads + i] = message[i+1];
}
// Do exportLimits change?
for(int from = 0 ; from < so.num_threads ; from++){
int maxImport = 0;
int minImport = 1000;
for(int to = 0 ; to < so.num_threads ; to++){
if(from != to){
maxImport = std::max(importLimits[to*so.num_threads+from], maxImport);
minImport = std::min(importLimits[to*so.num_threads+from], minImport);
}
}
// ExportRule: maxImport + 2?
if(exportLimits[from] != maxImport+2){
//fprintf(stderr, "Changing export rule for %d: %d -> %d\n", from, exportLimits[from], maxImport+2);
int newSize = maxImport+2;
exportLimits[from] = newSize;
MPI_Bsend(&newSize, 1, MPI_INT, from+1, NEW_EXPORT_LIMIT, MPI_COMM_WORLD);
}
}
}
void Slave::sendReport() {
if (FULL_DEBUG) fprintf(stderr, "%d: Forming report\n", thread_no);
static bool firstCall = true;
if(firstCall){
if(engine.decisionLevel() == 0 && so.shareBounds)
exportBounds();
}
Report& r = *((Report*) (int*) report_message);
r.status = status;
if (FULL_DEBUG) fprintf(stderr, "%d: Sending report to master\n", thread_no);
profile_start();
MPI_Bsend((int*) report_message, report_message.size(), MPI_INT, 0, REPORT_TAG, MPI_COMM_WORLD);
profile_end("send result", report_message.size());
if (FULL_DEBUG) fprintf(stderr, "%d: Sent report to master\n", thread_no);
report_message.clear();
report_message.growTo(sizeof(Report)/sizeof(int),0);
unitFound = false;
}
void Master::sendLearnts(int thread_no) {
if (PAR_DEBUG) fprintf(stderr, "Sending learnts to %d\n", thread_no);
vec<int> message(sizeof(Report)/sizeof(int),0);
int num_learnts = 0;
SClause *sc = (SClause*) &global_learnts[lhead[thread_no]];
for ( ; (int*) sc != &global_learnts[global_learnts.size()]; sc = sc->getNext()) {
if (sc->source == thread_no) continue;
sc->pushInVec(message);
num_learnts++;
}
lhead[thread_no] = global_learnts.size();
Report& r = *((Report*) (int*) message);
r.num_learnts = num_learnts;
profile_start();
MPI_Bsend((int*) message, message.size(), MPI_INT, thread_no+1, LEARNTS_TAG, MPI_COMM_WORLD);
last_send_learnts[thread_no] = wallClockTime();
profile_end("send learnts", message.size())
}
void Slave::splitJob() {
vec<int> message;
int num_splits;
//fprintf(stderr, "%d: Split job called, assumptions.size()=%d, DL=%d!\n", thread_no, engine.assumptions.size(), engine.decisionLevel());
profile_start();
MPI_Recv(&num_splits, 1, MPI_INT, 0, STEAL_TAG, MPI_COMM_WORLD, &s);
int max_splits = engine.decisionLevel() - engine.assumptions.size() - 1;
if (num_splits > max_splits) num_splits = max_splits;
if (num_splits < 0) num_splits = 0;
if (FULL_DEBUG) fprintf(stderr, "%d: Splitting %d jobs\n", thread_no, num_splits);
for (int i = 0; i < num_splits; i++) {
engine.assumptions.push(toInt(sat.decLit(engine.assumptions.size()+1)));
sat.incVarUse(engine.assumptions.last()/2);
}
assert(num_splits == 0 || engine.decisionLevel() > engine.assumptions.size());
vec<Lit> ps;
for (int i = 0; i < engine.assumptions.size(); i++) ps.push(toLit(engine.assumptions[i]));
Clause *c = Clause_new(ps);
sat.convertToSClause(*c);
free(c);
message.push(num_splits);
sat.temp_sc->pushInVec(message);
MPI_Bsend((int*) message, message.size(), MPI_INT, 0, SPLIT_TAG, MPI_COMM_WORLD);
profile_end("send split job", message.size());
if (FULL_DEBUG) fprintf(stderr, "%d: Sent %d split job to master\n", thread_no, message[0]);
}
/* called by an actor when the program should exit.
* err_code should be a value that is in the readme so it is possible to track the problem */
void actor_terminate(int err_code)
{
printf("error %d: terminate message being sent\n", err_code);
int stop = ACTORTERMINATE;
MPI_Bsend(&stop, 1, MPI_INT, PROCPOOL, 10, frame_comm);
die();
}
void Communicator::sendDiffusionBlip(const char *out_blip, int other_rank)
{
// any outstanding requests here can deadlock like the grim reaper
site->node->communicator->handleRequests();
long *countp = (long *)out_blip;
unsigned int charcount = calcDiffusionBlipSize(*countp);
if (parameters.outputMPIMessages)
{
site->node->outputcontroller->log(
"Sending diffusion to %d, message size = %u\n",
other_rank, charcount);
site->node->outputcontroller->logfile.flush();
}
// have to use buffered send so I can keep checking for data requests
Communicator::ensure_MPI_send_buffer_size(charcount);
MPI_Bsend((void*)out_blip, charcount, MPI_UNSIGNED_CHAR,
other_rank, DIFFUSION_TAG, MPI_COMM_WORLD);
// if (parameters.outputMPIMessages)
// {
// site->node->outputcontroller->log.form( "[%i] Sent\n", parameters.rank);
// site->node->outputcontroller->log.flush();
// }
}
// MPI_TEST will be executed every this many seconds: so this determines the minimum time taken for every send operation!!
//#define VERBOSE_MPISENDRECV
int MpiNode::relion_MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
{
int result;
double start_time = MPI_Wtime();
#define ONLY_NORMAL_SEND
#ifdef ONLY_NORMAL_SEND
result = MPI_Send(buf, count, datatype, dest, tag, comm);
if (result != MPI_SUCCESS)
{
report_MPI_ERROR(result);
}
#else
// Only use Bsend for larger messages, otherwise use normal send
if (count > 100)
{
int size;
MPI_Pack_size( count, datatype, comm, &size );
char *membuff;
// Allocate memory for the package to be sent
int attach_result = MPI_Buffer_attach( malloc(size + MPI_BSEND_OVERHEAD ), size + MPI_BSEND_OVERHEAD );
if (attach_result != MPI_SUCCESS)
{
report_MPI_ERROR(result);
}
// Actually start sending the message
result = MPI_Bsend(buf, count, datatype, dest, tag, comm);
if (result != MPI_SUCCESS)
{
report_MPI_ERROR(result);
}
// The following will only complete once the message has been successfully sent (i.e. also received on the other side)
int deattach_result = MPI_Buffer_detach( &membuff, &size);
if (deattach_result != MPI_SUCCESS)
{
report_MPI_ERROR(result);
}
}
else
{
result = MPI_Send(buf, count, datatype, dest, tag, comm);
if (result != MPI_SUCCESS)
{
report_MPI_ERROR(result);
}
}
#endif
#ifdef VERBOSE_MPISENDRECV
if (count > 100)
std::cerr <<" relion_MPI_Send: message to " << dest << " of size "<< count << " arrived in " << MPI_Wtime() - start_time << " seconds" << std::endl;
#endif
return result;
}
void SendData(ArgStruct *p)
{
#ifdef BSEND
MPI_Bsend(p->buff, p->bufflen, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD);
#else
MPI_Send(p->buff, p->bufflen, MPI_BYTE, p->prot.nbor, 1, MPI_COMM_WORLD);
#endif
}
void SendRepeat(ArgStruct *p, int rpt)
{
#ifdef BSEND
MPI_Bsend(&rpt, 1, MPI_INT, p->prot.nbor, 2, MPI_COMM_WORLD);
#else
MPI_Send(&rpt, 1, MPI_INT, p->prot.nbor, 2, MPI_COMM_WORLD);
#endif
}
void SendTime(ArgStruct *p, double *t)
{
#ifdef BSEND
MPI_Bsend(t, 1, MPI_DOUBLE, p->prot.nbor, 2, MPI_COMM_WORLD);
#else
MPI_Send(t, 1, MPI_DOUBLE, p->prot.nbor, 2, MPI_COMM_WORLD);
#endif
}
/* called by an actor to create a new actor, floats are sent to tell a new frog its position */
void create_actor(int actor_type, float x, float y)
{
float command[5];
command[0] = ACTORSTART;
command[1] = actor_type;
command[3] = x;
command[4] = y;
MPI_Bsend(command, 5, MPI_FLOAT, PROCPOOL, 0, frame_comm);
}
int main(int argc, char *argv[])
{
MPI_Status status;
MPI_Comm comm, scomm;
int a[10], b[10];
int buf[BUFSIZE], *bptr, bl, i, j, rank, size, color, errs = 0;
MTest_Init(0, 0);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
color = rank % 2;
MPI_Comm_split(MPI_COMM_WORLD, color, rank, &scomm);
MPI_Intercomm_create(scomm, 0, MPI_COMM_WORLD, 1 - color, 52, &comm);
MPI_Comm_rank(comm, &rank);
MPI_Comm_remote_size(comm, &size);
MPI_Buffer_attach(buf, BUFSIZE);
for (j = 0; j < 10; j++) {
for (i = 0; i < 10; i++) {
a[i] = (rank + 10 * j) * size + i;
}
MPI_Bsend(a, 10, MPI_INT, 0, 27 + j, comm);
}
if (rank == 0) {
for (i = 0; i < size; i++) {
for (j = 0; j < 10; j++) {
int k;
status.MPI_TAG = -10;
status.MPI_SOURCE = -20;
MPI_Recv(b, 10, MPI_INT, i, 27 + j, comm, &status);
if (status.MPI_TAG != 27 + j) {
errs++;
printf("Wrong tag = %d\n", status.MPI_TAG);
}
if (status.MPI_SOURCE != i) {
errs++;
printf("Wrong source = %d\n", status.MPI_SOURCE);
}
for (k = 0; k < 10; k++) {
if (b[k] != (i + 10 * j) * size + k) {
errs++;
printf("received b[%d] = %d from %d tag %d\n", k, b[k], i, 27 + j);
}
}
}
}
}
MPI_Buffer_detach(&bptr, &bl);
MPI_Comm_free(&scomm);
MPI_Comm_free(&comm);
MTest_Finalize(errs);
return MTestReturnValue(errs);
}
/* sends a message to the processor with the specified actor, tagged with the recipients id */
void send_msg(int *sendbuf, int count, MPI_Datatype type, int actor_dest, MPI_Comm comm)
{
int tag = actor_dest;
int dest;
dest = find_actor(actor_dest);
MPI_Bsend(sendbuf, count, type, dest, tag, comm);
}
int main (int argc, char *argv[])
{
int numtasks, rank, i, tag=111, dest=1, source=0, count=0;
char data[MSGSIZE];
double start, end, result;
MPI_Status status;
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD, &numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0) {
printf ("mpi_bug5 has started...\n");
if (numtasks > 2)
printf("INFO: Number of tasks= %d. Only using 2 tasks.\n", numtasks);
}
/******************************* Send task **********************************/
if (rank == 0) {
/* Initialize send data */
for(i=0; i<MSGSIZE; i++)
data[i] = 'x';
start = MPI_Wtime();
while (1) {
/*
* BUG: sending way more frequently than receiving, exhausting
* buffer space.
* SOLUTION: use Bsend
*/
MPI_Bsend(data, MSGSIZE, MPI_BYTE, dest, tag, MPI_COMM_WORLD);
count++;
if (count % 10 == 0) {
end = MPI_Wtime();
printf("Count= %d Time= %f sec.\n", count, end-start);
start = MPI_Wtime();
}
}
}
/****************************** Receive task ********************************/
if (rank == 1) {
while (1) {
MPI_Recv(data, MSGSIZE, MPI_BYTE, source, tag, MPI_COMM_WORLD, &status);
/* Do some work - at least more than the send task */
result = 0.0;
for (i=0; i < 1000000; i++)
result = result + (double)random();
}
}
MPI_Finalize();
}
/*called by processpool, stops all actors on a process*/
void kill_all_actors(int dest)
{
int command[3];
command[0] = ACTORDIE;
command[1] = 0;
command[2] = 0;
MPI_Bsend(command, 3, MPI_INT, dest, 10, frame_comm);
}
/**
* vsg_packed_msg_bsend:
* @pm: a #VsgPackedMsg.
* @dst: the destination task id.
* @tag: an integer message tag.
*
* Sends stored message to the specified destination with the specified tag.
*/
void vsg_packed_msg_bsend (VsgPackedMsg *pm, gint dst, gint tag)
{
gint ierr;
_trace_write_msg_send (pm, "bsend", dst, tag);
ierr = MPI_Bsend (pm->buffer, pm->position, MPI_PACKED, dst, tag,
pm->communicator);
if (ierr != MPI_SUCCESS) vsg_mpi_error_output (ierr);
}
void ompi_bsend_f(char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *ierr)
{
int c_ierr;
MPI_Comm c_comm;
MPI_Datatype c_type = MPI_Type_f2c(*datatype);
c_comm = MPI_Comm_f2c (*comm);
c_ierr = MPI_Bsend(OMPI_F2C_BOTTOM(buf), OMPI_FINT_2_INT(*count),
c_type, OMPI_FINT_2_INT(*dest),
OMPI_FINT_2_INT(*tag), c_comm);
if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
}
/* called by the processpool, sends the data to actorcontrol to start a new actor */
void start_actor(int actor_num, int actor_type, int initial, int tot_landcells, int infected)
{
int command[6];
int dest;
dest = find_actor(actor_num);
command[0] = ACTORSTART;
command[1] = actor_type;
command[2] = actor_num;
command[3] = initial;
command[4] = tot_landcells;
command[5] = infected;
MPI_Bsend(command, 6, MPI_INT, dest, 0, frame_comm);
}
void Master::stealJobs() {
int num_splits = 4-2*job_queue.size()/(num_threads-1);
assert(num_splits > 0);
//fprintf(stderr, "Check for busy slave, free slaves = %d\n", num_free_slaves);
profile_start();
int slave = -1;
double cur_time = wallClockTime();
double oldest_job = cur_time;
if(nextSlaveToStealFrom >= 0
&& job_start_time[nextSlaveToStealFrom] < cur_time
&& job_start_time[nextSlaveToStealFrom] != NOT_WORKING
&& (!so.greedyInit || slaveStates[nextSlaveToStealFrom] == NORMAL_SEARCH)){
slave = nextSlaveToStealFrom;
oldest_job = job_start_time[nextSlaveToStealFrom];
nextSlaveToStealFrom = -1;
}
else{
for (int i = 0; i < num_threads; i++) {
if (job_start_time[i] < oldest_job
&& (!so.greedyInit || slaveStates[i] == NORMAL_SEARCH)
) {
slave = i;
oldest_job = job_start_time[i];
}
}
}
if (oldest_job < cur_time-min_job_time) {
assert(slaveStates[slave] == NORMAL_SEARCH);
MPI_Bsend(&num_splits, 1, MPI_INT, slave+1, STEAL_TAG, MPI_COMM_WORLD);
if (PAR_DEBUG) fprintf(stderr, "Steal request sent to %d to steal %d jobs\n", slave, num_splits);
assert(job_start_time[slave] != NOT_WORKING);
job_start_time_backup[slave] = job_start_time[slave];
job_start_time[slave] = DONT_DISTURB;
}
else{
//fprintf(stderr, "Stole no job, oldest=%lf, cur=%lf, min_time=%lf\n", oldest_job, cur_time, min_job_time);
}
profile_end("steal job", 0);
// fprintf(stderr, "Sent steal message to %d\n", slave);
}
/*-------------------------------------------------------------------------------*/
void OneStepCirculation(int step)
{
MPI_Status status;
int n = SIZE * LOCAL_SIZE;
int m = 1;
int sizeOneMsg;
MPI_Pack_size(n, MPI_DOUBLE, MPI_COMM_WORLD, &sizeOneMsg);
int size = m * (sizeOneMsg + MPI_BSEND_OVERHEAD);
double *buf = (double*) malloc(size);
MPI_Buffer_attach(buf, size);
MPI_Bsend(A_Slice, SIZE * LOCAL_SIZE, MPI_DOUBLE, ((Me - 1) + NbPE) % NbPE, 0, MPI_COMM_WORLD);
MPI_Recv(A_Slice, SIZE * LOCAL_SIZE, MPI_DOUBLE, ((Me + 1)) % NbPE, 0, MPI_COMM_WORLD, &status);
MPI_Buffer_detach(&buf, &size);
/******************************** TO DO ******************************************/
}
int main(int argc, char *argv[])
{
int i,bufsize,N=1024*10;
int myrank, nprocs,src, dest,tag;
MPI_Status status;
double A[N],B[N],sum;
double *buf;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
for(i=0;i<N;i++)
A[i]=(double)myrank;
MPI_Pack_size(N,MPI_DOUBLE, MPI_COMM_WORLD, &bufsize);
// MPI_Type_size(MPI_DOUBLE,&bufsize);
// bufsize = N*bufsize;
bufsize = MPI_BSEND_OVERHEAD+bufsize;//定义缓冲方式所需额外开销
buf=(double *)malloc(bufsize);
MPI_Buffer_attach(buf,bufsize);
src = myrank-1;
if(src<0)
src=nprocs-1;
dest = myrank+1;
if(dest>=nprocs)
dest = 0;
tag =111;
MPI_Bsend(A, N, MPI_DOUBLE, dest, tag, MPI_COMM_WORLD);
MPI_Recv(B, N, MPI_DOUBLE, src, tag, MPI_COMM_WORLD, &status);
sum = 0.0;
for (i=0;i<N;i++)
sum =sum +B[i];
printf("Process %d ,values = %f\n",myrank, (double)sum/N);
MPI_Buffer_detach(&buf, &bufsize);
free(buf);
MPI_Finalize();
return 0;
}
int main (int argc, char **argv) {
// initialize MPI
MPI_Init (&argc, &argv);
// we have to remember the number of PEs
int numpes;
MPI_Comm_size (MPI_COMM_WORLD, &numpes);
//for this we need 2 PEs
assert(numpes == 2);
// which rank does this process have?
int myid;
MPI_Comm_rank (MPI_COMM_WORLD, &myid);
// deadlock avoidance: PE 0 sends and recieves using the same function call, PE 1 uses
// its own buffer to avoid blocking on send.
if (myid == 0) {
// send message to 1, wait for message from 1
char buf[10000];
MPI_Status stat;
MPI_Sendrecv_replace (buf, 10000, MPI_CHAR, 1, 0, 1, 0,MPI_COMM_WORLD, &stat);
printf ("0: done\n");
} else {
// send message to 0, wait for message from 0
char buf[10000];
char intermediate_buffer[10000 + MPI_BSEND_OVERHEAD];
MPI_Buffer_attach (&intermediate_buffer, 10000 + MPI_BSEND_OVERHEAD);
MPI_Status stat;
MPI_Bsend (buf, 10000, MPI_CHAR, 0, 0, MPI_COMM_WORLD);
// we can use buf again, as intermediate_buffer will take care of buffering
MPI_Recv (buf, 10000, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &stat);
printf ("1: done\n");
}
MPI_Finalize ();
return EXIT_SUCCESS;
}
static PyObject *bsend_string(PyObject *self, PyObject *args) {
char *s;
int destination, tag, length;
int error;
/* Process the parameters. */
if (!PyArg_ParseTuple(args, "s#ii", &s, &length, &destination, &tag))
return NULL;
/* Call the MPI routine. */
error = MPI_Bsend(s, length, MPI_CHAR, destination, tag, MPI_COMM_WORLD);
if (error != 0) {
rank_raise_mpi_runtime(error, "MPI_Bsend");
return NULL;
}
Py_INCREF(Py_None);
return (Py_None);
}
static PyObject *bsend_array(PyObject *self, PyObject *args) {
PyObject *input;
PyArrayObject *x;
int destination;
int tag;
int count;
MPI_Datatype mpi_type;
int error;
/* Process the parameters. */
if (!PyArg_ParseTuple(args, "Oii", &input, &destination, &tag))
return NULL;
/* Make Numpy array from general sequence type (no cost if already Numpy). */
x = (PyArrayObject *)
PyArray_ContiguousFromObject(input, NPY_NOTYPE, 0, 0);
/* Input check and determination of MPI type */
mpi_type = type_map(x, &count);
if (!mpi_type)
return NULL;
/* Call the MPI routine */
error = MPI_Bsend(x->data, count, mpi_type, destination, tag,
MPI_COMM_WORLD);
Py_DECREF(x);
if (error != 0) {
rank_raise_mpi_runtime(error, "MPI_Bsend");
return NULL;
}
Py_INCREF(Py_None);
return (Py_None);
}
/*! \brief
*
* <pre>
* Purpose
* =======
* Perform local block modifications: lsum[i] -= L_i,k * X[k].
* </pre>
*/
void dlsum_fmod
/************************************************************************/
(
double *lsum, /* Sum of local modifications. */
double *x, /* X array (local) */
double *xk, /* X[k]. */
double *rtemp, /* Result of full matrix-vector multiply. */
int nrhs, /* Number of right-hand sides. */
int knsupc, /* Size of supernode k. */
int_t k, /* The k-th component of X. */
int_t *fmod, /* Modification count for L-solve. */
int_t nlb, /* Number of L blocks. */
int_t lptr, /* Starting position in lsub[*]. */
int_t luptr, /* Starting position in lusup[*]. */
int_t *xsup,
gridinfo_t *grid,
LocalLU_t *Llu,
MPI_Request send_req[],
SuperLUStat_t *stat
)
{
double alpha = 1.0, beta = 0.0;
double *lusup, *lusup1;
double *dest;
int iam, iknsupc, myrow, nbrow, nsupr, nsupr1, p, pi;
int_t i, ii, ik, il, ikcol, irow, j, lb, lk, rel;
int_t *lsub, *lsub1, nlb1, lptr1, luptr1;
int_t *ilsum = Llu->ilsum; /* Starting position of each supernode in lsum. */
int_t *frecv = Llu->frecv;
int_t **fsendx_plist = Llu->fsendx_plist;
MPI_Status status;
int test_flag;
iam = grid->iam;
myrow = MYROW( iam, grid );
lk = LBj( k, grid ); /* Local block number, column-wise. */
lsub = Llu->Lrowind_bc_ptr[lk];
lusup = Llu->Lnzval_bc_ptr[lk];
nsupr = lsub[1];
for (lb = 0; lb < nlb; ++lb) {
ik = lsub[lptr]; /* Global block number, row-wise. */
nbrow = lsub[lptr+1];
#ifdef _CRAY
SGEMM( ftcs2, ftcs2, &nbrow, &nrhs, &knsupc,
&alpha, &lusup[luptr], &nsupr, xk,
&knsupc, &beta, rtemp, &nbrow );
#else
dgemm_( "N", "N", &nbrow, &nrhs, &knsupc,
&alpha, &lusup[luptr], &nsupr, xk,
&knsupc, &beta, rtemp, &nbrow );
#endif
stat->ops[SOLVE] += 2 * nbrow * nrhs * knsupc + nbrow * nrhs;
lk = LBi( ik, grid ); /* Local block number, row-wise. */
iknsupc = SuperSize( ik );
il = LSUM_BLK( lk );
dest = &lsum[il];
lptr += LB_DESCRIPTOR;
rel = xsup[ik]; /* Global row index of block ik. */
for (i = 0; i < nbrow; ++i) {
irow = lsub[lptr++] - rel; /* Relative row. */
RHS_ITERATE(j)
dest[irow + j*iknsupc] -= rtemp[i + j*nbrow];
}
luptr += nbrow;
if ( (--fmod[lk])==0 ) { /* Local accumulation done. */
ikcol = PCOL( ik, grid );
p = PNUM( myrow, ikcol, grid );
if ( iam != p ) {
#ifdef ISEND_IRECV
MPI_Isend( &lsum[il - LSUM_H], iknsupc * nrhs + LSUM_H,
MPI_DOUBLE, p, LSUM, grid->comm,
&send_req[Llu->SolveMsgSent++] );
#else
#ifdef BSEND
MPI_Bsend( &lsum[il - LSUM_H], iknsupc * nrhs + LSUM_H,
MPI_DOUBLE, p, LSUM, grid->comm );
#else
MPI_Send( &lsum[il - LSUM_H], iknsupc * nrhs + LSUM_H,
MPI_DOUBLE, p, LSUM, grid->comm );
#endif
#endif
#if ( DEBUGlevel>=2 )
printf("(%2d) Sent LSUM[%2.0f], size %2d, to P %2d\n",
iam, lsum[il-LSUM_H], iknsupc*nrhs+LSUM_H, p);
#endif
} else { /* Diagonal process: X[i] += lsum[i]. */
ii = X_BLK( lk );
RHS_ITERATE(j)
for (i = 0; i < iknsupc; ++i)
x[i + ii + j*iknsupc] += lsum[i + il + j*iknsupc];
if ( frecv[lk]==0 ) { /* Becomes a leaf node. */
fmod[lk] = -1; /* Do not solve X[k] in the future. */
lk = LBj( ik, grid );/* Local block number, column-wise. */
lsub1 = Llu->Lrowind_bc_ptr[lk];
lusup1 = Llu->Lnzval_bc_ptr[lk];
nsupr1 = lsub1[1];
#ifdef _CRAY
STRSM(ftcs1, ftcs1, ftcs2, ftcs3, &iknsupc, &nrhs, &alpha,
lusup1, &nsupr1, &x[ii], &iknsupc);
#else
dtrsm_("L", "L", "N", "U", &iknsupc, &nrhs, &alpha,
lusup1, &nsupr1, &x[ii], &iknsupc);
#endif
stat->ops[SOLVE] += iknsupc * (iknsupc - 1) * nrhs;
#if ( DEBUGlevel>=2 )
printf("(%2d) Solve X[%2d]\n", iam, ik);
#endif
/*
* Send Xk to process column Pc[k].
*/
for (p = 0; p < grid->nprow; ++p) {
if ( fsendx_plist[lk][p] != EMPTY ) {
pi = PNUM( p, ikcol, grid );
#ifdef ISEND_IRECV
MPI_Isend( &x[ii - XK_H], iknsupc * nrhs + XK_H,
MPI_DOUBLE, pi, Xk, grid->comm,
&send_req[Llu->SolveMsgSent++] );
#else
#ifdef BSEND
MPI_Bsend( &x[ii - XK_H], iknsupc * nrhs + XK_H,
MPI_DOUBLE, pi, Xk, grid->comm );
#else
MPI_Send( &x[ii - XK_H], iknsupc * nrhs + XK_H,
MPI_DOUBLE, pi, Xk, grid->comm );
#endif
#endif
#if ( DEBUGlevel>=2 )
printf("(%2d) Sent X[%2.0f] to P %2d\n",
iam, x[ii-XK_H], pi);
#endif
}
}
/*
* Perform local block modifications.
*/
nlb1 = lsub1[0] - 1;
lptr1 = BC_HEADER + LB_DESCRIPTOR + iknsupc;
luptr1 = iknsupc; /* Skip diagonal block L(I,I). */
dlsum_fmod(lsum, x, &x[ii], rtemp, nrhs, iknsupc, ik,
fmod, nlb1, lptr1, luptr1, xsup,
grid, Llu, send_req, stat);
} /* if frecv[lk] == 0 */
} /* if iam == p */
} /* if fmod[lk] == 0 */
} /* for lb ... */
int main(int argc, char *argv[])
{
int errs = 0;
int rank, size, source, dest;
unsigned char *buf, *bufp;
int minsize = 2;
int i, msgsize, bufsize, outsize;
unsigned char *msg1, *msg2, *msg3;
MPI_Comm comm;
MPI_Status status1, status2, status3;
MTest_Init(&argc, &argv);
/* The following illustrates the use of the routines to
* run through a selection of communicators and datatypes.
* Use subsets of these for tests that do not involve combinations
* of communicators, datatypes, and counts of datatypes */
msgsize = 128 * 1024;
msg1 = (unsigned char *) malloc(3 * msgsize);
msg2 = msg1 + msgsize;
msg3 = msg2 + msgsize;
while (MTestGetIntracommGeneral(&comm, minsize, 1)) {
if (comm == MPI_COMM_NULL)
continue;
/* Determine the sender and receiver */
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &size);
source = 0;
dest = size - 1;
/* Here is the test: The sender */
if (rank == source) {
/* Get a bsend buffer. Make it large enough that the Bsend
* internals will (probably) not use a eager send for the data.
* Have three such messages */
bufsize = 3 * (MPI_BSEND_OVERHEAD + msgsize);
buf = (unsigned char *) malloc(bufsize);
if (!buf) {
fprintf(stderr, "Unable to allocate a buffer of %d bytes\n", bufsize);
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Buffer_attach(buf, bufsize);
/* Initialize the buffers */
for (i = 0; i < msgsize; i++) {
msg1[i] = 0xff ^ (i & 0xff);
msg2[i] = 0xff ^ (3 * i & 0xff);
msg3[i] = 0xff ^ (5 * i & 0xff);
}
/* Initiate the bsends */
MPI_Bsend(msg1, msgsize, MPI_UNSIGNED_CHAR, dest, 0, comm);
MPI_Bsend(msg2, msgsize, MPI_UNSIGNED_CHAR, dest, 0, comm);
MPI_Bsend(msg3, msgsize, MPI_UNSIGNED_CHAR, dest, 0, comm);
/* Synchronize with our partner */
MPI_Sendrecv(NULL, 0, MPI_UNSIGNED_CHAR, dest, 10,
NULL, 0, MPI_UNSIGNED_CHAR, dest, 10, comm, MPI_STATUS_IGNORE);
/* Detach the buffers. There should be pending operations */
MPI_Buffer_detach(&bufp, &outsize);
if (bufp != buf) {
fprintf(stderr, "Wrong buffer returned\n");
errs++;
}
if (outsize != bufsize) {
fprintf(stderr, "Wrong buffer size returned\n");
errs++;
}
}
else if (rank == dest) {
double tstart;
/* Clear the message buffers */
for (i = 0; i < msgsize; i++) {
msg1[i] = 0;
msg2[i] = 0;
msg3[i] = 0;
}
/* Wait for the synchronize */
MPI_Sendrecv(NULL, 0, MPI_UNSIGNED_CHAR, source, 10,
NULL, 0, MPI_UNSIGNED_CHAR, source, 10, comm, MPI_STATUS_IGNORE);
/* Wait 2 seconds */
tstart = MPI_Wtime();
while (MPI_Wtime() - tstart < 2.0);
/* Now receive the messages */
MPI_Recv(msg1, msgsize, MPI_UNSIGNED_CHAR, source, 0, comm, &status1);
MPI_Recv(msg2, msgsize, MPI_UNSIGNED_CHAR, source, 0, comm, &status2);
MPI_Recv(msg3, msgsize, MPI_UNSIGNED_CHAR, source, 0, comm, &status3);
/* Check that we have the correct data */
for (i = 0; i < msgsize; i++) {
if (msg1[i] != (0xff ^ (i & 0xff))) {
if (errs < 10) {
fprintf(stderr, "msg1[%d] = %d\n", i, msg1[i]);
}
errs++;
}
if (msg2[i] != (0xff ^ (3 * i & 0xff))) {
if (errs < 10) {
fprintf(stderr, "msg2[%d] = %d\n", i, msg2[i]);
}
errs++;
}
if (msg3[i] != (0xff ^ (5 * i & 0xff))) {
if (errs < 10) {
fprintf(stderr, "msg2[%d] = %d\n", i, msg2[i]);
}
errs++;
}
}
}
MTestFreeComm(&comm);
}
free(msg1);
MTest_Finalize(errs);
MPI_Finalize();
return 0;
}
bool Foam::mpiOPstreamImpl::write
(
const PstreamImpl::commsTypes commsType,
const int toProcNo,
const char* buf,
const std::streamsize bufSize
)
{
bool transferFailed = true;
if (commsType == PstreamImpl::blocking)
{
transferFailed = MPI_Bsend
(
const_cast<char*>(buf),
bufSize,
MPI_PACKED,
Pstream::procID(toProcNo),
Pstream::msgType(),
MPI_COMM_WORLD
);
}
else if (commsType == PstreamImpl::scheduled)
{
transferFailed = MPI_Send
(
const_cast<char*>(buf),
bufSize,
MPI_PACKED,
Pstream::procID(toProcNo),
Pstream::msgType(),
MPI_COMM_WORLD
);
}
else if (commsType == PstreamImpl::nonBlocking)
{
MPI_Request request;
transferFailed = MPI_Isend
(
const_cast<char*>(buf),
bufSize,
MPI_PACKED,
Pstream::procID(toProcNo),
Pstream::msgType(),
MPI_COMM_WORLD,
&request
);
PstreamGlobals::OPstream_outstandingRequests_.append(request);
}
else
{
FatalErrorIn
(
"mpiOPstreamImpl::write"
"(const int fromProcNo, char* buf, std::streamsize bufSize)"
) << "Unsupported communications type " << commsType
<< Foam::abort(FatalError);
}
return !transferFailed;
}
void main(int argc,char * argv[])
{
const int BUFSIZE=MPI_BSEND_OVERHEAD+4;
unsigned char buf[BUFSIZE];
int rank,ierr,ibufsize,rbuf;
struct MPI_Status status;
ierr=MPI_Init(&argc,&argv);
ierr=MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank%2 == 0) {
if (rank != 0) {
ierr=MPI_Buffer_attach(buf,BUFSIZE);
ierr=MPI_Bsend(&rank,1,MPI_INT,rank+1,5,MPI_COMM_WORLD);
// sending variable rank
ierr=MPI_Buffer_detach(&buf, &BUFSIZE);
}
} else {
if (rank != 1) {
ierr=MPI_Recv(&rbuf,1,MPI_INT,rank-1,5,MPI_COMM_WORLD,&status);
printf("Process %i received %i from process %i\n",rank, rbuf, status.MPI_SOURCE);
}
}
ierr=MPI_Finalize();
}
