int MPI_Reduce(void* sbuff, void* rbuff, int cnt, MPI_Datatype dt,
MPI_Op op, int root, MPI_Comm comm)
{
int g2g = 1;
char call[COLLCHK_SM_STRLEN];
sprintf(call, "REDUCE");
/* Check if init has been called */
g2g = CollChk_is_init();
if(g2g) {
/* check call consistancy */
CollChk_same_call(comm, call);
/* check root consistancy */
CollChk_same_root(comm, root, call);
/* check operation consistancy */
CollChk_same_op(comm, op, call);
/* check datatype signature consistancy */
CollChk_dtype_bcast(comm, dt, cnt, root, call);
/* make the call */
return PMPI_Reduce(sbuff, rbuff, cnt, dt, op, root, comm);
}
else {
/* init not called */
return CollChk_err_han("MPI_Init() has not been called!",
COLLCHK_ERR_NOT_INIT, call, comm);
}
}
/* STUB */
int PMPI_Reduce_scatter ( void *sendbuf, void *recvbuf, int *recvcnts, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm ) {
if ( !recvcnts ) {
_MPI_ERR_ROUTINE (MPI_ERR_COUNT, "MPI_ERR_COUNT : Invalid recv count argument");
MPI_Abort(comm, MPI_ERR_OTHER);
return _MPI_NOT_OK;
}
return PMPI_Reduce(sendbuf,recvbuf,*recvcnts,datatype,op,0,comm);
}
int MPI_Reduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)
{
int ret;
cqueue_t* mycqueue = handle_get_cqueue(comm);
if (mycqueue != NULL)
{
MPI_Request request;
ret = cqueue_ireduce(mycqueue, sendbuf, recvbuf, count, datatype, op, root, comm, &request);
MPI_Wait(&request, MPI_STATUS_IGNORE);
}
else
ret = PMPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm);
return ret;
}
static int
mpiPi_mergept2ptStats ()
{
int matrix_size;
int all_call_count;
double *pt2pt_send_results, *pt2pt_send_local;
int i, x, y, z;
if (mpiPi.do_pt2pt_stats_report)
{
all_call_count = mpiPi_DEF_END - mpiPi_BASE + 1;
matrix_size =
all_call_count * mpiPi.pt2pt_comm_histogram.hist_size *
mpiPi.pt2pt_size_histogram.hist_size;
mpiPi_msg_debug ("matrix_size is %d, histogram data is %d\n",
matrix_size, sizeof (mpiPi.pt2pt_send_stats));
pt2pt_send_local = (double *) malloc (matrix_size * sizeof (double));
pt2pt_send_results = (double *) malloc (matrix_size * sizeof (double));
i = 0;
for (x = 0; x < mpiPi_DEF_END - mpiPi_BASE; x++)
for (y = 0; y < 32; y++)
for (z = 0; z < 32; z++)
pt2pt_send_local[i++] = mpiPi.pt2pt_send_stats[x][y][z];
/* Collect Collective statistic data were used */
PMPI_Reduce (pt2pt_send_local, pt2pt_send_results, matrix_size,
MPI_DOUBLE, MPI_SUM, mpiPi.collectorRank, mpiPi.comm);
i = 0;
for (x = 0; x < mpiPi_DEF_END - mpiPi_BASE; x++)
for (y = 0; y < 32; y++)
for (z = 0; z < 32; z++)
mpiPi.pt2pt_send_stats[x][y][z] = pt2pt_send_results[i++];
free (pt2pt_send_local);
free (pt2pt_send_results);
}
return 1;
}
int
mpiPi_mergeResults ()
{
int ac;
callsite_stats_t **av;
int totalCount = 0;
int maxCount = 0;
int retval = 1, sendval;
/* gather local task data */
h_gather_data (mpiPi.task_callsite_stats, &ac, (void ***) &av);
/* determine size of space necessary on collector */
PMPI_Allreduce (&ac, &totalCount, 1, MPI_INT, MPI_SUM, mpiPi.comm);
PMPI_Reduce (&ac, &maxCount, 1, MPI_INT, MPI_MAX, mpiPi.collectorRank,
mpiPi.comm);
if (totalCount < 1)
{
mpiPi_msg_warn
("Collector found no records to merge. Omitting report.\n");
return 0;
}
/* gather global data at collector */
if (mpiPi.rank == mpiPi.collectorRank)
{
int i;
int ndx = 0;
#ifdef ENABLE_BFD
if (mpiPi.appFullName != NULL)
{
if (open_bfd_executable (mpiPi.appFullName) == 0)
mpiPi.do_lookup = 0;
}
#elif defined(USE_LIBDWARF)
if (mpiPi.appFullName != NULL)
{
if (open_dwarf_executable (mpiPi.appFullName) == 0)
mpiPi.do_lookup = 0;
}
#endif
#if defined(ENABLE_BFD) || defined(USE_LIBDWARF)
else
{
mpiPi_msg_warn ("Failed to open executable\n");
mpiPi.do_lookup = 0;
}
#endif
/* convert data to src line; merge, if nec */
mpiPi.global_callsite_stats = h_open (mpiPi.tableSize,
mpiPi_callsite_stats_src_hashkey,
mpiPi_callsite_stats_src_comparator);
mpiPi.global_callsite_stats_agg = h_open (mpiPi.tableSize,
mpiPi_callsite_stats_src_id_hashkey,
mpiPi_callsite_stats_src_id_comparator);
if (callsite_pc_cache == NULL)
{
callsite_pc_cache = h_open (mpiPi.tableSize,
callsite_pc_cache_hashkey,
callsite_pc_cache_comparator);
}
if (callsite_src_id_cache == NULL)
{
callsite_src_id_cache = h_open (mpiPi.tableSize,
callsite_src_id_cache_hashkey,
callsite_src_id_cache_comparator);
}
/* Try to allocate space for max count of callsite info from all tasks */
mpiPi.rawCallsiteData =
(callsite_stats_t *) calloc (maxCount, sizeof (callsite_stats_t));
if (mpiPi.rawCallsiteData == NULL)
{
mpiPi_msg_warn
("Failed to allocate memory to collect callsite info");
retval = 0;
}
/* Clear global_mpi_time and global_mpi_size before accumulation in mpiPi_insert_callsite_records */
mpiPi.global_mpi_time = 0.0;
mpiPi.global_mpi_size = 0.0;
if (retval == 1)
{
/* Insert collector callsite data into global and task-specific hash tables */
for (ndx = 0; ndx < ac; ndx++)
{
mpiPi_insert_callsite_records (av[ndx]);
}
ndx = 0;
for (i = 1; i < mpiPi.size; i++) /* n-1 */
{
MPI_Status status;
int count;
int j;
/* okay in any order */
PMPI_Probe (MPI_ANY_SOURCE, mpiPi.tag, mpiPi.comm, &status);
PMPI_Get_count (&status, MPI_CHAR, &count);
PMPI_Recv (&(mpiPi.rawCallsiteData[ndx]), count, MPI_CHAR,
status.MPI_SOURCE, mpiPi.tag, mpiPi.comm, &status);
count /= sizeof (callsite_stats_t);
for (j = 0; j < count; j++)
{
mpiPi_insert_callsite_records (&(mpiPi.rawCallsiteData[j]));
}
}
free (mpiPi.rawCallsiteData);
}
}
else
{
int ndx;
char *sbuf = (char *) malloc (ac * sizeof (callsite_stats_t));
for (ndx = 0; ndx < ac; ndx++)
{
bcopy (av[ndx],
&(sbuf[ndx * sizeof (callsite_stats_t)]),
sizeof (callsite_stats_t));
}
PMPI_Send (sbuf, ac * sizeof (callsite_stats_t),
MPI_CHAR, mpiPi.collectorRank, mpiPi.tag, mpiPi.comm);
free (sbuf);
}
if (mpiPi.rank == mpiPi.collectorRank && retval == 1)
{
if (mpiPi.collective_report == 0)
mpiPi_msg_debug
("MEMORY : Allocated for global_callsite_stats : %13ld\n",
h_count (mpiPi.global_callsite_stats) * sizeof (callsite_stats_t));
mpiPi_msg_debug
("MEMORY : Allocated for global_callsite_stats_agg : %13ld\n",
h_count (mpiPi.global_callsite_stats_agg) *
sizeof (callsite_stats_t));
}
/* TODO: need to free all these pointers as well. */
free (av);
if (mpiPi.rank == mpiPi.collectorRank)
{
if (mpiPi.do_lookup == 1)
{
#ifdef ENABLE_BFD
/* clean up */
close_bfd_executable ();
#elif defined(USE_LIBDWARF)
close_dwarf_executable ();
#endif
}
}
/* Quadrics MPI does not appear to support MPI_IN_PLACE */
sendval = retval;
PMPI_Allreduce (&sendval, &retval, 1, MPI_INT, MPI_MIN, mpiPi.comm);
return retval;
}
/* STUB */
int PMPI_Allreduce ( void *sendbuf, void *recvbuf, int count,
MPI_Datatype datatype, MPI_Op op, MPI_Comm comm )
{
_MPI_COVERAGE();
return PMPI_Reduce( sendbuf,recvbuf,count,datatype,op,0,comm);
}
int MPI_Reduce(void *sendbuf, void *recvbuf, int count,
MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)
{
return PMPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm);
}
