static int
mpiPi_mergept2ptHashStats ()
{
int ac;
pt2pt_stats_t **av;
int totalCount = 0;
if (mpiPi.do_pt2pt_detail_report)
{
/* gather local task data */
h_gather_data (mpiPi.task_pt2pt_stats, &ac, (void ***) &av);
/* Make sure we have data to collect, otherwise skip */
PMPI_Allreduce (&ac, &totalCount, 1, MPI_INT, MPI_SUM, mpiPi.comm);
mpiPi_msg_debug("(%d) Merging pt2pt stats: totalCount: %d\n",
mpiPi.rank, totalCount);
if (totalCount < 1)
{
if (mpiPi.rank == mpiPi.collectorRank)
{
mpiPi_msg_warn
("Collector found no records to merge. Omitting report.\n");
}
return 1;
}
/* Gather the ac for all ranks at the root */
if (mpiPi.rank == mpiPi.collectorRank)
{
mpiPi.accumulatedPt2ptCounts = (int*)calloc(mpiPi.size, sizeof(int));
assert(mpiPi.accumulatedPt2ptCounts);
}
PMPI_Gather(&ac, 1, MPI_INT, mpiPi.accumulatedPt2ptCounts,
1, MPI_INT, mpiPi.collectorRank, mpiPi.comm);
/* gather global data at collector */
if (mpiPi.rank == mpiPi.collectorRank)
{
mpiPi_recv_pt2pt_stats(ac,av);
}
else
{
/* Send all pt2pt data to collector */
mpiPi_send_pt2pt_stats(ac,av);
}
}
return 1;
}
/* Aggregate individual MPI call data by iterating through call sites. */
static int
mpiPi_insert_MPI_records ()
{
callsite_stats_t *csp = NULL;
int i, ac;
callsite_stats_t **av;
callsite_stats_t *p;
if (mpiPi.rank == mpiPi.collectorRank)
{
/* Open hash table for MPI call data. */
mpiPi.global_MPI_stats_agg = h_open (mpiPi.tableSize,
mpiPi_callsite_stats_MPI_id_hashkey,
mpiPi_callsite_stats_op_comparator);
/* Get individual call data. */
h_gather_data (mpiPi.global_callsite_stats_agg, &ac, (void ***) &av);
/* Sort by MPI op. */
qsort (av, ac, sizeof (void *), callsite_sort_by_MPI_op);
/* For each call site, add call site info to hash table entry for MPI op, independent of rank. */
for (i = 0; i < ac; i++)
{
p = av[i];
/* Check if there is already an entry for the MPI op. */
if (NULL ==
h_search (mpiPi.global_MPI_stats_agg, p, (void **) &csp))
{
callsite_stats_t *newp = NULL;
newp = (callsite_stats_t *) malloc (sizeof (callsite_stats_t));
memcpy (newp, p, sizeof (callsite_stats_t));
newp->rank = -1;
newp->csid = p->op - mpiPi_BASE;
/* insert new record into global */
h_insert (mpiPi.global_MPI_stats_agg, newp);
}
else
{
mpiPi_merge_individual_callsite_records (csp, p);
}
}
}
return 1;
}
int
mpiPi_mergeResults ()
{
int ac;
callsite_stats_t **av;
int totalCount = 0;
int maxCount = 0;
int retval = 1, sendval;
/* gather local task data */
h_gather_data (mpiPi.task_callsite_stats, &ac, (void ***) &av);
/* determine size of space necessary on collector */
PMPI_Allreduce (&ac, &totalCount, 1, MPI_INT, MPI_SUM, mpiPi.comm);
PMPI_Reduce (&ac, &maxCount, 1, MPI_INT, MPI_MAX, mpiPi.collectorRank,
mpiPi.comm);
if (totalCount < 1)
{
mpiPi_msg_warn
("Collector found no records to merge. Omitting report.\n");
return 0;
}
/* gather global data at collector */
if (mpiPi.rank == mpiPi.collectorRank)
{
int i;
int ndx = 0;
#ifdef ENABLE_BFD
if (mpiPi.appFullName != NULL)
{
if (open_bfd_executable (mpiPi.appFullName) == 0)
mpiPi.do_lookup = 0;
}
#elif defined(USE_LIBDWARF)
if (mpiPi.appFullName != NULL)
{
if (open_dwarf_executable (mpiPi.appFullName) == 0)
mpiPi.do_lookup = 0;
}
#endif
#if defined(ENABLE_BFD) || defined(USE_LIBDWARF)
else
{
mpiPi_msg_warn ("Failed to open executable\n");
mpiPi.do_lookup = 0;
}
#endif
/* convert data to src line; merge, if nec */
mpiPi.global_callsite_stats = h_open (mpiPi.tableSize,
mpiPi_callsite_stats_src_hashkey,
mpiPi_callsite_stats_src_comparator);
mpiPi.global_callsite_stats_agg = h_open (mpiPi.tableSize,
mpiPi_callsite_stats_src_id_hashkey,
mpiPi_callsite_stats_src_id_comparator);
if (callsite_pc_cache == NULL)
{
callsite_pc_cache = h_open (mpiPi.tableSize,
callsite_pc_cache_hashkey,
callsite_pc_cache_comparator);
}
if (callsite_src_id_cache == NULL)
{
callsite_src_id_cache = h_open (mpiPi.tableSize,
callsite_src_id_cache_hashkey,
callsite_src_id_cache_comparator);
}
/* Try to allocate space for max count of callsite info from all tasks */
mpiPi.rawCallsiteData =
(callsite_stats_t *) calloc (maxCount, sizeof (callsite_stats_t));
if (mpiPi.rawCallsiteData == NULL)
{
mpiPi_msg_warn
("Failed to allocate memory to collect callsite info");
retval = 0;
}
/* Clear global_mpi_time and global_mpi_size before accumulation in mpiPi_insert_callsite_records */
mpiPi.global_mpi_time = 0.0;
mpiPi.global_mpi_size = 0.0;
if (retval == 1)
{
/* Insert collector callsite data into global and task-specific hash tables */
for (ndx = 0; ndx < ac; ndx++)
{
mpiPi_insert_callsite_records (av[ndx]);
}
ndx = 0;
for (i = 1; i < mpiPi.size; i++) /* n-1 */
{
MPI_Status status;
int count;
int j;
/* okay in any order */
PMPI_Probe (MPI_ANY_SOURCE, mpiPi.tag, mpiPi.comm, &status);
PMPI_Get_count (&status, MPI_CHAR, &count);
PMPI_Recv (&(mpiPi.rawCallsiteData[ndx]), count, MPI_CHAR,
status.MPI_SOURCE, mpiPi.tag, mpiPi.comm, &status);
count /= sizeof (callsite_stats_t);
for (j = 0; j < count; j++)
{
mpiPi_insert_callsite_records (&(mpiPi.rawCallsiteData[j]));
}
}
free (mpiPi.rawCallsiteData);
}
}
else
{
int ndx;
char *sbuf = (char *) malloc (ac * sizeof (callsite_stats_t));
for (ndx = 0; ndx < ac; ndx++)
{
bcopy (av[ndx],
&(sbuf[ndx * sizeof (callsite_stats_t)]),
sizeof (callsite_stats_t));
}
PMPI_Send (sbuf, ac * sizeof (callsite_stats_t),
MPI_CHAR, mpiPi.collectorRank, mpiPi.tag, mpiPi.comm);
free (sbuf);
}
if (mpiPi.rank == mpiPi.collectorRank && retval == 1)
{
if (mpiPi.collective_report == 0)
mpiPi_msg_debug
("MEMORY : Allocated for global_callsite_stats : %13ld\n",
h_count (mpiPi.global_callsite_stats) * sizeof (callsite_stats_t));
mpiPi_msg_debug
("MEMORY : Allocated for global_callsite_stats_agg : %13ld\n",
h_count (mpiPi.global_callsite_stats_agg) *
sizeof (callsite_stats_t));
}
/* TODO: need to free all these pointers as well. */
free (av);
if (mpiPi.rank == mpiPi.collectorRank)
{
if (mpiPi.do_lookup == 1)
{
#ifdef ENABLE_BFD
/* clean up */
close_bfd_executable ();
#elif defined(USE_LIBDWARF)
close_dwarf_executable ();
#endif
}
}
/* Quadrics MPI does not appear to support MPI_IN_PLACE */
sendval = retval;
PMPI_Allreduce (&sendval, &retval, 1, MPI_INT, MPI_MIN, mpiPi.comm);
return retval;
}
