int
mpiP_find_src_loc (void *i_addr_hex,
char **o_file_str, int *o_lineno, char **o_funct_str)
{
const struct AddrToSourceInfo *addrToSrcMapping = NULL;
const struct FunctionInfo *functionInfo = NULL;
char addr_buf[24];
/* initialize the out parameters */
assert (o_file_str != NULL);
assert (o_lineno != NULL);
assert (o_funct_str != NULL);
*o_file_str = NULL;
*o_lineno = 0;
*o_funct_str = NULL;
/* determine if we have source line info for this address */
addrToSrcMapping = AddrToSourceMap_Find (i_addr_hex);
if (addrToSrcMapping != NULL)
{
*o_file_str = addrToSrcMapping->fileName;
*o_lineno = addrToSrcMapping->lineNumber;
if (mpiPi.baseNames == 0 && *o_file_str != NULL)
{
char *h;
h = strrchr (*o_file_str, '/');
if (h != NULL)
*o_file_str = h + 1;
}
}
else
{
mpiPi_msg_warn ("unable to find source line info for address 0x%p\n",
i_addr_hex);
/*
* the rest of the mpiP code seems to expect that the filename
* will be set to "??" if we are unable to associate it with a file
*/
*o_file_str = "??";
}
/* determine if we have function info for this address */
functionInfo = FunctionMap_Find (i_addr_hex);
if (functionInfo != NULL)
{
*o_funct_str = functionInfo->name;
}
else
{
mpiPi_msg_warn ("unable to find function info for address %s\n",
mpiP_format_address (i_addr_hex, addr_buf));
}
return (((addrToSrcMapping != NULL) || (functionInfo != NULL)) ? 0 : 1);
}
void
mpiPi_generateReport (int report_style)
{
mpiP_TIMER dur;
mpiPi_TIME timer_start, timer_end;
int mergeResult;
mpiPi_GETTIME (&mpiPi.endTime);
if (mpiPi.enabled)
{
dur =
(mpiPi_GETTIMEDIFF (&mpiPi.endTime, &mpiPi.startTime) / 1000000.0);
mpiPi.cumulativeTime += dur;
assert (mpiPi.cumulativeTime >= 0);
mpiPi_GETTIME (&mpiPi.startTime);
}
if (time (&mpiPi.stop_timeofday) == (time_t) - 1)
{
mpiPi_msg_warn ("Could not get time of day from time()\n");
}
/* collect results and publish */
mpiPi_msg_debug0 ("starting collect_basics\n");
mpiPi_GETTIME (&timer_start);
mpiPi_collect_basics ();
mpiPi_GETTIME (&timer_end);
dur = (mpiPi_GETTIMEDIFF (&timer_end, &timer_start) / 1000000.0);
mpiPi_msg_debug0 ("TIMING : collect_basics_time is %12.6f\n", dur);
mpiPi_msg_debug0 ("starting mergeResults\n");
mpiPi_GETTIME (&timer_start);
mergeResult = mpiPi_mergeResults ();
mpiPi_GETTIME (&timer_end);
dur = (mpiPi_GETTIMEDIFF (&timer_end, &timer_start) / 1000000.0);
mpiPi_msg_debug0 ("TIMING : merge time is %12.6f\n", dur);
mpiPi_msg_debug0 ("starting publishResults\n");
if (mergeResult == 1)
{
mpiPi_GETTIME (&timer_start);
if (mpiPi.report_style == mpiPi_style_both)
{
mpiPi_publishResults (mpiPi_style_concise);
mpiPi_publishResults (mpiPi_style_verbose);
}
else
mpiPi_publishResults (report_style);
mpiPi_GETTIME (&timer_end);
dur = (mpiPi_GETTIMEDIFF (&timer_end, &timer_start) / 1000000.0);
mpiPi_msg_debug0 ("TIMING : publish time is %12.6f\n", dur);
}
}
static int
mpiPi_mergept2ptHashStats ()
{
int ac;
pt2pt_stats_t **av;
int totalCount = 0;
if (mpiPi.do_pt2pt_detail_report)
{
/* gather local task data */
h_gather_data (mpiPi.task_pt2pt_stats, &ac, (void ***) &av);
/* Make sure we have data to collect, otherwise skip */
PMPI_Allreduce (&ac, &totalCount, 1, MPI_INT, MPI_SUM, mpiPi.comm);
mpiPi_msg_debug("(%d) Merging pt2pt stats: totalCount: %d\n",
mpiPi.rank, totalCount);
if (totalCount < 1)
{
if (mpiPi.rank == mpiPi.collectorRank)
{
mpiPi_msg_warn
("Collector found no records to merge. Omitting report.\n");
}
return 1;
}
/* Gather the ac for all ranks at the root */
if (mpiPi.rank == mpiPi.collectorRank)
{
mpiPi.accumulatedPt2ptCounts = (int*)calloc(mpiPi.size, sizeof(int));
assert(mpiPi.accumulatedPt2ptCounts);
}
PMPI_Gather(&ac, 1, MPI_INT, mpiPi.accumulatedPt2ptCounts,
1, MPI_INT, mpiPi.collectorRank, mpiPi.comm);
/* gather global data at collector */
if (mpiPi.rank == mpiPi.collectorRank)
{
mpiPi_recv_pt2pt_stats(ac,av);
}
else
{
/* Send all pt2pt data to collector */
mpiPi_send_pt2pt_stats(ac,av);
}
}
return 1;
}
void
mpiPi_publishResults (int report_style)
{
FILE *fp = NULL;
static int printCount = 0;
if (mpiPi.collectorRank == mpiPi.rank)
{
/* Generate output filename, and open */
do
{
printCount++;
snprintf (mpiPi.oFilename, 256, "%s/%s.%d.%d.%d.mpiP",
mpiPi.outputDir, mpiPi.appName, mpiPi.size, mpiPi.procID,
printCount);
}
while (access (mpiPi.oFilename, F_OK) == 0);
fp = fopen (mpiPi.oFilename, "w");
if (fp == NULL)
{
mpiPi_msg_warn ("Could not open [%s], writing to stdout\n",
mpiPi.oFilename);
fp = stdout;
}
else
{
mpiPi_msg ("\n");
mpiPi_msg ("Storing mpiP output in [%s].\n", mpiPi.oFilename);
mpiPi_msg ("\n");
}
}
mpiPi_profile_print (fp, report_style);
PMPI_Barrier (MPI_COMM_WORLD);
if (fp != stdout && fp != NULL)
{
fclose (fp);
}
}
void
mpiPi_getenv ()
{ /* NEED TO ADD SANITY CHECKS HERE */
char *cp = NULL;
char *ep = NULL;
mpiPi.outputDir = ".";
ep = getenv ("MPIP");
mpiPi.envStr = (ep ? strdup (ep) : 0);
optind = 1; /* reset to avoid conflicts if getopt already called */
if (ep != NULL)
{
int c;
extern char *optarg;
int ac = 0;
char *av[64];
char *sep = " \t,";
if (mpiPi.rank == 0)
mpiPi_msg ("Found MPIP environment variable [%s]\n", ep);
av[0] = "JUNK";
for (cp = strtok (ep, sep), ac = 1; (ac < 64) && (NULL != cp); ac++)
{
av[ac] = cp;
/*mpiPi_msg("av[%d] = %s\n", ac, av[ac]); */
cp = strtok (NULL, sep);
}
av[ac] = NULL;
for (; ((c = getopt (ac, av, "cdef:gk:lm:noprs:t:vx:yz")) != EOF);)
{
switch (c)
{
case 'f':
mpiPi.outputDir = optarg;
if (mpiPi.rank == 0)
mpiPi_msg ("Set the output directory to [%s].\n",
mpiPi.outputDir);
break;
case 'g':
mpiPi_debug = 1;
if (mpiPi.rank == 0)
mpiPi_msg ("Enabled mpiPi debug mode.\n");
break;
case 's':
{
int defaultSize = mpiPi.tableSize;
mpiPi.tableSize = atoi (optarg);
if (mpiPi.tableSize < 2)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("-s tablesize invalid %d. Using default.\n",
mpiPi.tableSize);
mpiPi.tableSize = defaultSize;
}
if (mpiPi.tableSize < 128)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("tablesize small %d. Consider making it larger w/ -s.\n",
mpiPi.tableSize);
}
if (mpiPi.rank == 0)
mpiPi_msg
("Set the callsite table size to [%d].\n",
mpiPi.tableSize);
}
break;
case 'k':
{
mpiPi.stackDepth = atoi (optarg);
if (mpiPi.stackDepth < 0)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("-k stackdepth invalid %d. Using 0.\n",
mpiPi.stackDepth);
mpiPi.stackDepth = 0;
mpiPi.print_callsite_detail = 0;
}
if (mpiPi.stackDepth > MPIP_CALLSITE_STACK_DEPTH_MAX)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("stackdepth of %d too large. Using %d.\n",
mpiPi.stackDepth, MPIP_CALLSITE_STACK_DEPTH_MAX);
mpiPi.stackDepth = MPIP_CALLSITE_STACK_DEPTH_MAX;
}
else if (mpiPi.stackDepth > 4)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("stackdepth of %d is large. Consider making it smaller.\n",
mpiPi.stackDepth);
}
// If the stack depth is 0, we are accumulating data
// for each MPI op (i.e. potentially multiple callsites),
// resulting in data that would not be useful for calculating COV.
if (mpiPi.stackDepth == 0)
mpiPi.calcCOV = 0;
if (mpiPi.rank == 0)
mpiPi_msg
("Set the callsite stack traceback depth to [%d].\n",
mpiPi.stackDepth);
}
break;
case 't':
{
int defaultThreshold = mpiPi.reportPrintThreshold;
mpiPi.reportPrintThreshold = atof (optarg);
if (mpiPi.reportPrintThreshold < 0)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("-t report print threshold invalid %g. Using default.\n",
mpiPi.reportPrintThreshold);
mpiPi.reportPrintThreshold = defaultThreshold;
}
if (mpiPi.reportPrintThreshold >= 100)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("report print threshold large %g. Making it default.\n",
mpiPi.reportPrintThreshold);
mpiPi.reportPrintThreshold = defaultThreshold;
}
if (mpiPi.rank == 0)
mpiPi_msg
("Set the report print threshold to [%3.2lf%%].\n",
mpiPi.reportPrintThreshold);
}
break;
case 'o':
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("Disabling MPIP at Init. Code must use Pcontrol to enable.\n");
mpiPi.enabled = 0;
mpiPi.enabledCount = 0;
}
break;
case 'n':
mpiPi.baseNames = 1;
break;
case 'e':
mpiPi.reportFormat = MPIP_REPORT_FLT_FORMAT;
break;
case 'c':
mpiPi.report_style = mpiPi_style_concise;
break;
case 'v':
mpiPi.report_style = mpiPi_style_both;
break;
case 'm':
mpiPi.messageCountThreshold = atoi (optarg);
mpiPi_msg_debug ("Set messageCountThreshold to %d\n",
mpiPi.messageCountThreshold);
break;
case 'x':
if (optarg != NULL)
{
mpiPi.appFullName = strdup (optarg);
mpiPi.av[0] = strdup (optarg);
mpiPi.appName = strdup (GetBaseAppName (mpiPi.appFullName));
mpiPi_msg_debug ("Set mpiPi.appFullName to %s\n",
mpiPi.appFullName);
}
break;
case 'd':
/* Suppress/Activate printing of call site detail based on default. */
mpiPi.print_callsite_detail ^= 1;
break;
case 'l':
/* Use low-memory use approach using MPI collectives
for report generation */
mpiPi.collective_report = 1;
break;
case 'r':
/* Use collector task to aggregate all task data and
generate report */
mpiPi.collective_report = 0;
break;
case 'z':
mpiPi.disable_finalize_report = 1;
break;
case 'y':
mpiPi.do_collective_stats_report = 1;
break;
case 'p':
mpiPi.do_pt2pt_stats_report = 1;
break;
case 'a':
case 'b':
case 'h':
case 'i':
case 'j':
case 'q':
case 'u':
case 'w':
default:
if (mpiPi.rank == 0)
mpiPi_msg_warn
("Option flag (-%c) not recognized. Ignored.\n", c);
break;
}
}
}
if (mpiPi.rank == 0)
mpiPi_msg ("\n");
optind = 1; /* reset to avoid conflicts if getopt called again */
}
int
open_dwarf_executable (char *fileName)
{
int dwStatus = -1;
mpiPi_msg_debug ("enter open_dwarf_executable\n");
if (fileName == NULL)
{
mpiPi_msg_warn ("Executable file name is NULL!\n");
mpiPi_msg_warn
("If this is a Fortran application, you may be using the incorrect mpiP library.\n");
}
/* open the executable */
assert (dwFd == -1);
mpiPi_msg_debug ("opening file %s\n", fileName);
dwFd = open (fileName, O_RDONLY);
if (dwFd == -1)
{
mpiPi_msg_warn ("could not open file %s\n", fileName);
return 0;
}
/* initialize the DWARF library */
assert (dwHandle == NULL);
dwStatus = dwarf_init (dwFd, /* exe file descriptor */
DW_DLC_READ, /* desired access */
NULL, /* error handler */
NULL, /* error argument */
&dwHandle, /* session handle */
&dw_err); /* error object */
if (dwStatus == DW_DLV_ERROR)
{
close (dwFd);
dwFd = -1;
mpiPi_abort ("could not initialize DWARF library : %s\n", dwarf_errmsg(dw_err));
}
if (dwStatus == DW_DLV_NO_ENTRY)
{
mpiPi_msg_warn ("No symbols in the executable\n");
return 0;
}
/* initialize our function and addr-to-source mappings */
AddrToSourceMap_Init ();
FunctionMap_Init ();
/* access symbol info */
while (1)
{
Dwarf_Unsigned nextCompilationUnitHeaderOffset = 0;
Dwarf_Die currCompileUnitDIE = NULL;
Dwarf_Line *lineEntries = NULL;
Dwarf_Signed nLineEntries = 0;
Dwarf_Half currDIETag;
Dwarf_Die currChildDIE = NULL;
Dwarf_Die nextChildDIE = NULL;
Dwarf_Die oldChildDIE = NULL;
/* access next compilation unit header */
dwStatus = dwarf_next_cu_header (dwHandle, NULL, /* cu_header_length */
NULL, /* version_stamp */
NULL, /* abbrev_offset */
NULL, /* address_size */
&nextCompilationUnitHeaderOffset, &dw_err); /* error object */
if (dwStatus != DW_DLV_OK)
{
if (dwStatus != DW_DLV_NO_ENTRY)
{
mpiPi_abort ("failed to access next DWARF cu header : %s\n", dwarf_errmsg(dw_err));
}
break;
}
/* access the first debug info entry (DIE) for this computation unit */
dwStatus = dwarf_siblingof (dwHandle, NULL, /* current DIE */
&currCompileUnitDIE, /* sibling DIE */
&dw_err); /* error object */
if (dwStatus != DW_DLV_OK)
{
mpiPi_abort ("failed to access first DWARF DIE : %s\n", dwarf_errmsg(dw_err));
}
/* get line number information for this compilation
* unit, if available
*/
dwStatus = dwarf_srclines (currCompileUnitDIE,
&lineEntries, &nLineEntries, &dw_err);
if (dwStatus == DW_DLV_OK)
{
unsigned int i;
/*
* Extract and save address-to-source line mapping.
*
* NOTE: At least on the Cray X1, we see line number
* information with the same address mapping to different lines.
* It seems like when there are multiple entries for a given
* address, the last one is the correct one. (Needs verification.)
* If we see multiple entries for a given address, we only
* save the last one.
*/
for (i = 0; i < nLineEntries; i++)
{
Dwarf_Unsigned lineNumber = 0;
Dwarf_Addr lineAddress = 0;
char *lineSourceFile = NULL;
int lineNoStatus, lineAddrStatus, lineSrcFileStatus;
lineNoStatus = dwarf_lineno (lineEntries[i],
&lineNumber,
&dw_err);
if (lineNoStatus != DW_DLV_OK)
mpiPi_msg_debug("Failed to get line number : %s\n", dwarf_errmsg(dw_err));
lineAddrStatus = dwarf_lineaddr (lineEntries[i],
&lineAddress,
&dw_err);
if (lineAddrStatus != DW_DLV_OK)
mpiPi_msg_debug("Failed to get line address : %s\n", dwarf_errmsg(dw_err));
lineSrcFileStatus = dwarf_linesrc (lineEntries[i],
&lineSourceFile,
&dw_err);
if (lineSrcFileStatus != DW_DLV_OK)
mpiPi_msg_debug("Failed to get source file status : %s\n", dwarf_errmsg(dw_err));
if ((lineNoStatus == DW_DLV_OK) &&
(lineAddrStatus == DW_DLV_OK)
&& (lineSrcFileStatus == DW_DLV_OK))
{
int saveCurrentEntry = 0; /* bool */
if (i < (nLineEntries - 1))
{
/*
* We're not on the last line number entry -
* check the address associated with the next
* entry to see if it differs from this one.
* Only save the entry if it the next address
* differs.
*/
Dwarf_Addr nextLineAddress = 0;
int nextLineAddrStatus =
dwarf_lineaddr (lineEntries[i + 1],
&nextLineAddress,
&dw_err);
assert (nextLineAddrStatus == DW_DLV_OK);
if (nextLineAddress != lineAddress)
{
saveCurrentEntry = 1;
}
}
else
{
/* we're on the last line number entry */
saveCurrentEntry = 1;
}
if (saveCurrentEntry)
{
/* save the mapping entry */
AddrToSourceMap_Add (lineAddress, lineSourceFile,
lineNumber);
}
}
if (lineSourceFile != NULL)
{
dwarf_dealloc (dwHandle, lineSourceFile, DW_DLA_STRING);
}
}
/* release the line number info */
for (i = 0; i < nLineEntries; i++)
{
dwarf_dealloc (dwHandle, lineEntries[i], DW_DLA_LINE);
}
dwarf_dealloc (dwHandle, lineEntries, DW_DLA_LIST);
}
else if (dwStatus != DW_DLV_ERROR)
{
/*
* no line information for the current DIE -
* not an error, just unfortunate
*/
}
else
{
mpiPi_abort ("failed to obtain line info for the current DIE : %s\n", dwarf_errmsg(dw_err));
}
/* discover function information for the current compilation unit */
/*
* NOTE: DWARF debug information entries can be organized in
* a hierarchy. However, we presume that the function entries are
* all children of the compilation unit DIE.
*/
dwStatus = dwarf_tag (currCompileUnitDIE, &currDIETag, &dw_err);
assert ((dwStatus == DW_DLV_OK) && (currDIETag == DW_TAG_compile_unit));
/* access the first child DIE of the compile unit DIE */
dwStatus = dwarf_child (currCompileUnitDIE, &currChildDIE, &dw_err);
if (dwStatus == DW_DLV_NO_ENTRY)
{
// On some Cray systems, executables are linked with assembly compile units
// with no functions.
// mpiPi_abort ("no child DIEs of compile unit DIE\n");
}
else if (dwStatus != DW_DLV_OK)
{
mpiPi_abort
("failed to access first child DIE of compile unit DIE\n");
}
while (dwStatus == DW_DLV_OK)
{
/* determine the type of the child DIE */
dwStatus = dwarf_tag (currChildDIE, &currDIETag, &dw_err);
if (dwStatus == DW_DLV_OK)
{
if ((currDIETag == DW_TAG_subprogram) ||
(currDIETag == DW_TAG_entry_point))
{
HandleFunctionDIE (dwHandle, currChildDIE);
}
}
else
{
mpiPi_abort ("unable to determine tag of current child DIE : %s \n", dwarf_errmsg(dw_err));
}
/* advance to the next child DIE */
oldChildDIE = currChildDIE;
dwStatus = dwarf_siblingof (dwHandle,
currChildDIE, &nextChildDIE, &dw_err);
if (dwStatus == DW_DLV_OK)
{
currChildDIE = nextChildDIE;
nextChildDIE = NULL;
}
else if (dwStatus != DW_DLV_NO_ENTRY)
{
mpiPi_abort
("unable to access next child DIE of current compilation unit : %s\n", dwarf_errmsg(dw_err));
}
if (oldChildDIE != NULL)
{
dwarf_dealloc (dwHandle, oldChildDIE, DW_DLA_DIE);
}
}
/* release the current compilation unit DIE */
dwarf_dealloc (dwHandle, currCompileUnitDIE, DW_DLA_DIE);
}
/*
* DWARF address-to-source line information does not give
* address ranges, so we need to patch the address ranges
* in our address-to-source map.
*/
AddrToSourceMap_PatchRanges ();
return 1;
}
int
mpiPi_mergeResults ()
{
int ac;
callsite_stats_t **av;
int totalCount = 0;
int maxCount = 0;
int retval = 1, sendval;
/* gather local task data */
h_gather_data (mpiPi.task_callsite_stats, &ac, (void ***) &av);
/* determine size of space necessary on collector */
PMPI_Allreduce (&ac, &totalCount, 1, MPI_INT, MPI_SUM, mpiPi.comm);
PMPI_Reduce (&ac, &maxCount, 1, MPI_INT, MPI_MAX, mpiPi.collectorRank,
mpiPi.comm);
if (totalCount < 1)
{
mpiPi_msg_warn
("Collector found no records to merge. Omitting report.\n");
return 0;
}
/* gather global data at collector */
if (mpiPi.rank == mpiPi.collectorRank)
{
int i;
int ndx = 0;
#ifdef ENABLE_BFD
if (mpiPi.appFullName != NULL)
{
if (open_bfd_executable (mpiPi.appFullName) == 0)
mpiPi.do_lookup = 0;
}
#elif defined(USE_LIBDWARF)
if (mpiPi.appFullName != NULL)
{
if (open_dwarf_executable (mpiPi.appFullName) == 0)
mpiPi.do_lookup = 0;
}
#endif
#if defined(ENABLE_BFD) || defined(USE_LIBDWARF)
else
{
mpiPi_msg_warn ("Failed to open executable\n");
mpiPi.do_lookup = 0;
}
#endif
/* convert data to src line; merge, if nec */
mpiPi.global_callsite_stats = h_open (mpiPi.tableSize,
mpiPi_callsite_stats_src_hashkey,
mpiPi_callsite_stats_src_comparator);
mpiPi.global_callsite_stats_agg = h_open (mpiPi.tableSize,
mpiPi_callsite_stats_src_id_hashkey,
mpiPi_callsite_stats_src_id_comparator);
if (callsite_pc_cache == NULL)
{
callsite_pc_cache = h_open (mpiPi.tableSize,
callsite_pc_cache_hashkey,
callsite_pc_cache_comparator);
}
if (callsite_src_id_cache == NULL)
{
callsite_src_id_cache = h_open (mpiPi.tableSize,
callsite_src_id_cache_hashkey,
callsite_src_id_cache_comparator);
}
/* Try to allocate space for max count of callsite info from all tasks */
mpiPi.rawCallsiteData =
(callsite_stats_t *) calloc (maxCount, sizeof (callsite_stats_t));
if (mpiPi.rawCallsiteData == NULL)
{
mpiPi_msg_warn
("Failed to allocate memory to collect callsite info");
retval = 0;
}
/* Clear global_mpi_time and global_mpi_size before accumulation in mpiPi_insert_callsite_records */
mpiPi.global_mpi_time = 0.0;
mpiPi.global_mpi_size = 0.0;
if (retval == 1)
{
/* Insert collector callsite data into global and task-specific hash tables */
for (ndx = 0; ndx < ac; ndx++)
{
mpiPi_insert_callsite_records (av[ndx]);
}
ndx = 0;
for (i = 1; i < mpiPi.size; i++) /* n-1 */
{
MPI_Status status;
int count;
int j;
/* okay in any order */
PMPI_Probe (MPI_ANY_SOURCE, mpiPi.tag, mpiPi.comm, &status);
PMPI_Get_count (&status, MPI_CHAR, &count);
PMPI_Recv (&(mpiPi.rawCallsiteData[ndx]), count, MPI_CHAR,
status.MPI_SOURCE, mpiPi.tag, mpiPi.comm, &status);
count /= sizeof (callsite_stats_t);
for (j = 0; j < count; j++)
{
mpiPi_insert_callsite_records (&(mpiPi.rawCallsiteData[j]));
}
}
free (mpiPi.rawCallsiteData);
}
}
else
{
int ndx;
char *sbuf = (char *) malloc (ac * sizeof (callsite_stats_t));
for (ndx = 0; ndx < ac; ndx++)
{
bcopy (av[ndx],
&(sbuf[ndx * sizeof (callsite_stats_t)]),
sizeof (callsite_stats_t));
}
PMPI_Send (sbuf, ac * sizeof (callsite_stats_t),
MPI_CHAR, mpiPi.collectorRank, mpiPi.tag, mpiPi.comm);
free (sbuf);
}
if (mpiPi.rank == mpiPi.collectorRank && retval == 1)
{
if (mpiPi.collective_report == 0)
mpiPi_msg_debug
("MEMORY : Allocated for global_callsite_stats : %13ld\n",
h_count (mpiPi.global_callsite_stats) * sizeof (callsite_stats_t));
mpiPi_msg_debug
("MEMORY : Allocated for global_callsite_stats_agg : %13ld\n",
h_count (mpiPi.global_callsite_stats_agg) *
sizeof (callsite_stats_t));
}
/* TODO: need to free all these pointers as well. */
free (av);
if (mpiPi.rank == mpiPi.collectorRank)
{
if (mpiPi.do_lookup == 1)
{
#ifdef ENABLE_BFD
/* clean up */
close_bfd_executable ();
#elif defined(USE_LIBDWARF)
close_dwarf_executable ();
#endif
}
}
/* Quadrics MPI does not appear to support MPI_IN_PLACE */
sendval = retval;
PMPI_Allreduce (&sendval, &retval, 1, MPI_INT, MPI_MIN, mpiPi.comm);
return retval;
}
/* task level init
- executed by each MPI task only once immediately after MPI_Init
*/
void
mpiPi_init (char *appName)
{
if (time (&mpiPi.start_timeofday) == (time_t) - 1)
{
mpiPi_msg_warn ("Could not get time of day from time()\n");
}
mpiPi.toolname = "mpiP";
mpiPi.comm = MPI_COMM_WORLD;
mpiPi.tag = 9821;
mpiPi.procID = getpid ();
mpiPi.appName = strdup (appName);
PMPI_Comm_rank (mpiPi.comm, &mpiPi.rank);
PMPI_Comm_size (mpiPi.comm, &mpiPi.size);
PMPI_Get_processor_name (mpiPi.hostname, &mpiPi.hostnamelen);
mpiPi.stdout_ = stdout;
mpiPi.stderr_ = stderr;
mpiPi.lookup = mpiPi_lookup;
mpiPi.enabled = 1;
mpiPi.enabledCount = 1;
mpiPi.cumulativeTime = 0.0;
mpiPi.global_app_time = 0.0;
mpiPi.global_mpi_time = 0.0;
mpiPi.global_mpi_size = 0.0;
mpiPi.global_mpi_io = 0.0;
mpiPi.global_mpi_msize_threshold_count = 0;
mpiPi.global_mpi_sent_count = 0;
mpiPi.global_time_callsite_count = 0;
mpiPi.global_task_info = NULL;
char tmpfilename[64];
sprintf(tmpfilename,"%d.trace\0",mpiPi.rank);
mpiPi.recfile = fopen(tmpfilename,"wb");
printf("Open Rec File %s !\n", tmpfilename);
/* set some defaults values */
mpiPi.collectorRank = 0;
mpiPi.tableSize = 256;
mpiPi.stackDepth = 1; /* the value 2 includes parent wrapper function */
mpiPi.reportPrintThreshold = 0.0;
mpiPi.baseNames = 0;
mpiPi.reportFormat = MPIP_REPORT_SCI_FORMAT;
mpiPi.calcCOV = 1;
mpiPi.inAPIrtb = 0;
mpiPi.do_lookup = 1;
mpiPi.messageCountThreshold = -1;
mpiPi.report_style = mpiPi_style_verbose;
mpiPi.print_callsite_detail = 1;
#ifdef COLLECTIVE_REPORT_DEFAULT
mpiPi.collective_report = 1;
#else
mpiPi.collective_report = 0;
#endif
mpiPi_getenv ();
mpiPi.task_callsite_stats =
h_open (mpiPi.tableSize, mpiPi_callsite_stats_pc_hashkey,
mpiPi_callsite_stats_pc_comparator);
/* -- welcome msg only collector */
if (mpiPi.collectorRank == mpiPi.rank)
{
mpiPi_msg ("\n");
mpiPi_msg ("%s V%d.%d.%d (Build %s/%s)\n", mpiPi.toolname, mpiPi_vmajor,
mpiPi_vminor, mpiPi_vpatch, mpiPi_vdate, mpiPi_vtime);
mpiPi_msg ("Direct questions and errors to %s\n", MPIP_HELP_LIST);
mpiPi_msg ("\n");
}
mpiPi_msg_debug ("appName is %s\n", appName);
mpiPi_msg_debug ("successful init on %d, %s\n", mpiPi.rank, mpiPi.hostname);
if (mpiPi.enabled)
{
mpiPi_GETTIME (&mpiPi.startTime);
}
return;
}
void
mpiPi_recv_pt2pt_stats(int ac, pt2pt_stats_t** av)
{
int i;
int pt2pt_size = sizeof(pt2pt_stats_t);
int nsenders = 0;
/* Count number of senders, receiver will wait for them */
/* i = 0 is copied locally */
for(i = 1; i < mpiPi.size; i++)
{
if (mpiPi.accumulatedPt2ptCounts[i])
nsenders++;
}
mpiPi_msg_debug("(%d) Waiting for %d senders\n",mpiPi.rank,nsenders);
/* Allocate a pointer for each rank */
mpiPi.accumulatedPt2ptData = (pt2pt_stats_t **) calloc (mpiPi.size, sizeof(pt2pt_stats_t*));
if (mpiPi.accumulatedPt2ptData == NULL)
{
mpiPi_msg_warn
("Failed to allocate memory to collect point to point info");
assert(0);
}
/* Copy Data for collector rank */
if (ac)
{
mpiPi.accumulatedPt2ptData[mpiPi.rank] = *av;
}
i = 0;
/* Insert pt2pt data into aggregate array indexed by rank */
while(i < nsenders)
{
MPI_Status status;
int count;
pt2pt_stats_t* ptp;
unsigned src_rank;
/* okay in any order */
PMPI_Probe (MPI_ANY_SOURCE, mpiPi.tag, mpiPi.comm, &status);
PMPI_Get_count (&status, MPI_CHAR, &count);
src_rank = status.MPI_SOURCE;
/* Allocate space for count number of pt2pt_stat_t structs */
ptp = (pt2pt_stats_t*) calloc(count, pt2pt_size);
mpiPi_msg_debug("(%d): Receiving %d bytes in pt2pt records from %d...\n",
mpiPi.rank, count, src_rank);
PMPI_Recv (ptp, count, MPI_CHAR, src_rank,
mpiPi.tag, mpiPi.comm, &status);
mpiPi_msg_debug("(%d): Received\n",mpiPi.rank);
count /= pt2pt_size;
assert(src_rank < mpiPi.size);
assert(mpiPi.accumulatedPt2ptCounts[src_rank] == count);
mpiPi.accumulatedPt2ptData[src_rank] = ptp;
i++;
}
}
void
mpiPi_init (char *appName)
{
if (time (&mpiPi.start_timeofday) == (time_t) - 1)
{
mpiPi_msg_warn ("Could not get time of day from time()\n");
}
mpiPi.toolname = "mpiP";
mpiPi.comm = MPI_COMM_WORLD;
mpiPi.tag = 9821;
mpiPi.procID = getpid ();
mpiPi.appName = strdup (appName);
PMPI_Comm_rank (mpiPi.comm, &mpiPi.rank);
PMPI_Comm_size (mpiPi.comm, &mpiPi.size);
PMPI_Get_processor_name (mpiPi.hostname, &mpiPi.hostnamelen);
mpiPi.stdout_ = stdout;
mpiPi.stderr_ = stderr;
mpiPi.lookup = mpiPi_lookup;
mpiPi.enabled = 1;
mpiPi.enabledCount = 1;
mpiPi.cumulativeTime = 0.0;
mpiPi.global_app_time = 0.0;
mpiPi.global_mpi_time = 0.0;
mpiPi.global_mpi_size = 0.0;
mpiPi.global_mpi_io = 0.0;
mpiPi.global_mpi_rma = 0.0;
mpiPi.global_mpi_msize_threshold_count = 0;
mpiPi.global_mpi_sent_count = 0;
mpiPi.global_time_callsite_count = 0;
mpiPi.global_task_hostnames = NULL;
mpiPi.global_task_app_time = NULL;
mpiPi.global_task_mpi_time = NULL;
/* set some defaults values */
mpiPi.collectorRank = 0;
mpiPi.tableSize = 256;
mpiPi.reportPrintThreshold = 0.0;
mpiPi.baseNames = 0;
mpiPi.reportFormat = MPIP_REPORT_SCI_FORMAT;
mpiPi.calcCOV = 1;
mpiPi.inAPIrtb = 0;
mpiPi.do_lookup = 1;
mpiPi.messageCountThreshold = -1;
if (DEFAULT_REPORT_FORMAT == mpiPi_style_concise)
{
mpiPi.report_style = mpiPi_style_concise;
mpiPi.stackDepth = 0;
mpiPi.print_callsite_detail = 0;
}
else // verbose default
{
mpiPi.report_style = mpiPi_style_verbose;
mpiPi.stackDepth = 1;
mpiPi.print_callsite_detail = 1;
}
#ifdef COLLECTIVE_REPORT_DEFAULT
mpiPi.collective_report = 1;
#else
mpiPi.collective_report = 0;
#endif
mpiPi.disable_finalize_report = 0;
mpiPi.do_collective_stats_report = 0;
mpiPi.do_pt2pt_stats_report = 0;
#ifdef SO_LOOKUP
mpiPi.so_info = NULL;
#endif
mpiPi.do_pt2pt_detail_report = 0;
mpiPi_getenv ();
mpiPi.task_callsite_stats =
h_open (mpiPi.tableSize, mpiPi_callsite_stats_pc_hashkey,
mpiPi_callsite_stats_pc_comparator);
mpiPi.accumulatedPt2ptCounts = NULL;
mpiPi.accumulatedPt2ptData = NULL;
if (mpiPi.do_collective_stats_report == 1)
{
init_histogram (&mpiPi.coll_comm_histogram, 7, 32, NULL);
init_histogram (&mpiPi.coll_size_histogram, 7, 32, NULL);
}
if (mpiPi.do_pt2pt_stats_report == 1)
{
init_histogram (&mpiPi.pt2pt_comm_histogram, 7, 32, NULL);
init_histogram (&mpiPi.pt2pt_size_histogram, 7, 32, NULL);
if(mpiPi.do_pt2pt_detail_report == 1)
{
mpiPi.task_pt2pt_stats =
h_open(mpiPi.tableSize, mpiPi_pt2pt_stats_rank_hashkey,
mpiPi_pt2pt_stats_rank_comparator);
}
}
/* -- welcome msg only collector */
if (mpiPi.collectorRank == mpiPi.rank)
{
mpiPi_msg ("");
mpiPi_msg ("%s V%d.%d.%d (Build %s/%s)\n", mpiPi.toolname, mpiPi_vmajor,
mpiPi_vminor, mpiPi_vpatch, mpiPi_vdate, mpiPi_vtime);
mpiPi_msg ("Direct questions and errors to %s\n", MPIP_HELP_LIST);
mpiPi_msg ("\n");
}
mpiPi_msg_debug ("appName is %s\n", appName);
mpiPi_msg_debug ("sizeof(callsite_stats_t) is %d\n",
sizeof (callsite_stats_t));
mpiPi_msg_debug ("successful init on %d, %s\n", mpiPi.rank, mpiPi.hostname);
if (mpiPi.enabled)
{
mpiPi_GETTIME (&mpiPi.startTime);
}
return;
}
