void
mpiPi_publishResults (int report_style)
{
FILE *fp = NULL;
static int printCount = 0;
if (mpiPi.collectorRank == mpiPi.rank)
{
/* Generate output filename, and open */
do
{
printCount++;
snprintf (mpiPi.oFilename, 256, "%s/%s.%d.%d.%d.mpiP",
mpiPi.outputDir, mpiPi.appName, mpiPi.size, mpiPi.procID,
printCount);
}
while (access (mpiPi.oFilename, F_OK) == 0);
fp = fopen (mpiPi.oFilename, "w");
if (fp == NULL)
{
mpiPi_msg_warn ("Could not open [%s], writing to stdout\n",
mpiPi.oFilename);
fp = stdout;
}
else
{
mpiPi_msg ("\n");
mpiPi_msg ("Storing mpiP output in [%s].\n", mpiPi.oFilename);
mpiPi_msg ("\n");
}
}
mpiPi_profile_print (fp, report_style);
PMPI_Barrier (MPI_COMM_WORLD);
if (fp != stdout && fp != NULL)
{
fclose (fp);
}
}
void
mpiPi_getenv ()
{ /* NEED TO ADD SANITY CHECKS HERE */
char *cp = NULL;
char *ep = NULL;
mpiPi.outputDir = ".";
ep = getenv ("MPIP");
mpiPi.envStr = (ep ? strdup (ep) : 0);
optind = 1; /* reset to avoid conflicts if getopt already called */
if (ep != NULL)
{
int c;
extern char *optarg;
int ac = 0;
char *av[64];
char *sep = " \t,";
if (mpiPi.rank == 0)
mpiPi_msg ("Found MPIP environment variable [%s]\n", ep);
av[0] = "JUNK";
for (cp = strtok (ep, sep), ac = 1; (ac < 64) && (NULL != cp); ac++)
{
av[ac] = cp;
/*mpiPi_msg("av[%d] = %s\n", ac, av[ac]); */
cp = strtok (NULL, sep);
}
av[ac] = NULL;
for (; ((c = getopt (ac, av, "cdef:gk:lm:noprs:t:vx:yz")) != EOF);)
{
switch (c)
{
case 'f':
mpiPi.outputDir = optarg;
if (mpiPi.rank == 0)
mpiPi_msg ("Set the output directory to [%s].\n",
mpiPi.outputDir);
break;
case 'g':
mpiPi_debug = 1;
if (mpiPi.rank == 0)
mpiPi_msg ("Enabled mpiPi debug mode.\n");
break;
case 's':
{
int defaultSize = mpiPi.tableSize;
mpiPi.tableSize = atoi (optarg);
if (mpiPi.tableSize < 2)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("-s tablesize invalid %d. Using default.\n",
mpiPi.tableSize);
mpiPi.tableSize = defaultSize;
}
if (mpiPi.tableSize < 128)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("tablesize small %d. Consider making it larger w/ -s.\n",
mpiPi.tableSize);
}
if (mpiPi.rank == 0)
mpiPi_msg
("Set the callsite table size to [%d].\n",
mpiPi.tableSize);
}
break;
case 'k':
{
mpiPi.stackDepth = atoi (optarg);
if (mpiPi.stackDepth < 0)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("-k stackdepth invalid %d. Using 0.\n",
mpiPi.stackDepth);
mpiPi.stackDepth = 0;
mpiPi.print_callsite_detail = 0;
}
if (mpiPi.stackDepth > MPIP_CALLSITE_STACK_DEPTH_MAX)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("stackdepth of %d too large. Using %d.\n",
mpiPi.stackDepth, MPIP_CALLSITE_STACK_DEPTH_MAX);
mpiPi.stackDepth = MPIP_CALLSITE_STACK_DEPTH_MAX;
}
else if (mpiPi.stackDepth > 4)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("stackdepth of %d is large. Consider making it smaller.\n",
mpiPi.stackDepth);
}
// If the stack depth is 0, we are accumulating data
// for each MPI op (i.e. potentially multiple callsites),
// resulting in data that would not be useful for calculating COV.
if (mpiPi.stackDepth == 0)
mpiPi.calcCOV = 0;
if (mpiPi.rank == 0)
mpiPi_msg
("Set the callsite stack traceback depth to [%d].\n",
mpiPi.stackDepth);
}
break;
case 't':
{
int defaultThreshold = mpiPi.reportPrintThreshold;
mpiPi.reportPrintThreshold = atof (optarg);
if (mpiPi.reportPrintThreshold < 0)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("-t report print threshold invalid %g. Using default.\n",
mpiPi.reportPrintThreshold);
mpiPi.reportPrintThreshold = defaultThreshold;
}
if (mpiPi.reportPrintThreshold >= 100)
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("report print threshold large %g. Making it default.\n",
mpiPi.reportPrintThreshold);
mpiPi.reportPrintThreshold = defaultThreshold;
}
if (mpiPi.rank == 0)
mpiPi_msg
("Set the report print threshold to [%3.2lf%%].\n",
mpiPi.reportPrintThreshold);
}
break;
case 'o':
{
if (mpiPi.rank == 0)
mpiPi_msg_warn
("Disabling MPIP at Init. Code must use Pcontrol to enable.\n");
mpiPi.enabled = 0;
mpiPi.enabledCount = 0;
}
break;
case 'n':
mpiPi.baseNames = 1;
break;
case 'e':
mpiPi.reportFormat = MPIP_REPORT_FLT_FORMAT;
break;
case 'c':
mpiPi.report_style = mpiPi_style_concise;
break;
case 'v':
mpiPi.report_style = mpiPi_style_both;
break;
case 'm':
mpiPi.messageCountThreshold = atoi (optarg);
mpiPi_msg_debug ("Set messageCountThreshold to %d\n",
mpiPi.messageCountThreshold);
break;
case 'x':
if (optarg != NULL)
{
mpiPi.appFullName = strdup (optarg);
mpiPi.av[0] = strdup (optarg);
mpiPi.appName = strdup (GetBaseAppName (mpiPi.appFullName));
mpiPi_msg_debug ("Set mpiPi.appFullName to %s\n",
mpiPi.appFullName);
}
break;
case 'd':
/* Suppress/Activate printing of call site detail based on default. */
mpiPi.print_callsite_detail ^= 1;
break;
case 'l':
/* Use low-memory use approach using MPI collectives
for report generation */
mpiPi.collective_report = 1;
break;
case 'r':
/* Use collector task to aggregate all task data and
generate report */
mpiPi.collective_report = 0;
break;
case 'z':
mpiPi.disable_finalize_report = 1;
break;
case 'y':
mpiPi.do_collective_stats_report = 1;
break;
case 'p':
mpiPi.do_pt2pt_stats_report = 1;
break;
case 'a':
case 'b':
case 'h':
case 'i':
case 'j':
case 'q':
case 'u':
case 'w':
default:
if (mpiPi.rank == 0)
mpiPi_msg_warn
("Option flag (-%c) not recognized. Ignored.\n", c);
break;
}
}
}
if (mpiPi.rank == 0)
mpiPi_msg ("\n");
optind = 1; /* reset to avoid conflicts if getopt called again */
}
/* task level init
- executed by each MPI task only once immediately after MPI_Init
*/
void
mpiPi_init (char *appName)
{
if (time (&mpiPi.start_timeofday) == (time_t) - 1)
{
mpiPi_msg_warn ("Could not get time of day from time()\n");
}
mpiPi.toolname = "mpiP";
mpiPi.comm = MPI_COMM_WORLD;
mpiPi.tag = 9821;
mpiPi.procID = getpid ();
mpiPi.appName = strdup (appName);
PMPI_Comm_rank (mpiPi.comm, &mpiPi.rank);
PMPI_Comm_size (mpiPi.comm, &mpiPi.size);
PMPI_Get_processor_name (mpiPi.hostname, &mpiPi.hostnamelen);
mpiPi.stdout_ = stdout;
mpiPi.stderr_ = stderr;
mpiPi.lookup = mpiPi_lookup;
mpiPi.enabled = 1;
mpiPi.enabledCount = 1;
mpiPi.cumulativeTime = 0.0;
mpiPi.global_app_time = 0.0;
mpiPi.global_mpi_time = 0.0;
mpiPi.global_mpi_size = 0.0;
mpiPi.global_mpi_io = 0.0;
mpiPi.global_mpi_msize_threshold_count = 0;
mpiPi.global_mpi_sent_count = 0;
mpiPi.global_time_callsite_count = 0;
mpiPi.global_task_info = NULL;
char tmpfilename[64];
sprintf(tmpfilename,"%d.trace\0",mpiPi.rank);
mpiPi.recfile = fopen(tmpfilename,"wb");
printf("Open Rec File %s !\n", tmpfilename);
/* set some defaults values */
mpiPi.collectorRank = 0;
mpiPi.tableSize = 256;
mpiPi.stackDepth = 1; /* the value 2 includes parent wrapper function */
mpiPi.reportPrintThreshold = 0.0;
mpiPi.baseNames = 0;
mpiPi.reportFormat = MPIP_REPORT_SCI_FORMAT;
mpiPi.calcCOV = 1;
mpiPi.inAPIrtb = 0;
mpiPi.do_lookup = 1;
mpiPi.messageCountThreshold = -1;
mpiPi.report_style = mpiPi_style_verbose;
mpiPi.print_callsite_detail = 1;
#ifdef COLLECTIVE_REPORT_DEFAULT
mpiPi.collective_report = 1;
#else
mpiPi.collective_report = 0;
#endif
mpiPi_getenv ();
mpiPi.task_callsite_stats =
h_open (mpiPi.tableSize, mpiPi_callsite_stats_pc_hashkey,
mpiPi_callsite_stats_pc_comparator);
/* -- welcome msg only collector */
if (mpiPi.collectorRank == mpiPi.rank)
{
mpiPi_msg ("\n");
mpiPi_msg ("%s V%d.%d.%d (Build %s/%s)\n", mpiPi.toolname, mpiPi_vmajor,
mpiPi_vminor, mpiPi_vpatch, mpiPi_vdate, mpiPi_vtime);
mpiPi_msg ("Direct questions and errors to %s\n", MPIP_HELP_LIST);
mpiPi_msg ("\n");
}
mpiPi_msg_debug ("appName is %s\n", appName);
mpiPi_msg_debug ("successful init on %d, %s\n", mpiPi.rank, mpiPi.hostname);
if (mpiPi.enabled)
{
mpiPi_GETTIME (&mpiPi.startTime);
}
return;
}
void
mpiPi_init (char *appName)
{
if (time (&mpiPi.start_timeofday) == (time_t) - 1)
{
mpiPi_msg_warn ("Could not get time of day from time()\n");
}
mpiPi.toolname = "mpiP";
mpiPi.comm = MPI_COMM_WORLD;
mpiPi.tag = 9821;
mpiPi.procID = getpid ();
mpiPi.appName = strdup (appName);
PMPI_Comm_rank (mpiPi.comm, &mpiPi.rank);
PMPI_Comm_size (mpiPi.comm, &mpiPi.size);
PMPI_Get_processor_name (mpiPi.hostname, &mpiPi.hostnamelen);
mpiPi.stdout_ = stdout;
mpiPi.stderr_ = stderr;
mpiPi.lookup = mpiPi_lookup;
mpiPi.enabled = 1;
mpiPi.enabledCount = 1;
mpiPi.cumulativeTime = 0.0;
mpiPi.global_app_time = 0.0;
mpiPi.global_mpi_time = 0.0;
mpiPi.global_mpi_size = 0.0;
mpiPi.global_mpi_io = 0.0;
mpiPi.global_mpi_rma = 0.0;
mpiPi.global_mpi_msize_threshold_count = 0;
mpiPi.global_mpi_sent_count = 0;
mpiPi.global_time_callsite_count = 0;
mpiPi.global_task_hostnames = NULL;
mpiPi.global_task_app_time = NULL;
mpiPi.global_task_mpi_time = NULL;
/* set some defaults values */
mpiPi.collectorRank = 0;
mpiPi.tableSize = 256;
mpiPi.reportPrintThreshold = 0.0;
mpiPi.baseNames = 0;
mpiPi.reportFormat = MPIP_REPORT_SCI_FORMAT;
mpiPi.calcCOV = 1;
mpiPi.inAPIrtb = 0;
mpiPi.do_lookup = 1;
mpiPi.messageCountThreshold = -1;
if (DEFAULT_REPORT_FORMAT == mpiPi_style_concise)
{
mpiPi.report_style = mpiPi_style_concise;
mpiPi.stackDepth = 0;
mpiPi.print_callsite_detail = 0;
}
else // verbose default
{
mpiPi.report_style = mpiPi_style_verbose;
mpiPi.stackDepth = 1;
mpiPi.print_callsite_detail = 1;
}
#ifdef COLLECTIVE_REPORT_DEFAULT
mpiPi.collective_report = 1;
#else
mpiPi.collective_report = 0;
#endif
mpiPi.disable_finalize_report = 0;
mpiPi.do_collective_stats_report = 0;
mpiPi.do_pt2pt_stats_report = 0;
#ifdef SO_LOOKUP
mpiPi.so_info = NULL;
#endif
mpiPi.do_pt2pt_detail_report = 0;
mpiPi_getenv ();
mpiPi.task_callsite_stats =
h_open (mpiPi.tableSize, mpiPi_callsite_stats_pc_hashkey,
mpiPi_callsite_stats_pc_comparator);
mpiPi.accumulatedPt2ptCounts = NULL;
mpiPi.accumulatedPt2ptData = NULL;
if (mpiPi.do_collective_stats_report == 1)
{
init_histogram (&mpiPi.coll_comm_histogram, 7, 32, NULL);
init_histogram (&mpiPi.coll_size_histogram, 7, 32, NULL);
}
if (mpiPi.do_pt2pt_stats_report == 1)
{
init_histogram (&mpiPi.pt2pt_comm_histogram, 7, 32, NULL);
init_histogram (&mpiPi.pt2pt_size_histogram, 7, 32, NULL);
if(mpiPi.do_pt2pt_detail_report == 1)
{
mpiPi.task_pt2pt_stats =
h_open(mpiPi.tableSize, mpiPi_pt2pt_stats_rank_hashkey,
mpiPi_pt2pt_stats_rank_comparator);
}
}
/* -- welcome msg only collector */
if (mpiPi.collectorRank == mpiPi.rank)
{
mpiPi_msg ("");
mpiPi_msg ("%s V%d.%d.%d (Build %s/%s)\n", mpiPi.toolname, mpiPi_vmajor,
mpiPi_vminor, mpiPi_vpatch, mpiPi_vdate, mpiPi_vtime);
mpiPi_msg ("Direct questions and errors to %s\n", MPIP_HELP_LIST);
mpiPi_msg ("\n");
}
mpiPi_msg_debug ("appName is %s\n", appName);
mpiPi_msg_debug ("sizeof(callsite_stats_t) is %d\n",
sizeof (callsite_stats_t));
mpiPi_msg_debug ("successful init on %d, %s\n", mpiPi.rank, mpiPi.hostname);
if (mpiPi.enabled)
{
mpiPi_GETTIME (&mpiPi.startTime);
}
return;
}
