void
FunctionMap_Add (const char *funcName,
Dwarf_Addr lowAddress, Dwarf_Addr highAddress)
{
/* build a new entry for this function */
struct FunctionInfo *newEntry =
(struct FunctionInfo *) malloc (sizeof (struct FunctionInfo));
if (newEntry == NULL)
{
mpiPi_abort ("malloc failed\n");
}
mpiPi_msg_debug ("FunctionMap::Add: %s [0x%016llx,0x%016llx]\n",
funcName, lowAddress, highAddress);
newEntry->name = strdup (funcName);
if (newEntry->name == NULL)
{
mpiPi_abort ("malloc failed\n");
}
/* add the function to our ordered collection of functions */
assert (functionMap != NULL);
AddrRangeMap_Add (functionMap, lowAddress, highAddress, newEntry);
}
static void
AddrToSourceMap_Init (void)
{
addrToSourceMap =
(struct AddrRangeMap *) malloc (sizeof (struct AddrRangeMap));
if (addrToSourceMap == NULL)
{
mpiPi_abort ("malloc failed\n");
}
AddrRangeMap_Init (addrToSourceMap);
}
static void
FunctionMap_Init (void)
{
assert (functionMap == NULL);
functionMap = (struct AddrRangeMap *) malloc (sizeof (struct AddrRangeMap));
if (functionMap == NULL)
{
mpiPi_abort ("malloc failed\n");
}
AddrRangeMap_Init (functionMap);
}
char *
getProcExeLink ()
{
int pid, exelen, insize = 256;
char *inbuf = NULL, file[256];
pid = getpid ();
snprintf (file, 256, "/proc/%d/exe", pid);
inbuf = malloc (insize);
if (inbuf == NULL)
{
mpiPi_abort ("unable to allocate space for full executable path.\n");
}
exelen = readlink (file, inbuf, 256);
if (exelen == -1)
{
if (errno != ENOENT)
{
while (exelen == -1 && errno == ENAMETOOLONG)
{
insize += 256;
inbuf = realloc (inbuf, insize);
exelen = readlink (file, inbuf, insize);
}
inbuf[exelen] = '\0';
return inbuf;
}
else
free (inbuf);
}
else
{
inbuf[exelen] = '\0';
return inbuf;
}
return NULL;
}
static void
AddrToSourceMap_Add (Dwarf_Addr addr,
const char *fileName, unsigned int lineNumber)
{
/* build a new entry for this mapping */
struct AddrToSourceInfo *newEntry =
(struct AddrToSourceInfo *) malloc (sizeof (struct AddrToSourceInfo));
if (newEntry == NULL)
{
mpiPi_abort ("malloc failed\n");
}
newEntry->fileName = UniqueFileName_Get (fileName);
assert (newEntry->fileName != NULL);
newEntry->lineNumber = lineNumber;
mpiPi_msg_debug ("AddrToSourceMap::Add: 0x%016llx => %s: %d\n",
addr, newEntry->fileName, newEntry->lineNumber);
assert (addrToSourceMap != NULL);
AddrRangeMap_Add (addrToSourceMap, addr, addr, /* we will patch range ends later */
newEntry);
}
int
open_dwarf_executable (char *fileName)
{
int dwStatus = -1;
mpiPi_msg_debug ("enter open_dwarf_executable\n");
if (fileName == NULL)
{
mpiPi_msg_warn ("Executable file name is NULL!\n");
mpiPi_msg_warn
("If this is a Fortran application, you may be using the incorrect mpiP library.\n");
}
/* open the executable */
assert (dwFd == -1);
mpiPi_msg_debug ("opening file %s\n", fileName);
dwFd = open (fileName, O_RDONLY);
if (dwFd == -1)
{
mpiPi_msg_warn ("could not open file %s\n", fileName);
return 0;
}
/* initialize the DWARF library */
assert (dwHandle == NULL);
dwStatus = dwarf_init (dwFd, /* exe file descriptor */
DW_DLC_READ, /* desired access */
NULL, /* error handler */
NULL, /* error argument */
&dwHandle, /* session handle */
&dw_err); /* error object */
if (dwStatus == DW_DLV_ERROR)
{
close (dwFd);
dwFd = -1;
mpiPi_abort ("could not initialize DWARF library : %s\n", dwarf_errmsg(dw_err));
}
if (dwStatus == DW_DLV_NO_ENTRY)
{
mpiPi_msg_warn ("No symbols in the executable\n");
return 0;
}
/* initialize our function and addr-to-source mappings */
AddrToSourceMap_Init ();
FunctionMap_Init ();
/* access symbol info */
while (1)
{
Dwarf_Unsigned nextCompilationUnitHeaderOffset = 0;
Dwarf_Die currCompileUnitDIE = NULL;
Dwarf_Line *lineEntries = NULL;
Dwarf_Signed nLineEntries = 0;
Dwarf_Half currDIETag;
Dwarf_Die currChildDIE = NULL;
Dwarf_Die nextChildDIE = NULL;
Dwarf_Die oldChildDIE = NULL;
/* access next compilation unit header */
dwStatus = dwarf_next_cu_header (dwHandle, NULL, /* cu_header_length */
NULL, /* version_stamp */
NULL, /* abbrev_offset */
NULL, /* address_size */
&nextCompilationUnitHeaderOffset, &dw_err); /* error object */
if (dwStatus != DW_DLV_OK)
{
if (dwStatus != DW_DLV_NO_ENTRY)
{
mpiPi_abort ("failed to access next DWARF cu header : %s\n", dwarf_errmsg(dw_err));
}
break;
}
/* access the first debug info entry (DIE) for this computation unit */
dwStatus = dwarf_siblingof (dwHandle, NULL, /* current DIE */
&currCompileUnitDIE, /* sibling DIE */
&dw_err); /* error object */
if (dwStatus != DW_DLV_OK)
{
mpiPi_abort ("failed to access first DWARF DIE : %s\n", dwarf_errmsg(dw_err));
}
/* get line number information for this compilation
* unit, if available
*/
dwStatus = dwarf_srclines (currCompileUnitDIE,
&lineEntries, &nLineEntries, &dw_err);
if (dwStatus == DW_DLV_OK)
{
unsigned int i;
/*
* Extract and save address-to-source line mapping.
*
* NOTE: At least on the Cray X1, we see line number
* information with the same address mapping to different lines.
* It seems like when there are multiple entries for a given
* address, the last one is the correct one. (Needs verification.)
* If we see multiple entries for a given address, we only
* save the last one.
*/
for (i = 0; i < nLineEntries; i++)
{
Dwarf_Unsigned lineNumber = 0;
Dwarf_Addr lineAddress = 0;
char *lineSourceFile = NULL;
int lineNoStatus, lineAddrStatus, lineSrcFileStatus;
lineNoStatus = dwarf_lineno (lineEntries[i],
&lineNumber,
&dw_err);
if (lineNoStatus != DW_DLV_OK)
mpiPi_msg_debug("Failed to get line number : %s\n", dwarf_errmsg(dw_err));
lineAddrStatus = dwarf_lineaddr (lineEntries[i],
&lineAddress,
&dw_err);
if (lineAddrStatus != DW_DLV_OK)
mpiPi_msg_debug("Failed to get line address : %s\n", dwarf_errmsg(dw_err));
lineSrcFileStatus = dwarf_linesrc (lineEntries[i],
&lineSourceFile,
&dw_err);
if (lineSrcFileStatus != DW_DLV_OK)
mpiPi_msg_debug("Failed to get source file status : %s\n", dwarf_errmsg(dw_err));
if ((lineNoStatus == DW_DLV_OK) &&
(lineAddrStatus == DW_DLV_OK)
&& (lineSrcFileStatus == DW_DLV_OK))
{
int saveCurrentEntry = 0; /* bool */
if (i < (nLineEntries - 1))
{
/*
* We're not on the last line number entry -
* check the address associated with the next
* entry to see if it differs from this one.
* Only save the entry if it the next address
* differs.
*/
Dwarf_Addr nextLineAddress = 0;
int nextLineAddrStatus =
dwarf_lineaddr (lineEntries[i + 1],
&nextLineAddress,
&dw_err);
assert (nextLineAddrStatus == DW_DLV_OK);
if (nextLineAddress != lineAddress)
{
saveCurrentEntry = 1;
}
}
else
{
/* we're on the last line number entry */
saveCurrentEntry = 1;
}
if (saveCurrentEntry)
{
/* save the mapping entry */
AddrToSourceMap_Add (lineAddress, lineSourceFile,
lineNumber);
}
}
if (lineSourceFile != NULL)
{
dwarf_dealloc (dwHandle, lineSourceFile, DW_DLA_STRING);
}
}
/* release the line number info */
for (i = 0; i < nLineEntries; i++)
{
dwarf_dealloc (dwHandle, lineEntries[i], DW_DLA_LINE);
}
dwarf_dealloc (dwHandle, lineEntries, DW_DLA_LIST);
}
else if (dwStatus != DW_DLV_ERROR)
{
/*
* no line information for the current DIE -
* not an error, just unfortunate
*/
}
else
{
mpiPi_abort ("failed to obtain line info for the current DIE : %s\n", dwarf_errmsg(dw_err));
}
/* discover function information for the current compilation unit */
/*
* NOTE: DWARF debug information entries can be organized in
* a hierarchy. However, we presume that the function entries are
* all children of the compilation unit DIE.
*/
dwStatus = dwarf_tag (currCompileUnitDIE, &currDIETag, &dw_err);
assert ((dwStatus == DW_DLV_OK) && (currDIETag == DW_TAG_compile_unit));
/* access the first child DIE of the compile unit DIE */
dwStatus = dwarf_child (currCompileUnitDIE, &currChildDIE, &dw_err);
if (dwStatus == DW_DLV_NO_ENTRY)
{
// On some Cray systems, executables are linked with assembly compile units
// with no functions.
// mpiPi_abort ("no child DIEs of compile unit DIE\n");
}
else if (dwStatus != DW_DLV_OK)
{
mpiPi_abort
("failed to access first child DIE of compile unit DIE\n");
}
while (dwStatus == DW_DLV_OK)
{
/* determine the type of the child DIE */
dwStatus = dwarf_tag (currChildDIE, &currDIETag, &dw_err);
if (dwStatus == DW_DLV_OK)
{
if ((currDIETag == DW_TAG_subprogram) ||
(currDIETag == DW_TAG_entry_point))
{
HandleFunctionDIE (dwHandle, currChildDIE);
}
}
else
{
mpiPi_abort ("unable to determine tag of current child DIE : %s \n", dwarf_errmsg(dw_err));
}
/* advance to the next child DIE */
oldChildDIE = currChildDIE;
dwStatus = dwarf_siblingof (dwHandle,
currChildDIE, &nextChildDIE, &dw_err);
if (dwStatus == DW_DLV_OK)
{
currChildDIE = nextChildDIE;
nextChildDIE = NULL;
}
else if (dwStatus != DW_DLV_NO_ENTRY)
{
mpiPi_abort
("unable to access next child DIE of current compilation unit : %s\n", dwarf_errmsg(dw_err));
}
if (oldChildDIE != NULL)
{
dwarf_dealloc (dwHandle, oldChildDIE, DW_DLA_DIE);
}
}
/* release the current compilation unit DIE */
dwarf_dealloc (dwHandle, currCompileUnitDIE, DW_DLA_DIE);
}
/*
* DWARF address-to-source line information does not give
* address ranges, so we need to patch the address ranges
* in our address-to-source map.
*/
AddrToSourceMap_PatchRanges ();
return 1;
}
static char *
UniqueFileName_Get (const char *testFileName)
{
char *ret = NULL;
struct UniqueFileNameListNode *currInfo = NULL;
assert (testFileName != NULL);
currInfo = uniqueFileNameList;
while (currInfo != NULL)
{
if (strcmp (testFileName, currInfo->ufi.fileName) == 0)
{
/* we found a match */
break;
}
/* advance to the next file name */
currInfo = currInfo->next;
}
if (currInfo != NULL)
{
/*
* We found a match during our search -
* use the already-allocated string.
*/
ret = currInfo->ufi.fileName;
}
else
{
/*
* We didn't find a match during our search.
* Allocate a new string.
*/
currInfo = (struct UniqueFileNameListNode *)
malloc (sizeof (struct UniqueFileNameListNode));
if (currInfo == NULL)
{
mpiPi_abort ("malloc failed\n");
}
currInfo->ufi.fileName = strdup (testFileName);
if (currInfo->ufi.fileName == NULL)
{
mpiPi_abort ("malloc failed\n");
}
/*
* link the new entry into our list
* since the list is unordered, just link it in
* at the beginning for simplicity.
*/
currInfo->next = uniqueFileNameList;
uniqueFileNameList = currInfo;
ret = currInfo->ufi.fileName;
}
assert (ret != NULL);
return ret;
}
static void
AddrRangeMap_Add (struct AddrRangeMap *map,
Dwarf_Addr lowAddr, Dwarf_Addr highAddr, void *info)
{
struct AddrRangeMapNode *currNode = NULL;
struct AddrRangeMapNode *newEntry = NULL;
mpiPi_msg_debug ("AddrRangeMap::Add: [0x%016llx,0x%016llx]\n",
lowAddr, highAddr);
/* build a new entry for this mapping */
newEntry =
(struct AddrRangeMapNode *) malloc (sizeof (struct AddrRangeMapNode));
if (newEntry == NULL)
{
mpiPi_abort ("malloc failed\n");
}
newEntry->color = AddrRangeMapNodeColor_Red;
assert (lowAddr <= highAddr);
newEntry->lowAddr = lowAddr;
newEntry->highAddr = highAddr;
newEntry->info = info;
newEntry->parent = NULL;
newEntry->leftChild = NULL;
newEntry->rightChild = NULL;
/* add the new node to our map */
if (map->root == NULL)
{
/* new node is the first node to be added */
map->root = newEntry;
}
else
{
/* new node is not the first node to be added */
currNode = map->root;
while (currNode != NULL)
{
if (highAddr <= currNode->lowAddr)
{
/* target address is below range covered by current node */
if (currNode->leftChild != NULL)
{
/* compare with ranges smaller than ours */
currNode = currNode->leftChild;
}
else
{
/* adopt new node as our left child */
currNode->leftChild = newEntry;
newEntry->parent = currNode;
break;
}
}
else if (lowAddr >= currNode->highAddr)
{
/* target address is above range covered by current node */
if (currNode->rightChild != NULL)
{
/* compare with ranges larger than ours */
currNode = currNode->rightChild;
}
else
{
/* adopt new node as our right child */
currNode->rightChild = newEntry;
newEntry->parent = currNode;
break;
}
}
else
{
/* new range overlaps with our range! */
mpiPi_abort
("new address node range [0x%016llx,0x%016llx] overlaps our range [0x%016llx,0x%016llx]\n",
lowAddr, highAddr, currNode->lowAddr, currNode->highAddr);
}
}
}
/*
* new node has been inserted, but its insertion
* may have unbalanced the tree -
* re-balance it.
* Based on red-black insert algorithm given in
* Cormen, Lieserson, Rivest "Introduction to Algorithms"
*/
currNode = newEntry;
while ((currNode != map->root) &&
(currNode->parent->color == AddrRangeMapNodeColor_Red))
{
struct AddrRangeMapNode *uncleNode = NULL;
if (currNode->parent == currNode->parent->parent->leftChild)
{
/* currNode's parent is its parent's left child */
uncleNode = currNode->parent->parent->rightChild;
if ((uncleNode != NULL) &&
(uncleNode->color == AddrRangeMapNodeColor_Red))
{
currNode->parent->color = AddrRangeMapNodeColor_Black;
uncleNode->color = AddrRangeMapNodeColor_Black;
currNode->parent->parent->color = AddrRangeMapNodeColor_Red;
currNode = currNode->parent->parent;
}
else
{
/* uncleNode is NULL (NULL is assumed black) or is black */
if (currNode == currNode->parent->rightChild)
{
currNode = currNode->parent;
AddrRangeMap_LeftRotate (map, currNode);
}
currNode->parent->color = AddrRangeMapNodeColor_Black;
currNode->parent->parent->color = AddrRangeMapNodeColor_Red;
AddrRangeMap_RightRotate (map, currNode->parent->parent);
}
}
else
{
/* currNode's parent is its parent's right child */
uncleNode = currNode->parent->parent->leftChild;
if ((uncleNode != NULL) &&
(uncleNode->color == AddrRangeMapNodeColor_Red))
{
currNode->parent->color = AddrRangeMapNodeColor_Black;
uncleNode->color = AddrRangeMapNodeColor_Black;
currNode->parent->parent->color = AddrRangeMapNodeColor_Red;
currNode = currNode->parent->parent;
}
else
{
/* uncleNode is NULL (NULL is assumed black) or is black */
if (currNode == currNode->parent->leftChild)
{
currNode = currNode->parent;
AddrRangeMap_RightRotate (map, currNode);
}
currNode->parent->color = AddrRangeMapNodeColor_Black;
currNode->parent->parent->color = AddrRangeMapNodeColor_Red;
AddrRangeMap_LeftRotate (map, currNode->parent->parent);
}
}
}
assert (map->root != NULL);
map->root->color = AddrRangeMapNodeColor_Black;
}
/*
* mpiPi_collect_basics() - all tasks send their basic info to the
* collectorRank.
*/
void
mpiPi_collect_basics ()
{
int i = 0;
double app_time = mpiPi.cumulativeTime;
int cnt;
mpiPi_task_info_t mti;
int blockcounts[4] = { 1, 1, 1, MPIPI_HOSTNAME_LEN_MAX };
MPI_Datatype types[4] = { MPI_DOUBLE, MPI_DOUBLE, MPI_INT, MPI_CHAR };
MPI_Aint displs[4];
MPI_Datatype mti_type;
MPI_Request *recv_req_arr;
mpiPi_msg_debug ("Collect Basics\n");
cnt = 0;
PMPI_Address (&mti.mpi_time, &displs[cnt++]);
PMPI_Address (&mti.app_time, &displs[cnt++]);
PMPI_Address (&mti.rank, &displs[cnt++]);
PMPI_Address (&mti.hostname, &displs[cnt++]);
for (i = (cnt - 1); i >= 0; i--)
{
displs[i] -= displs[0];
}
PMPI_Type_struct (cnt, blockcounts, displs, types, &mti_type);
PMPI_Type_commit (&mti_type);
if (mpiPi.rank == mpiPi.collectorRank)
{
/* In the case where multiple reports are generated per run,
only allocate memory for global_task_info once */
if (mpiPi.global_task_info == NULL)
{
mpiPi.global_task_info =
(mpiPi_task_info_t *) calloc (mpiPi.size,
sizeof (mpiPi_task_info_t));
if (mpiPi.global_task_info == NULL)
mpiPi_abort ("Failed to allocate memory for global_task_info");
mpiPi_msg_debug
("MEMORY : Allocated for global_task_info : %13ld\n",
mpiPi.size * sizeof (mpiPi_task_info_t));
}
bzero (mpiPi.global_task_info, mpiPi.size * sizeof (mpiPi_task_info_t));
recv_req_arr =
(MPI_Request *) malloc (sizeof (MPI_Request) * mpiPi.size);
for (i = 0; i < mpiPi.size; i++)
{
mpiPi_task_info_t *p = &mpiPi.global_task_info[i];
if (i != mpiPi.collectorRank)
{
PMPI_Irecv (p, 1, mti_type, i, mpiPi.tag,
mpiPi.comm, &(recv_req_arr[i]));
}
else
{
strcpy (p->hostname, mpiPi.hostname);
p->app_time = app_time;
p->rank = mpiPi.rank;
recv_req_arr[i] = MPI_REQUEST_NULL;
}
}
PMPI_Waitall (mpiPi.size, recv_req_arr, MPI_STATUSES_IGNORE);
free (recv_req_arr);
/* task MPI time is calculated from callsites data
in mpiPi_insert_callsite_records.
*/
for (i = 0; i < mpiPi.size; i++)
mpiPi.global_task_info[i].mpi_time = 0.0;
}
else
{
strcpy (mti.hostname, mpiPi.hostname);
mti.app_time = app_time;
mti.rank = mpiPi.rank;
PMPI_Send (&mti, 1, mti_type, mpiPi.collectorRank,
mpiPi.tag, mpiPi.comm);
}
PMPI_Type_free (&mti_type);
return;
}
/*
* mpiPi_collect_basics() - all tasks send their basic info to the
* collectorRank.
*/
static void
mpiPi_collect_basics (int report_style)
{
mpiPi_msg_debug ("Collect Basics\n");
if (mpiPi.rank == mpiPi.collectorRank)
{
/* In the case where multiple reports are generated per run,
only allocate memory for global_task_info once */
if (mpiPi.global_task_app_time == NULL)
{
mpiPi.global_task_app_time =
(double *) calloc (mpiPi.size, sizeof (double));
if (mpiPi.global_task_app_time == NULL)
mpiPi_abort
("Failed to allocate memory for global_task_app_time");
mpiPi_msg_debug
("MEMORY : Allocated for global_task_app_time : %13ld\n",
mpiPi.size * sizeof (double));
}
bzero (mpiPi.global_task_app_time, mpiPi.size * sizeof (double));
if (mpiPi.global_task_mpi_time == NULL)
{
mpiPi.global_task_mpi_time =
(double *) calloc (mpiPi.size, sizeof (double));
if (mpiPi.global_task_mpi_time == NULL)
mpiPi_abort
("Failed to allocate memory for global_task_mpi_time");
mpiPi_msg_debug
("MEMORY : Allocated for global_task_mpi_time : %13ld\n",
mpiPi.size * sizeof (double));
}
bzero (mpiPi.global_task_mpi_time, mpiPi.size * sizeof (double));
// Only allocate hostname storage if we are doing a verbose report
if (mpiPi.global_task_hostnames == NULL
&& (report_style == mpiPi_style_verbose
|| report_style == mpiPi_style_both))
{
mpiPi.global_task_hostnames =
(mpiPi_hostname_t *) calloc (mpiPi.size,
sizeof (char) *
MPIPI_HOSTNAME_LEN_MAX);
if (mpiPi.global_task_hostnames == NULL)
mpiPi_abort
("Failed to allocate memory for global_task_hostnames");
mpiPi_msg_debug
("MEMORY : Allocated for global_task_hostnames : %13ld\n",
mpiPi.size * sizeof (char) * MPIPI_HOSTNAME_LEN_MAX);
}
if (mpiPi.global_task_hostnames != NULL)
bzero (mpiPi.global_task_hostnames,
mpiPi.size * sizeof (char) * MPIPI_HOSTNAME_LEN_MAX);
}
PMPI_Gather (&mpiPi.cumulativeTime, 1, MPI_DOUBLE,
mpiPi.global_task_app_time, 1, MPI_DOUBLE,
mpiPi.collectorRank, mpiPi.comm);
if (report_style == mpiPi_style_verbose || report_style == mpiPi_style_both)
{
PMPI_Gather (mpiPi.hostname, MPIPI_HOSTNAME_LEN_MAX, MPI_CHAR,
mpiPi.global_task_hostnames, MPIPI_HOSTNAME_LEN_MAX,
MPI_CHAR, mpiPi.collectorRank, mpiPi.comm);
}
return;
}
