/* run a pairwise flood test sending iters messages
* of a message of size msgsz bytes and no acknowledgements -
* messages are shoveled into a send queue of size queuesz,
* as quickly as MPI will take them
* uses nonblocking recvs and nonblocking sends
* returns the total number of microseconds consumed during the test
*/
double floodtest(int iters, int msgsz) {
int numsent = 0, numrecvd = 0, numrecvposted = 0;
int64_t starttime, endtime;
int iamsender = (rank % 2 == 0);
int iamreceiver = !iamsender || peerid == rank; /* handle loopback */
MPI_Request *recvHandle = NULL;
MPI_Request *sendHandle = NULL;
char *sendbuffer = NULL;
char *recvbuffer = NULL;
int *indextmp = malloc(sizeof(int)*queuedepth);
MPI_Status *statustmp = malloc(sizeof(MPI_Status)*queuedepth);
if (iters < queuedepth) {
fprintf(stderr, "ERROR: iters must be >= queuedepth\n");
abort();
}
if (iamsender) {
int i;
sendbuffer = (char*)malloc(msgsz*queuedepth);
sendHandle = (MPI_Request*)malloc(sizeof(MPI_Request)*queuedepth);
assert(sendbuffer && sendHandle);
for (i=0; i < queuedepth; i++) {
sendHandle[i] = MPI_REQUEST_NULL;
}
}
if (iamreceiver) {
recvbuffer = (char*)malloc(msgsz*queuedepth);
recvHandle = (MPI_Request*)malloc(sizeof(MPI_Request)*queuedepth);
assert(recvbuffer && recvHandle);
while(numrecvposted < queuedepth && numrecvposted < iters) {
recvHandle[numrecvposted] = MPI_REQUEST_NULL;
/* prepost recvs */
MPI_SAFE(MPI_Irecv(BUFFER_CALC(recvbuffer,msgsz*numrecvposted), msgsz, MPI_BYTE,
peerid, MPI_ANY_TAG, MPI_COMM_WORLD,
&recvHandle[numrecvposted]));
assert(recvHandle[numrecvposted] != MPI_REQUEST_NULL);
numrecvposted++;
}
}
barrier();
starttime = getMicrosecondTimeStamp();
if (iamsender) { /* fill the outgoing pipe */
while (numsent < iters && numsent < queuedepth) {
char *buf = BUFFER_CALC(sendbuffer,msgsz*numsent);
WRITEMSG(buf, msgsz);
MPI_SAFE(MPI_Isend(buf, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD, &sendHandle[numsent]));
assert(sendHandle[numsent] != MPI_REQUEST_NULL);
numsent++;
}
}
while ( (iamsender && numsent < iters) ||
(iamreceiver && numrecvd < iters)) {
if (iamreceiver) {
int numcomplete = 0;
/* reap any completions and do more recvs */
MPI_SAFE(mpi_testwait_some(queuedepth, recvHandle, &numcomplete, indextmp, statustmp));
while (numcomplete != MPI_UNDEFINED && numcomplete > 0) {
int idx = indextmp[--numcomplete];
char *buf = BUFFER_CALC(recvbuffer,msgsz*idx);
CHECKTAG(statustmp[numcomplete].MPI_TAG);
READMSG(buf, msgsz);
numrecvd++;
assert(recvHandle[idx] == MPI_REQUEST_NULL);
if (numrecvposted < iters) { /* not done yet - recv another */
MPI_SAFE(MPI_Irecv(buf, msgsz, MPI_BYTE,
peerid, MPI_ANY_TAG, MPI_COMM_WORLD,
&recvHandle[idx]));
assert(recvHandle[idx] != MPI_REQUEST_NULL);
numrecvposted++;
}
}
}
if (iamsender) {
int numcomplete = 0;
/* reap any completions and do more sends */
MPI_SAFE(mpi_testwait_some(queuedepth, sendHandle, &numcomplete, indextmp, statustmp));
while (numcomplete != MPI_UNDEFINED && numcomplete > 0) {
int idx = indextmp[--numcomplete];
char *buf = BUFFER_CALC(sendbuffer,msgsz*idx);
assert(sendHandle[idx] == MPI_REQUEST_NULL);
if (numsent < iters) { /* not done yet - send another */
WRITEMSG(buf, msgsz);
MPI_SAFE(MPI_Isend(buf, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD, &sendHandle[idx]));
assert(sendHandle[idx] != MPI_REQUEST_NULL);
numsent++;
}
}
}
}
if (iamsender) { /* pause for all sends to complete locally */
MPI_SAFE(MPI_Waitall(queuedepth, sendHandle, statustmp));
}
endtime = getMicrosecondTimeStamp();
if (recvHandle) free(recvHandle);
if (sendHandle) free(sendHandle);
if (sendbuffer) free(sendbuffer);
if (recvbuffer) free(recvbuffer);
free(indextmp);
free(statustmp);
return (double)(endtime - starttime);
}
/*-------------------------------------------------------------------*/
int read_omadesc (DEVBLK *dev)
{
int rc; /* Return code */
int i; /* Array subscript */
size_t pathlen; /* Length of TDF path name */
int tdfsize; /* Size of TDF file in bytes */
int filecount; /* Number of files */
int stmt; /* TDF file statement number */
int fd; /* TDF file descriptor */
struct stat statbuf; /* TDF file information */
U32 blklen; /* Fixed block length */
int tdfpos; /* Position in TDF buffer */
char *tdfbuf; /* -> TDF file buffer */
char *tdfrec; /* -> TDF record */
char *tdffilenm; /* -> Filename in TDF record */
char *tdfformat; /* -> Format in TDF record */
char *tdfreckwd; /* -> Keyword in TDF record */
char *tdfblklen; /* -> Length in TDF record */
OMATAPE_DESC *tdftab; /* -> Tape descriptor array */
BYTE c; /* Work area for sscanf */
char pathname[MAX_PATH]; /* file path in host format */
char *strtok_str = NULL; /* last token position */
/* Isolate the base path name of the TDF file */
for (pathlen = strlen(dev->filename); pathlen > 0; )
{
pathlen--;
if (dev->filename[pathlen-1] == '/') break;
}
#if 0
// JCS thinks this is bad
if (pathlen < 7
|| strncasecmp(dev->filename+pathlen-7, "/tapes/", 7) != 0)
{
WRITEMSG ("%1d:%04X TDF File '%s': invalid filename: TDF files must be in the TAPES subdirectory", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename+pathlen, "oma");
return -1;
}
pathlen -= 7;
#endif
/* Open the tape descriptor file */
hostpath(pathname, dev->filename, sizeof(pathname));
fd = HOPEN (pathname, O_RDONLY | O_BINARY);
if (fd < 0)
{
WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "open()", strerror(errno));
return -1;
}
/* Determine the size of the tape descriptor file */
rc = fstat (fd, &statbuf);
if (rc < 0)
{
WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "fstat()", strerror(errno));
close (fd);
return -1;
}
tdfsize = statbuf.st_size;
/* Obtain a buffer for the tape descriptor file */
tdfbuf = malloc (tdfsize);
if (tdfbuf == NULL)
{
WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "malloc()", strerror(errno));
close (fd);
return -1;
}
/* Read the tape descriptor file into the buffer */
rc = read (fd, tdfbuf, tdfsize);
if (rc < tdfsize)
{
WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "read()", strerror(errno));
free (tdfbuf);
close (fd);
return -1;
}
/* Close the tape descriptor file */
close (fd); fd = -1;
/* Check that the first record is a TDF header */
if (memcmp(tdfbuf, "@TDF", 4) != 0)
{
WRMSG (HHC00206, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma");
free (tdfbuf);
return -1;
}
/* Count the number of linefeeds in the tape descriptor file
to determine the size of the descriptor array required */
for (i = 0, filecount = 0; i < tdfsize; i++)
{
if (tdfbuf[i] == '\n') filecount++;
} /* end for(i) */
/* ISW Add 1 to filecount to add an extra EOT marker */
filecount++;
/* Obtain storage for the tape descriptor array */
tdftab = (OMATAPE_DESC*)malloc (filecount * sizeof(OMATAPE_DESC));
if (tdftab == NULL)
{
WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "malloc()", strerror(errno));
free (tdfbuf);
return -1;
}
/* Build the tape descriptor array */
for (filecount = 0, tdfpos = 0, stmt = 1; ; filecount++)
{
/* Clear the tape descriptor array entry */
memset (&(tdftab[filecount]), 0, sizeof(OMATAPE_DESC));
/* Point past the next linefeed in the TDF file */
while (tdfpos < tdfsize && tdfbuf[tdfpos++] != '\n');
stmt++;
/* Exit at end of TDF file */
if (tdfpos >= tdfsize) break;
/* Mark the end of the TDF record with a null terminator */
tdfrec = tdfbuf + tdfpos;
while (tdfpos < tdfsize && tdfbuf[tdfpos]!='\r'
&& tdfbuf[tdfpos]!='\n') tdfpos++;
c = tdfbuf[tdfpos];
if (tdfpos >= tdfsize) break;
tdfbuf[tdfpos] = '\0';
/* Exit if TM or EOT record */
if (strcasecmp(tdfrec, "TM") == 0)
{
tdftab[filecount].format='X';
tdfbuf[tdfpos] = c;
continue;
}
if(strcasecmp(tdfrec, "EOT") == 0)
{
tdftab[filecount].format='E';
break;
}
/* Parse the TDF record */
tdffilenm = strtok_r (tdfrec, " \t", &strtok_str);
tdfformat = strtok_r (NULL, " \t", &strtok_str);
tdfreckwd = strtok_r (NULL, " \t", &strtok_str);
tdfblklen = strtok_r (NULL, " \t", &strtok_str);
/* Check for missing fields */
if (tdffilenm == NULL || tdfformat == NULL)
{
WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, "filename or format missing");
free (tdftab);
free (tdfbuf);
return -1;
}
/* Check that the file name is not too long */
if (pathlen + 1 + strlen(tdffilenm)
> sizeof(tdftab[filecount].filename) - 1)
{
char buf[MAX_PATH+32];
if ( strchr(tdffilenm, SPACE) == NULL)
MSGBUF(buf, "filename %s too long", tdffilenm);
else
MSGBUF(buf, "filename '%s' too long", tdffilenm);
WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, buf);
free (tdftab);
free (tdfbuf);
return -1;
}
/* Convert the file name to Unix format */
for (i = 0; i < (int)strlen(tdffilenm); i++)
{
if (tdffilenm[i] == '\\')
tdffilenm[i] = '/';
/* JCS */
// else
// tdffilenm[i] = tolower(tdffilenm[i]);
} /* end for(i) */
/* Prefix the file name with the base path name and
save it in the tape descriptor array */
/* but only if the filename lacks a leading slash - JCS */
/*
strncpy (tdftab[filecount].filename, dev->filename, pathlen);
if (tdffilenm[0] != '/')
stlrcat ( tdftab[filecount].filename, "/", sizeof(tdftab[filecount].filename) );
strlcat ( tdftab[filecount].filename, tdffilenm, sizeof(tdftab[filecount].filename) );
*/
tdftab[filecount].filename[0] = 0;
if ((tdffilenm[0] != '/') && (tdffilenm[1] != ':'))
{
strncpy (tdftab[filecount].filename, dev->filename, pathlen);
strlcat (tdftab[filecount].filename, "/", sizeof(tdftab[filecount].filename) );
}
strlcat (tdftab[filecount].filename, tdffilenm, sizeof(tdftab[filecount].filename) );
/* Check for valid file format code */
if (strcasecmp(tdfformat, "HEADERS") == 0)
{
tdftab[filecount].format = 'H';
}
else if (strcasecmp(tdfformat, "TEXT") == 0)
{
tdftab[filecount].format = 'T';
}
else if (strcasecmp(tdfformat, "FIXED") == 0)
{
/* Check for RECSIZE keyword */
if (tdfreckwd == NULL
|| strcasecmp(tdfreckwd, "RECSIZE") != 0)
{
WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, "keyword RECSIZE missing");
free (tdftab);
free (tdfbuf);
return -1;
}
/* Check for valid fixed block length */
if (tdfblklen == NULL
|| sscanf(tdfblklen, "%u%c", &blklen, &c) != 1
|| blklen < 1 || blklen > MAX_BLKLEN)
{
char buf[40];
MSGBUF(buf, "invalid record size %s", tdfblklen);
WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, buf);
free (tdftab);
free (tdfbuf);
return -1;
}
/* Set format and block length in descriptor array */
tdftab[filecount].format = 'F';
tdftab[filecount].blklen = blklen;
}
else
{
char buf[40];
MSGBUF(buf, "invalid record format '%s'", tdfformat);
WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, buf);
free (tdftab);
free (tdfbuf);
return -1;
}
tdfbuf[tdfpos] = c;
} /* end for(filecount) */
/* Force an EOT as last entry (filecount is correctly adjusted here) */
tdftab[filecount].format='E';
/* Save the file count and TDF array pointer in the device block */
dev->omafiles = filecount+1;
dev->omadesc = tdftab;
/* Release the TDF file buffer and exit */
free (tdfbuf);
return 0;
} /* end function read_omadesc */
double pingpongtest(int iters, int msgsz) {
int i;
int64_t starttime, endtime;
int iamsender = (rank % 2 == 0);
int iamreceiver = !iamsender || peerid == rank; /* handle loopback */
char *sendMsgbuffer = (char*)malloc(msgsz);
char *sendAckbuffer = (char*)malloc(msgsz);
char *recvMsgbuffer = (char*)malloc(msgsz);
char *recvAckbuffer = (char*)malloc(msgsz);
MPI_Request recvMsgHandle = MPI_REQUEST_NULL;
MPI_Request recvAckHandle = MPI_REQUEST_NULL;
MPI_Request sendMsgHandle = MPI_REQUEST_NULL;
MPI_Request sendAckHandle = MPI_REQUEST_NULL;
MPI_Status status;
#if USE_ZERO_BYTE_ACK
#define ACKSZ 0
#else
#define ACKSZ msgsz
#endif
if (iamreceiver) {
/* prepost a recv */
MPI_SAFE(MPI_Irecv(recvMsgbuffer, msgsz, MPI_BYTE,
peerid, MPI_ANY_TAG, MPI_COMM_WORLD,
&recvMsgHandle));
}
barrier();
starttime = getMicrosecondTimeStamp();
for (i=0; i < iters; i++) {
if (iamsender) {
/* send message */
WRITEMSG(sendMsgbuffer, msgsz);
#if USE_ISEND
MPI_SAFE(MPI_Isend(sendMsgbuffer, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD, &sendMsgHandle));
#else
MPI_SAFE(MPI_Send(sendMsgbuffer, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD));
#endif
/* prepost a recv for acknowledgement */
MPI_SAFE(MPI_Irecv(recvAckbuffer, ACKSZ, MPI_BYTE,
peerid, MPI_ANY_TAG, MPI_COMM_WORLD,
&recvAckHandle));
#if USE_ISEND
MPI_SAFE(MPI_Wait(&sendMsgHandle, &status));
#endif
}
if (iamreceiver) {
/* wait for message */
#if USE_TEST
int flag = 0;
while (!flag) MPI_SAFE(MPI_Test(&recvMsgHandle, &flag, &status));
#else
MPI_SAFE(MPI_Wait(&recvMsgHandle, &status));
#endif
CHECKTAG(status.MPI_TAG);
READMSG(recvMsgbuffer, msgsz);
/* send acknowledgement */
WRITEMSG(sendAckbuffer, 1);
#if USE_ISEND
MPI_SAFE(MPI_Isend(sendAckbuffer, ACKSZ, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD, &sendAckHandle));
#else
MPI_SAFE(MPI_Send(sendAckbuffer, ACKSZ, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD));
#endif
/* pre-post recv for next message */
MPI_SAFE(MPI_Irecv(recvMsgbuffer, msgsz, MPI_BYTE,
peerid, MPI_ANY_TAG, MPI_COMM_WORLD,
&recvMsgHandle));
#if USE_ISEND
MPI_SAFE(MPI_Wait(&sendAckHandle, &status));
#endif
}
if (iamsender) {
/* wait for acknowledgement */
MPI_SAFE(MPI_Wait(&recvAckHandle, &status));
CHECKTAG(status.MPI_TAG);
READMSG(recvAckbuffer, 1);
}
}
endtime = getMicrosecondTimeStamp();
/* last recv must be cancelled (not included in timing) */
#if 0
if (iamreceiver) MPI_SAFE(MPI_Cancel(&recvMsgHandle));
#else
/* apparently some MPI impls don't implement cancel at all.. (grr..) */
/* use an extra send instead to get the same effect */
if (iamsender)
MPI_SAFE(MPI_Send(sendMsgbuffer, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD));
#endif
if (iamreceiver) MPI_SAFE(MPI_Wait(&recvMsgHandle, &status));
free(sendMsgbuffer);
free(sendAckbuffer);
free(recvMsgbuffer);
free(recvAckbuffer);
return (double)(endtime - starttime);
}
/*-------------------------------------------------------------------*/
int open_omatape (DEVBLK *dev, BYTE *unitstat,BYTE code)
{
int fd; /* File descriptor integer */
int rc; /* Return code */
OMATAPE_DESC *omadesc; /* -> OMA descriptor entry */
char pathname[MAX_PATH]; /* file path in host format */
/* Check for no tape in drive */
if (!strcmp (dev->filename, TAPE_UNLOADED))
{
build_senseX(TAPE_BSENSE_TAPEUNLOADED,dev,unitstat,code);
return -1;
}
/* Read the OMA descriptor file if necessary */
if (dev->omadesc == NULL)
{
rc = read_omadesc (dev);
if (rc < 0)
{
build_senseX(TAPE_BSENSE_TAPELOADFAIL,dev,unitstat,code);
return -1;
}
dev->blockid = 0;
}
dev->fenced = 0;
/* Unit exception if beyond end of tape */
/* ISW: CHANGED PROCESSING - RETURN UNDEFINITE Tape Marks */
/* NOTE: The last entry in the TDF table is ALWAYS */
/* an EOT Condition */
/* This is ensured by the TDF reading routine */
#if 0
if (dev->curfilen >= dev->omafiles)
{
WRITEMSG (HHC00000E, SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename);
build_senseX(TAPE_BSENSE_ENDOFTAPE,dev,unitstat,code);
return -1;
}
#else
if(dev->curfilen>dev->omafiles)
{
dev->curfilen=dev->omafiles;
return(0);
}
#endif
/* Point to the current file entry in the OMA descriptor table */
omadesc = (OMATAPE_DESC*)(dev->omadesc);
omadesc += (dev->curfilen-1);
if(omadesc->format=='X')
{
return 0;
}
if(omadesc->format=='E')
{
return 0;
}
/* Open the OMATAPE file */
hostpath(pathname, omadesc->filename, sizeof(pathname));
fd = HOPEN (pathname, O_RDONLY | O_BINARY);
/* Check for successful open */
if (fd < 0 || lseek (fd, 0, SEEK_END) > LONG_MAX)
{
if (fd >= 0) /* (if open was successful, then it) */
errno = EOVERFLOW; /* (must have been a lseek overflow) */
WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, omadesc->filename, "oma", "open()", strerror(errno));
if (fd >= 0)
close(fd); /* (close the file if it was opened) */
build_senseX(TAPE_BSENSE_TAPELOADFAIL,dev,unitstat,code);
return -1;
}
/* OMA tapes are always read-only */
dev->readonly = 1;
/* Store the file descriptor in the device block */
dev->fd = fd;
return 0;
} /* end function open_omatape */
