/* run a pairwise flood test sending iters messages * of a message of size msgsz bytes and no acknowledgements - * messages are shoveled into a send queue of size queuesz, * as quickly as MPI will take them * uses nonblocking recvs and nonblocking sends * returns the total number of microseconds consumed during the test */ double floodtest(int iters, int msgsz) { int numsent = 0, numrecvd = 0, numrecvposted = 0; int64_t starttime, endtime; int iamsender = (rank % 2 == 0); int iamreceiver = !iamsender || peerid == rank; /* handle loopback */ MPI_Request *recvHandle = NULL; MPI_Request *sendHandle = NULL; char *sendbuffer = NULL; char *recvbuffer = NULL; int *indextmp = malloc(sizeof(int)*queuedepth); MPI_Status *statustmp = malloc(sizeof(MPI_Status)*queuedepth); if (iters < queuedepth) { fprintf(stderr, "ERROR: iters must be >= queuedepth\n"); abort(); } if (iamsender) { int i; sendbuffer = (char*)malloc(msgsz*queuedepth); sendHandle = (MPI_Request*)malloc(sizeof(MPI_Request)*queuedepth); assert(sendbuffer && sendHandle); for (i=0; i < queuedepth; i++) { sendHandle[i] = MPI_REQUEST_NULL; } } if (iamreceiver) { recvbuffer = (char*)malloc(msgsz*queuedepth); recvHandle = (MPI_Request*)malloc(sizeof(MPI_Request)*queuedepth); assert(recvbuffer && recvHandle); while(numrecvposted < queuedepth && numrecvposted < iters) { recvHandle[numrecvposted] = MPI_REQUEST_NULL; /* prepost recvs */ MPI_SAFE(MPI_Irecv(BUFFER_CALC(recvbuffer,msgsz*numrecvposted), msgsz, MPI_BYTE, peerid, MPI_ANY_TAG, MPI_COMM_WORLD, &recvHandle[numrecvposted])); assert(recvHandle[numrecvposted] != MPI_REQUEST_NULL); numrecvposted++; } } barrier(); starttime = getMicrosecondTimeStamp(); if (iamsender) { /* fill the outgoing pipe */ while (numsent < iters && numsent < queuedepth) { char *buf = BUFFER_CALC(sendbuffer,msgsz*numsent); WRITEMSG(buf, msgsz); MPI_SAFE(MPI_Isend(buf, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD, &sendHandle[numsent])); assert(sendHandle[numsent] != MPI_REQUEST_NULL); numsent++; } } while ( (iamsender && numsent < iters) || (iamreceiver && numrecvd < iters)) { if (iamreceiver) { int numcomplete = 0; /* reap any completions and do more recvs */ MPI_SAFE(mpi_testwait_some(queuedepth, recvHandle, &numcomplete, indextmp, statustmp)); while (numcomplete != MPI_UNDEFINED && numcomplete > 0) { int idx = indextmp[--numcomplete]; char *buf = BUFFER_CALC(recvbuffer,msgsz*idx); CHECKTAG(statustmp[numcomplete].MPI_TAG); READMSG(buf, msgsz); numrecvd++; assert(recvHandle[idx] == MPI_REQUEST_NULL); if (numrecvposted < iters) { /* not done yet - recv another */ MPI_SAFE(MPI_Irecv(buf, msgsz, MPI_BYTE, peerid, MPI_ANY_TAG, MPI_COMM_WORLD, &recvHandle[idx])); assert(recvHandle[idx] != MPI_REQUEST_NULL); numrecvposted++; } } } if (iamsender) { int numcomplete = 0; /* reap any completions and do more sends */ MPI_SAFE(mpi_testwait_some(queuedepth, sendHandle, &numcomplete, indextmp, statustmp)); while (numcomplete != MPI_UNDEFINED && numcomplete > 0) { int idx = indextmp[--numcomplete]; char *buf = BUFFER_CALC(sendbuffer,msgsz*idx); assert(sendHandle[idx] == MPI_REQUEST_NULL); if (numsent < iters) { /* not done yet - send another */ WRITEMSG(buf, msgsz); MPI_SAFE(MPI_Isend(buf, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD, &sendHandle[idx])); assert(sendHandle[idx] != MPI_REQUEST_NULL); numsent++; } } } } if (iamsender) { /* pause for all sends to complete locally */ MPI_SAFE(MPI_Waitall(queuedepth, sendHandle, statustmp)); } endtime = getMicrosecondTimeStamp(); if (recvHandle) free(recvHandle); if (sendHandle) free(sendHandle); if (sendbuffer) free(sendbuffer); if (recvbuffer) free(recvbuffer); free(indextmp); free(statustmp); return (double)(endtime - starttime); }
/*-------------------------------------------------------------------*/ int read_omadesc (DEVBLK *dev) { int rc; /* Return code */ int i; /* Array subscript */ size_t pathlen; /* Length of TDF path name */ int tdfsize; /* Size of TDF file in bytes */ int filecount; /* Number of files */ int stmt; /* TDF file statement number */ int fd; /* TDF file descriptor */ struct stat statbuf; /* TDF file information */ U32 blklen; /* Fixed block length */ int tdfpos; /* Position in TDF buffer */ char *tdfbuf; /* -> TDF file buffer */ char *tdfrec; /* -> TDF record */ char *tdffilenm; /* -> Filename in TDF record */ char *tdfformat; /* -> Format in TDF record */ char *tdfreckwd; /* -> Keyword in TDF record */ char *tdfblklen; /* -> Length in TDF record */ OMATAPE_DESC *tdftab; /* -> Tape descriptor array */ BYTE c; /* Work area for sscanf */ char pathname[MAX_PATH]; /* file path in host format */ char *strtok_str = NULL; /* last token position */ /* Isolate the base path name of the TDF file */ for (pathlen = strlen(dev->filename); pathlen > 0; ) { pathlen--; if (dev->filename[pathlen-1] == '/') break; } #if 0 // JCS thinks this is bad if (pathlen < 7 || strncasecmp(dev->filename+pathlen-7, "/tapes/", 7) != 0) { WRITEMSG ("%1d:%04X TDF File '%s': invalid filename: TDF files must be in the TAPES subdirectory", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename+pathlen, "oma"); return -1; } pathlen -= 7; #endif /* Open the tape descriptor file */ hostpath(pathname, dev->filename, sizeof(pathname)); fd = HOPEN (pathname, O_RDONLY | O_BINARY); if (fd < 0) { WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "open()", strerror(errno)); return -1; } /* Determine the size of the tape descriptor file */ rc = fstat (fd, &statbuf); if (rc < 0) { WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "fstat()", strerror(errno)); close (fd); return -1; } tdfsize = statbuf.st_size; /* Obtain a buffer for the tape descriptor file */ tdfbuf = malloc (tdfsize); if (tdfbuf == NULL) { WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "malloc()", strerror(errno)); close (fd); return -1; } /* Read the tape descriptor file into the buffer */ rc = read (fd, tdfbuf, tdfsize); if (rc < tdfsize) { WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "read()", strerror(errno)); free (tdfbuf); close (fd); return -1; } /* Close the tape descriptor file */ close (fd); fd = -1; /* Check that the first record is a TDF header */ if (memcmp(tdfbuf, "@TDF", 4) != 0) { WRMSG (HHC00206, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma"); free (tdfbuf); return -1; } /* Count the number of linefeeds in the tape descriptor file to determine the size of the descriptor array required */ for (i = 0, filecount = 0; i < tdfsize; i++) { if (tdfbuf[i] == '\n') filecount++; } /* end for(i) */ /* ISW Add 1 to filecount to add an extra EOT marker */ filecount++; /* Obtain storage for the tape descriptor array */ tdftab = (OMATAPE_DESC*)malloc (filecount * sizeof(OMATAPE_DESC)); if (tdftab == NULL) { WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", "malloc()", strerror(errno)); free (tdfbuf); return -1; } /* Build the tape descriptor array */ for (filecount = 0, tdfpos = 0, stmt = 1; ; filecount++) { /* Clear the tape descriptor array entry */ memset (&(tdftab[filecount]), 0, sizeof(OMATAPE_DESC)); /* Point past the next linefeed in the TDF file */ while (tdfpos < tdfsize && tdfbuf[tdfpos++] != '\n'); stmt++; /* Exit at end of TDF file */ if (tdfpos >= tdfsize) break; /* Mark the end of the TDF record with a null terminator */ tdfrec = tdfbuf + tdfpos; while (tdfpos < tdfsize && tdfbuf[tdfpos]!='\r' && tdfbuf[tdfpos]!='\n') tdfpos++; c = tdfbuf[tdfpos]; if (tdfpos >= tdfsize) break; tdfbuf[tdfpos] = '\0'; /* Exit if TM or EOT record */ if (strcasecmp(tdfrec, "TM") == 0) { tdftab[filecount].format='X'; tdfbuf[tdfpos] = c; continue; } if(strcasecmp(tdfrec, "EOT") == 0) { tdftab[filecount].format='E'; break; } /* Parse the TDF record */ tdffilenm = strtok_r (tdfrec, " \t", &strtok_str); tdfformat = strtok_r (NULL, " \t", &strtok_str); tdfreckwd = strtok_r (NULL, " \t", &strtok_str); tdfblklen = strtok_r (NULL, " \t", &strtok_str); /* Check for missing fields */ if (tdffilenm == NULL || tdfformat == NULL) { WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, "filename or format missing"); free (tdftab); free (tdfbuf); return -1; } /* Check that the file name is not too long */ if (pathlen + 1 + strlen(tdffilenm) > sizeof(tdftab[filecount].filename) - 1) { char buf[MAX_PATH+32]; if ( strchr(tdffilenm, SPACE) == NULL) MSGBUF(buf, "filename %s too long", tdffilenm); else MSGBUF(buf, "filename '%s' too long", tdffilenm); WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, buf); free (tdftab); free (tdfbuf); return -1; } /* Convert the file name to Unix format */ for (i = 0; i < (int)strlen(tdffilenm); i++) { if (tdffilenm[i] == '\\') tdffilenm[i] = '/'; /* JCS */ // else // tdffilenm[i] = tolower(tdffilenm[i]); } /* end for(i) */ /* Prefix the file name with the base path name and save it in the tape descriptor array */ /* but only if the filename lacks a leading slash - JCS */ /* strncpy (tdftab[filecount].filename, dev->filename, pathlen); if (tdffilenm[0] != '/') stlrcat ( tdftab[filecount].filename, "/", sizeof(tdftab[filecount].filename) ); strlcat ( tdftab[filecount].filename, tdffilenm, sizeof(tdftab[filecount].filename) ); */ tdftab[filecount].filename[0] = 0; if ((tdffilenm[0] != '/') && (tdffilenm[1] != ':')) { strncpy (tdftab[filecount].filename, dev->filename, pathlen); strlcat (tdftab[filecount].filename, "/", sizeof(tdftab[filecount].filename) ); } strlcat (tdftab[filecount].filename, tdffilenm, sizeof(tdftab[filecount].filename) ); /* Check for valid file format code */ if (strcasecmp(tdfformat, "HEADERS") == 0) { tdftab[filecount].format = 'H'; } else if (strcasecmp(tdfformat, "TEXT") == 0) { tdftab[filecount].format = 'T'; } else if (strcasecmp(tdfformat, "FIXED") == 0) { /* Check for RECSIZE keyword */ if (tdfreckwd == NULL || strcasecmp(tdfreckwd, "RECSIZE") != 0) { WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, "keyword RECSIZE missing"); free (tdftab); free (tdfbuf); return -1; } /* Check for valid fixed block length */ if (tdfblklen == NULL || sscanf(tdfblklen, "%u%c", &blklen, &c) != 1 || blklen < 1 || blklen > MAX_BLKLEN) { char buf[40]; MSGBUF(buf, "invalid record size %s", tdfblklen); WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, buf); free (tdftab); free (tdfbuf); return -1; } /* Set format and block length in descriptor array */ tdftab[filecount].format = 'F'; tdftab[filecount].blklen = blklen; } else { char buf[40]; MSGBUF(buf, "invalid record format '%s'", tdfformat); WRMSG (HHC00207, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename, "oma", stmt, buf); free (tdftab); free (tdfbuf); return -1; } tdfbuf[tdfpos] = c; } /* end for(filecount) */ /* Force an EOT as last entry (filecount is correctly adjusted here) */ tdftab[filecount].format='E'; /* Save the file count and TDF array pointer in the device block */ dev->omafiles = filecount+1; dev->omadesc = tdftab; /* Release the TDF file buffer and exit */ free (tdfbuf); return 0; } /* end function read_omadesc */
double pingpongtest(int iters, int msgsz) { int i; int64_t starttime, endtime; int iamsender = (rank % 2 == 0); int iamreceiver = !iamsender || peerid == rank; /* handle loopback */ char *sendMsgbuffer = (char*)malloc(msgsz); char *sendAckbuffer = (char*)malloc(msgsz); char *recvMsgbuffer = (char*)malloc(msgsz); char *recvAckbuffer = (char*)malloc(msgsz); MPI_Request recvMsgHandle = MPI_REQUEST_NULL; MPI_Request recvAckHandle = MPI_REQUEST_NULL; MPI_Request sendMsgHandle = MPI_REQUEST_NULL; MPI_Request sendAckHandle = MPI_REQUEST_NULL; MPI_Status status; #if USE_ZERO_BYTE_ACK #define ACKSZ 0 #else #define ACKSZ msgsz #endif if (iamreceiver) { /* prepost a recv */ MPI_SAFE(MPI_Irecv(recvMsgbuffer, msgsz, MPI_BYTE, peerid, MPI_ANY_TAG, MPI_COMM_WORLD, &recvMsgHandle)); } barrier(); starttime = getMicrosecondTimeStamp(); for (i=0; i < iters; i++) { if (iamsender) { /* send message */ WRITEMSG(sendMsgbuffer, msgsz); #if USE_ISEND MPI_SAFE(MPI_Isend(sendMsgbuffer, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD, &sendMsgHandle)); #else MPI_SAFE(MPI_Send(sendMsgbuffer, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD)); #endif /* prepost a recv for acknowledgement */ MPI_SAFE(MPI_Irecv(recvAckbuffer, ACKSZ, MPI_BYTE, peerid, MPI_ANY_TAG, MPI_COMM_WORLD, &recvAckHandle)); #if USE_ISEND MPI_SAFE(MPI_Wait(&sendMsgHandle, &status)); #endif } if (iamreceiver) { /* wait for message */ #if USE_TEST int flag = 0; while (!flag) MPI_SAFE(MPI_Test(&recvMsgHandle, &flag, &status)); #else MPI_SAFE(MPI_Wait(&recvMsgHandle, &status)); #endif CHECKTAG(status.MPI_TAG); READMSG(recvMsgbuffer, msgsz); /* send acknowledgement */ WRITEMSG(sendAckbuffer, 1); #if USE_ISEND MPI_SAFE(MPI_Isend(sendAckbuffer, ACKSZ, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD, &sendAckHandle)); #else MPI_SAFE(MPI_Send(sendAckbuffer, ACKSZ, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD)); #endif /* pre-post recv for next message */ MPI_SAFE(MPI_Irecv(recvMsgbuffer, msgsz, MPI_BYTE, peerid, MPI_ANY_TAG, MPI_COMM_WORLD, &recvMsgHandle)); #if USE_ISEND MPI_SAFE(MPI_Wait(&sendAckHandle, &status)); #endif } if (iamsender) { /* wait for acknowledgement */ MPI_SAFE(MPI_Wait(&recvAckHandle, &status)); CHECKTAG(status.MPI_TAG); READMSG(recvAckbuffer, 1); } } endtime = getMicrosecondTimeStamp(); /* last recv must be cancelled (not included in timing) */ #if 0 if (iamreceiver) MPI_SAFE(MPI_Cancel(&recvMsgHandle)); #else /* apparently some MPI impls don't implement cancel at all.. (grr..) */ /* use an extra send instead to get the same effect */ if (iamsender) MPI_SAFE(MPI_Send(sendMsgbuffer, msgsz, MPI_BYTE, peerid, peermpitag, MPI_COMM_WORLD)); #endif if (iamreceiver) MPI_SAFE(MPI_Wait(&recvMsgHandle, &status)); free(sendMsgbuffer); free(sendAckbuffer); free(recvMsgbuffer); free(recvAckbuffer); return (double)(endtime - starttime); }
/*-------------------------------------------------------------------*/ int open_omatape (DEVBLK *dev, BYTE *unitstat,BYTE code) { int fd; /* File descriptor integer */ int rc; /* Return code */ OMATAPE_DESC *omadesc; /* -> OMA descriptor entry */ char pathname[MAX_PATH]; /* file path in host format */ /* Check for no tape in drive */ if (!strcmp (dev->filename, TAPE_UNLOADED)) { build_senseX(TAPE_BSENSE_TAPEUNLOADED,dev,unitstat,code); return -1; } /* Read the OMA descriptor file if necessary */ if (dev->omadesc == NULL) { rc = read_omadesc (dev); if (rc < 0) { build_senseX(TAPE_BSENSE_TAPELOADFAIL,dev,unitstat,code); return -1; } dev->blockid = 0; } dev->fenced = 0; /* Unit exception if beyond end of tape */ /* ISW: CHANGED PROCESSING - RETURN UNDEFINITE Tape Marks */ /* NOTE: The last entry in the TDF table is ALWAYS */ /* an EOT Condition */ /* This is ensured by the TDF reading routine */ #if 0 if (dev->curfilen >= dev->omafiles) { WRITEMSG (HHC00000E, SSID_TO_LCSS(dev->ssid), dev->devnum, dev->filename); build_senseX(TAPE_BSENSE_ENDOFTAPE,dev,unitstat,code); return -1; } #else if(dev->curfilen>dev->omafiles) { dev->curfilen=dev->omafiles; return(0); } #endif /* Point to the current file entry in the OMA descriptor table */ omadesc = (OMATAPE_DESC*)(dev->omadesc); omadesc += (dev->curfilen-1); if(omadesc->format=='X') { return 0; } if(omadesc->format=='E') { return 0; } /* Open the OMATAPE file */ hostpath(pathname, omadesc->filename, sizeof(pathname)); fd = HOPEN (pathname, O_RDONLY | O_BINARY); /* Check for successful open */ if (fd < 0 || lseek (fd, 0, SEEK_END) > LONG_MAX) { if (fd >= 0) /* (if open was successful, then it) */ errno = EOVERFLOW; /* (must have been a lseek overflow) */ WRMSG (HHC00205, "E", SSID_TO_LCSS(dev->ssid), dev->devnum, omadesc->filename, "oma", "open()", strerror(errno)); if (fd >= 0) close(fd); /* (close the file if it was opened) */ build_senseX(TAPE_BSENSE_TAPELOADFAIL,dev,unitstat,code); return -1; } /* OMA tapes are always read-only */ dev->readonly = 1; /* Store the file descriptor in the device block */ dev->fd = fd; return 0; } /* end function open_omatape */