double measure_delayed_Alltoall(int send_count, MPI_Datatype send_dt, int recv_count, MPI_Datatype recv_dt, double delay, int node)
{
double start_time, end_time;
start_time = start_synchronization();
if( get_measurement_rank() == node ) while( wtime() < start_time + delay ) ;
MPI_Alltoall(get_send_buffer(), send_count, send_dt,
get_recv_buffer(), recv_count, recv_dt, get_measurement_comm());
end_time = stop_synchronization();
return end_time - start_time;
}
/**
* \brief Measures the time to write once to a large file.
*
* Only one process is active. It writes once to a file.
*
* Since SKaMPI measurement functions are not allowed to use MPI_Offset
* parameters, it is impossible to tell an init_-routine to create a file
* which is larger than \f$2^{\mbox{\texttt{sizeof(int)}}-1}-1\f$ directly. As
* a preliminary solution we use a parameter (<tt>power</tt>) which commits the
* power to 2 as an indicator for the file size.
*
* Remark concerning the <em>HP XC6000</em>:<br>
* Measurements showed that there is no significant difference between MPI-API
* and POSIX-API I/O accesses, if files are larger than 1MB. Thus there is no
* choice between these two modes like in measure_MPI_IO_read_file_once(),
* which makes type compatibilty problems much easier. Only MPI-API is
* supported.
*
* \param[in] power size of memory buffer; 2 to the power of `power' <tt>MPI_BYTE</tt>s
* \param[in] create_flag write into existing file (FALSE) or create it (TRUE)
*
* \return measured time
*/
double measure_MPI_IO_write_large_file_once (int power, int create_flag){
MPI_Offset size;
double start_time = 1.0, end_time = 0.0;
if (get_measurement_rank () == 0){
size = ((MPI_Offset) 1) << power;
if (create_flag == 0){
MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY, MPI_INFO_NULL, &io_fh);
}
else{ /* if create_flag != 0*/
MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY|MPI_MODE_CREATE, MPI_INFO_NULL, &io_fh);
}
MPI_File_set_view (io_fh, (MPI_Offset)0,
MPI_BYTE, MPI_BYTE,
"native", MPI_INFO_NULL);
start_time = start_synchronization ();
MPI_File_write (io_fh, get_send_buffer (), size, MPI_BYTE, MPI_STATUS_IGNORE);
end_time = MPI_Wtime ();
MPI_File_close (&io_fh);
}
else if (get_measurement_rank () != 0) {
start_synchronization ();
}
stop_synchronization ();
if (get_measurement_rank () == 0)
return end_time - start_time;
else
return -1.0;
}
// Send and recv functions. The functions simply write or read to our buffers
// and return the corresponting mbedtls want read/write in case the buffer is
// full/empty.
int dart_send(void *ctx, const unsigned char *buf, size_t len) {
char *send_buffer = get_send_buffer(ctx);
int written = buffer_write(send_buffer, buf, len, MBEDTLS_ERR_SSL_WANT_WRITE);
return written;
}
/**
* \brief Measures the time to write once to a file.
*
* Only one process is active. It writes once to a file.
*
* Remark:<br>
* With the <tt>O_DIRECT</tt> flag set, cache effects are minimized, because I/O
* is done directly to/from user space buffers. The operation system's page
* cache is bypassed. Under Linux 2.6 alignment to 512-byte boundaries is
* required for buffer and file offset. Thus the following parameters should be
* set in a SKaMPI input file:
* - <tt>set_send_buffert_alignment (512)</tt>
* - <tt>set_recv_buffert_alignment (512)</tt>
* - <tt>switch_buffer_cycling_off ()</tt><br>
*
* <tt>O_DIRECT</tt> is only relevant if the POSIX-API is used for I/O.
*
* For more information please refer to the <tt>open ()</tt> man pages.
*
* \param[in] size size of memory buffer, i.e. number of <tt>MPI_BYTE</tt>s
* \param[in] api POSIX-API or MPI-API for I/O accesses
* \param[in] create_flag write into existing file (FALSE) or create it (TRUE)
* \param[in] directio_flag open file with <tt>O_DIRECT</tt> flag to minimize
* cache effects
*
* \return measured time
*/
double measure_MPI_IO_write_file_once (int size, char *api, int create_flag, int directio_flag){
double start_time = 1.0, end_time = 0.0;
int open_flags;
char *error_string;
if (get_measurement_rank () == 0){
if (strcmp (api, POSIX_API) == 0){
if (directio_flag != 0)
open_flags = O_WRONLY | O_DIRECT;
else
open_flags = O_WRONLY;
errno = 0;
if (create_flag == 0){ /* open existing file */
if ((io_fd = open (io_filename, open_flags)) < 0){
error_string = strerror (errno);
error_with_abort (errno,
"\nmeasure_MPI_IO_write_file_once (int %d, char * %s, int %d, int %d) failed."
"\nCannot open local file (write only mode)."
"\nError: %s\n",
size, api, create_flag, directio_flag, error_string);
}
}
else { /* open nonexisting file and create it */
if ((io_fd = open (io_filename, open_flags|O_CREAT, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) < 0){
error_string = strerror (errno);
error_with_abort (errno,
"\nmeasure_MPI_IO_write_file_once (int %d, char * %s, int %d, int %d) failed."
"\nCannot open local file (write only mode)."
"\nError: %s\n",
size, api, create_flag, directio_flag, error_string);
}
}
start_time = start_synchronization ();
write (io_fd, get_send_buffer (), size);
fsync (io_fd);
end_time = MPI_Wtime ();
close (io_fd);
}
else{ /* if strcmp (api, POSIX_API) != 0 */
if (create_flag == 0){
MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY, MPI_INFO_NULL, &io_fh);
}
else{ /* if create_flag != 0*/
MPI_File_open (MPI_COMM_SELF, io_filename, MPI_MODE_WRONLY|MPI_MODE_CREATE, MPI_INFO_NULL, &io_fh);
}
MPI_File_set_view (io_fh, (MPI_Offset)0,
MPI_BYTE, MPI_BYTE,
"native", MPI_INFO_NULL);
start_time = start_synchronization ();
MPI_File_write (io_fh, get_send_buffer (), size, MPI_BYTE, MPI_STATUS_IGNORE);
MPI_File_sync (io_fh);
end_time = MPI_Wtime ();
MPI_File_close (&io_fh);
}
}
else if (get_measurement_rank () != 0) {
start_synchronization ();
}
stop_synchronization ();
if (get_measurement_rank () == 0)
return end_time - start_time;
else
return -1.0;
}
