int mca_coll_ml_allgather(const void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void* rbuf, int rcount,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
ompi_request_t *req;
int ret;
ML_VERBOSE(10, ("Starting blocking allgather"));
ret = mca_coll_ml_allgather_start (sbuf, scount, sdtype,
rbuf, rcount, rdtype,
comm, module, &req);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return ret;
}
ret = ompi_request_wait (&req, MPI_STATUS_IGNORE);
ML_VERBOSE(10, ("Blocking allgather is complete"));
return ret;
}
int mca_coll_ml_reduce(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype, struct ompi_op_t *op,
int root, struct ompi_communicator_t *comm,
mca_coll_base_module_t *module) {
mca_coll_ml_module_t *ml_module = (mca_coll_ml_module_t*)module;
int ret = OMPI_SUCCESS;
ompi_request_t *req;
if (OPAL_UNLIKELY(!ompi_op_is_commute(op) || !opal_datatype_is_contiguous_memory_layout(&dtype->super, count))) {
/* coll/ml does not handle non-communative operations at this time. fallback
* on another collective module */
return ml_module->fallback.coll_reduce (sbuf, rbuf, count, dtype, op, root, comm,
ml_module->fallback.coll_reduce_module);
}
ML_VERBOSE(10,("Calling Ml Reduce "));
ret = parallel_reduce_start(sbuf, rbuf, count, dtype, op,
root, comm, (mca_coll_ml_module_t *)module,
&req, ML_SMALL_DATA_REDUCE,
ML_LARGE_DATA_REDUCE);
if (OPAL_UNLIKELY(ret != OMPI_SUCCESS)) {
ML_VERBOSE(10, ("Failed to launch"));
return ret;
}
/* Blocking reduce */
ret = ompi_request_wait(&req, MPI_STATUS_IGNORE);
ML_VERBOSE(10, ("Blocking Reduce is done"));
return ret;
}
int mca_pml_ucx_send(const void *buf, size_t count, ompi_datatype_t *datatype, int dst,
int tag, mca_pml_base_send_mode_t mode,
struct ompi_communicator_t* comm)
{
ompi_request_t *req;
ucp_ep_h ep;
PML_UCX_TRACE_SEND("%s", buf, count, datatype, dst, tag, mode, comm, "send");
/* TODO special care to sync/buffered send */
ep = mca_pml_ucx_get_ep(comm, dst);
if (OPAL_UNLIKELY(NULL == ep)) {
PML_UCX_ERROR("Failed to get ep for rank %d", dst);
return OMPI_ERROR;
}
req = (ompi_request_t*)ucp_tag_send_nb(ep, buf, count,
mca_pml_ucx_get_datatype(datatype),
PML_UCX_MAKE_SEND_TAG(tag, comm),
mca_pml_ucx_send_completion);
if (OPAL_LIKELY(req == NULL)) {
return OMPI_SUCCESS;
} else if (!UCS_PTR_IS_ERR(req)) {
PML_UCX_VERBOSE(8, "got request %p", (void*)req);
ucp_worker_progress(ompi_pml_ucx.ucp_worker);
ompi_request_wait(&req, MPI_STATUS_IGNORE);
return OMPI_SUCCESS;
} else {
PML_UCX_ERROR("ucx send failed: %s", ucs_status_string(UCS_PTR_STATUS(req)));
return OMPI_ERROR;
}
}
int mca_sharedfp_sm_read_ordered_end(mca_io_ompio_file_t *fh,
void *buf,
ompi_status_public_t *status)
{
int ret = OMPI_SUCCESS;
ret = ompi_request_wait ( &fh->f_split_coll_req, status );
/* remove the flag again */
fh->f_split_coll_in_use = false;
return ret;
}
int ompi_coll_base_sendrecv_actual( const void* sendbuf, size_t scount,
ompi_datatype_t* sdatatype,
int dest, int stag,
void* recvbuf, size_t rcount,
ompi_datatype_t* rdatatype,
int source, int rtag,
struct ompi_communicator_t* comm,
ompi_status_public_t* status )
{ /* post receive first, then send, then wait... should be fast (I hope) */
int err, line = 0;
size_t rtypesize, stypesize;
ompi_request_t *req;
ompi_status_public_t rstatus;
/* post new irecv */
ompi_datatype_type_size(rdatatype, &rtypesize);
err = MCA_PML_CALL(irecv( recvbuf, rcount, rdatatype, source, rtag,
comm, &req));
if (err != MPI_SUCCESS) { line = __LINE__; goto error_handler; }
/* send data to children */
ompi_datatype_type_size(sdatatype, &stypesize);
err = MCA_PML_CALL(send( sendbuf, scount, sdatatype, dest, stag,
MCA_PML_BASE_SEND_STANDARD, comm));
if (err != MPI_SUCCESS) { line = __LINE__; goto error_handler; }
err = ompi_request_wait( &req, &rstatus);
if (err != MPI_SUCCESS) { line = __LINE__; goto error_handler; }
if (MPI_STATUS_IGNORE != status) {
*status = rstatus;
}
return (MPI_SUCCESS);
error_handler:
/* Error discovered during the posting of the irecv or send,
* and no status is available.
*/
OPAL_OUTPUT ((ompi_coll_base_framework.framework_output, "%s:%d: Error %d occurred\n",
__FILE__, line, err));
(void)line; // silence compiler warning
if (MPI_STATUS_IGNORE != status) {
status->MPI_ERROR = err;
}
return (err);
}
int mca_io_ompio_file_read_at_all_end (ompi_file_t *fh,
void *buf,
ompi_status_public_t * status)
{
int ret = OMPI_SUCCESS;
mca_io_ompio_data_t *data;
mca_io_ompio_file_t *fp=NULL;
data = (mca_io_ompio_data_t *) fh->f_io_selected_data;
fp = &data->ompio_fh;
ret = ompi_request_wait ( &fp->f_split_coll_req, status );
/* remove the flag again */
fp->f_split_coll_in_use = false;
return ret;
}
int mca_pml_ucx_mrecv(void *buf, size_t count, ompi_datatype_t *datatype,
struct ompi_message_t **message,
ompi_status_public_t* status)
{
ompi_request_t *req;
PML_UCX_TRACE_MRECV("mrecv", buf, count, datatype, message);
req = (ompi_request_t*)ucp_tag_msg_recv_nb(ompi_pml_ucx.ucp_worker, buf, count,
mca_pml_ucx_get_datatype(datatype),
(*message)->req_ptr,
mca_pml_ucx_recv_completion);
if (UCS_PTR_IS_ERR(req)) {
PML_UCX_ERROR("ucx msg recv failed: %s", ucs_status_string(UCS_PTR_STATUS(req)));
return OMPI_ERROR;
}
PML_UCX_MESSAGE_RELEASE(message);
ompi_request_wait(&req, status);
return OMPI_SUCCESS;
}
int
mca_fcoll_dynamic_gen2_file_read_all (mca_io_ompio_file_t *fh,
void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
MPI_Aint position = 0;
MPI_Aint total_bytes = 0; /* total bytes to be read */
MPI_Aint bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
MPI_Aint bytes_per_cycle = 0; /* total read in each cycle by each process*/
int index = 0, ret=OMPI_SUCCESS;
int cycles = 0;
int i=0, j=0, l=0;
int n=0; /* current position in total_bytes_per_process array */
MPI_Aint bytes_remaining = 0; /* how many bytes have been read from the current
value from total_bytes_per_process */
int *sorted_file_offsets=NULL, entries_per_aggregator=0;
int bytes_received = 0;
int blocks = 0;
/* iovec structure and count of the buffer passed in */
uint32_t iov_count = 0;
struct iovec *decoded_iov = NULL;
int iov_index = 0;
size_t current_position = 0;
struct iovec *local_iov_array=NULL, *global_iov_array=NULL;
char *receive_buf = NULL;
MPI_Aint *memory_displacements=NULL;
/* global iovec at the readers that contain the iovecs created from
file_set_view */
uint32_t total_fview_count = 0;
int local_count = 0;
int *fview_count = NULL, *disp_index=NULL, *temp_disp_index=NULL;
int current_index=0, temp_index=0;
int **blocklen_per_process=NULL;
MPI_Aint **displs_per_process=NULL;
char *global_buf = NULL;
MPI_Aint global_count = 0;
mca_io_ompio_local_io_array *file_offsets_for_agg=NULL;
/* array that contains the sorted indices of the global_iov */
int *sorted = NULL;
int *displs = NULL;
int dynamic_gen2_num_io_procs;
size_t max_data = 0;
MPI_Aint *total_bytes_per_process = NULL;
ompi_datatype_t **sendtype = NULL;
MPI_Request *send_req=NULL, recv_req=NULL;
int my_aggregator =-1;
bool recvbuf_is_contiguous=false;
size_t ftype_size;
OPAL_PTRDIFF_TYPE ftype_extent, lb;
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
double read_time = 0.0, start_read_time = 0.0, end_read_time = 0.0;
double rcomm_time = 0.0, start_rcomm_time = 0.0, end_rcomm_time = 0.0;
double read_exch = 0.0, start_rexch = 0.0, end_rexch = 0.0;
mca_io_ompio_print_entry nentry;
#endif
/**************************************************************************
** 1. In case the data is not contigous in memory, decode it into an iovec
**************************************************************************/
opal_datatype_type_size ( &datatype->super, &ftype_size );
opal_datatype_get_extent ( &datatype->super, &lb, &ftype_extent );
if ( (ftype_extent == (OPAL_PTRDIFF_TYPE) ftype_size) &&
opal_datatype_is_contiguous_memory_layout(&datatype->super,1) &&
0 == lb ) {
recvbuf_is_contiguous = true;
}
if (! recvbuf_is_contiguous ) {
ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *)fh,
datatype,
count,
buf,
&max_data,
&decoded_iov,
&iov_count);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
else {
max_data = count * datatype->super.size;
}
if ( MPI_STATUS_IGNORE != status ) {
status->_ucount = max_data;
}
fh->f_get_num_aggregators ( &dynamic_gen2_num_io_procs);
ret = fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *) fh,
dynamic_gen2_num_io_procs,
max_data);
if (OMPI_SUCCESS != ret){
goto exit;
}
my_aggregator = fh->f_procs_in_group[fh->f_aggregator_index];
/**************************************************************************
** 2. Determine the total amount of data to be written
**************************************************************************/
total_bytes_per_process = (MPI_Aint*)malloc(fh->f_procs_per_group*sizeof(MPI_Aint));
if (NULL == total_bytes_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rcomm_time = MPI_Wtime();
#endif
ret = fcoll_base_coll_allgather_array (&max_data,
1,
MPI_LONG,
total_bytes_per_process,
1,
MPI_LONG,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
for (i=0 ; i<fh->f_procs_per_group ; i++) {
total_bytes += total_bytes_per_process[i];
}
if (NULL != total_bytes_per_process) {
free (total_bytes_per_process);
total_bytes_per_process = NULL;
}
/*********************************************************************
*** 3. Generate the File offsets/lengths corresponding to this write
********************************************************************/
ret = fh->f_generate_current_file_view ((struct mca_io_ompio_file_t *) fh,
max_data,
&local_iov_array,
&local_count);
if (ret != OMPI_SUCCESS){
goto exit;
}
/*************************************************************
*** 4. Allgather the File View information at all processes
*************************************************************/
fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == fview_count) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rcomm_time = MPI_Wtime();
#endif
ret = fcoll_base_coll_allgather_array (&local_count,
1,
MPI_INT,
fview_count,
1,
MPI_INT,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
displs = (int*)malloc (fh->f_procs_per_group*sizeof(int));
if (NULL == displs) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs[0] = 0;
total_fview_count = fview_count[0];
for (i=1 ; i<fh->f_procs_per_group ; i++) {
total_fview_count += fview_count[i];
displs[i] = displs[i-1] + fview_count[i-1];
}
#if DEBUG_ON
if (my_aggregator == fh->f_rank) {
for (i=0 ; i<fh->f_procs_per_group ; i++) {
printf ("%d: PROCESS: %d ELEMENTS: %d DISPLS: %d\n",
fh->f_rank,
i,
fview_count[i],
displs[i]);
}
}
#endif
/* allocate the global iovec */
if (0 != total_fview_count) {
global_iov_array = (struct iovec*)malloc (total_fview_count *
sizeof(struct iovec));
if (NULL == global_iov_array) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rcomm_time = MPI_Wtime();
#endif
ret = fcoll_base_coll_allgatherv_array (local_iov_array,
local_count,
fh->f_iov_type,
global_iov_array,
fview_count,
displs,
fh->f_iov_type,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
/****************************************************************************************
*** 5. Sort the global offset/lengths list based on the offsets.
*** The result of the sort operation is the 'sorted', an integer array,
*** which contains the indexes of the global_iov_array based on the offset.
*** For example, if global_iov_array[x].offset is followed by global_iov_array[y].offset
*** in the file, and that one is followed by global_iov_array[z].offset, than
*** sorted[0] = x, sorted[1]=y and sorted[2]=z;
******************************************************************************************/
if (0 != total_fview_count) {
sorted = (int *)malloc (total_fview_count * sizeof(int));
if (NULL == sorted) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
fh->f_sort_iovec (global_iov_array, total_fview_count, sorted);
}
if (NULL != local_iov_array) {
free (local_iov_array);
local_iov_array = NULL;
}
#if DEBUG_ON
if (my_aggregator == fh->f_rank) {
for (i=0 ; i<total_fview_count ; i++) {
printf("%d: OFFSET: %p LENGTH: %d\n",
fh->f_rank,
global_iov_array[sorted[i]].iov_base,
global_iov_array[sorted[i]].iov_len);
}
}
#endif
/*************************************************************
*** 6. Determine the number of cycles required to execute this
*** operation
*************************************************************/
fh->f_get_bytes_per_agg ( (int *) &bytes_per_cycle);
cycles = ceil((double)total_bytes/bytes_per_cycle);
if ( my_aggregator == fh->f_rank) {
disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
blocklen_per_process = (int **)malloc (fh->f_procs_per_group * sizeof (int*));
if (NULL == blocklen_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process = (MPI_Aint **)malloc (fh->f_procs_per_group * sizeof (MPI_Aint*));
if (NULL == displs_per_process){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for (i=0;i<fh->f_procs_per_group;i++){
blocklen_per_process[i] = NULL;
displs_per_process[i] = NULL;
}
send_req = (MPI_Request *) malloc (fh->f_procs_per_group * sizeof(MPI_Request));
if (NULL == send_req){
opal_output ( 1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
global_buf = (char *) malloc (bytes_per_cycle);
if (NULL == global_buf){
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
sendtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
if (NULL == sendtype) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for(l=0;l<fh->f_procs_per_group;l++){
sendtype[l] = MPI_DATATYPE_NULL;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rexch = MPI_Wtime();
#endif
n = 0;
bytes_remaining = 0;
current_index = 0;
for (index = 0; index < cycles; index++) {
/**********************************************************************
*** 7a. Getting ready for next cycle: initializing and freeing buffers
**********************************************************************/
if (my_aggregator == fh->f_rank) {
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
fh->f_num_of_io_entries = 0;
if (NULL != sendtype){
for (i =0; i< fh->f_procs_per_group; i++) {
if ( MPI_DATATYPE_NULL != sendtype[i] ) {
ompi_datatype_destroy(&sendtype[i]);
sendtype[i] = MPI_DATATYPE_NULL;
}
}
}
for(l=0;l<fh->f_procs_per_group;l++){
disp_index[l] = 1;
if (NULL != blocklen_per_process[l]){
free(blocklen_per_process[l]);
blocklen_per_process[l] = NULL;
}
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
if (NULL == blocklen_per_process[l]) {
opal_output (1, "OUT OF MEMORY for blocklen\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
if (NULL == displs_per_process[l]){
opal_output (1, "OUT OF MEMORY for displs\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if(NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements = NULL;
}
} /* (my_aggregator == fh->f_rank */
/**************************************************************************
*** 7b. Determine the number of bytes to be actually read in this cycle
**************************************************************************/
if (cycles-1 == index) {
bytes_to_read_in_cycle = total_bytes - bytes_per_cycle*index;
}
else {
bytes_to_read_in_cycle = bytes_per_cycle;
}
#if DEBUG_ON
if (my_aggregator == fh->f_rank) {
printf ("****%d: CYCLE %d Bytes %d**********\n",
fh->f_rank,
index,
bytes_to_write_in_cycle);
}
#endif
/*****************************************************************
*** 7c. Calculate how much data will be contributed in this cycle
*** by each process
*****************************************************************/
bytes_received = 0;
while (bytes_to_read_in_cycle) {
/* This next block identifies which process is the holder
** of the sorted[current_index] element;
*/
blocks = fview_count[0];
for (j=0 ; j<fh->f_procs_per_group ; j++) {
if (sorted[current_index] < blocks) {
n = j;
break;
}
else {
blocks += fview_count[j+1];
}
}
if (bytes_remaining) {
/* Finish up a partially used buffer from the previous cycle */
if (bytes_remaining <= bytes_to_read_in_cycle) {
/* Data fits completely into the block */
if (my_aggregator == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_remaining;
displs_per_process[n][disp_index[n] - 1] =
(OPAL_PTRDIFF_TYPE)global_iov_array[sorted[current_index]].iov_base +
(global_iov_array[sorted[current_index]].iov_len - bytes_remaining);
blocklen_per_process[n] = (int *) realloc
((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
displs_per_process[n] = (MPI_Aint *) realloc
((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
blocklen_per_process[n][disp_index[n]] = 0;
displs_per_process[n][disp_index[n]] = 0;
disp_index[n] += 1;
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_received += bytes_remaining;
}
current_index ++;
bytes_to_read_in_cycle -= bytes_remaining;
bytes_remaining = 0;
continue;
}
else {
/* the remaining data from the previous cycle is larger than the
bytes_to_write_in_cycle, so we have to segment again */
if (my_aggregator == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
displs_per_process[n][disp_index[n] - 1] =
(OPAL_PTRDIFF_TYPE)global_iov_array[sorted[current_index]].iov_base +
(global_iov_array[sorted[current_index]].iov_len
- bytes_remaining);
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_received += bytes_to_read_in_cycle;
}
bytes_remaining -= bytes_to_read_in_cycle;
bytes_to_read_in_cycle = 0;
break;
}
}
else {
/* No partially used entry available, have to start a new one */
if (bytes_to_read_in_cycle <
(MPI_Aint) global_iov_array[sorted[current_index]].iov_len) {
/* This entry has more data than we can sendin one cycle */
if (my_aggregator == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
displs_per_process[n][disp_index[n] - 1] =
(OPAL_PTRDIFF_TYPE)global_iov_array[sorted[current_index]].iov_base ;
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_received += bytes_to_read_in_cycle;
}
bytes_remaining = global_iov_array[sorted[current_index]].iov_len -
bytes_to_read_in_cycle;
bytes_to_read_in_cycle = 0;
break;
}
else {
/* Next data entry is less than bytes_to_write_in_cycle */
if (my_aggregator == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] =
global_iov_array[sorted[current_index]].iov_len;
displs_per_process[n][disp_index[n] - 1] = (OPAL_PTRDIFF_TYPE)
global_iov_array[sorted[current_index]].iov_base;
blocklen_per_process[n] =
(int *) realloc ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
displs_per_process[n] = (MPI_Aint *)realloc
((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
blocklen_per_process[n][disp_index[n]] = 0;
displs_per_process[n][disp_index[n]] = 0;
disp_index[n] += 1;
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_received +=
global_iov_array[sorted[current_index]].iov_len;
}
bytes_to_read_in_cycle -=
global_iov_array[sorted[current_index]].iov_len;
current_index ++;
continue;
}
}
} /* end while (bytes_to_read_in_cycle) */
/*************************************************************************
*** 7d. Calculate the displacement on where to put the data and allocate
*** the recieve buffer (global_buf)
*************************************************************************/
if (my_aggregator == fh->f_rank) {
entries_per_aggregator=0;
for (i=0;i<fh->f_procs_per_group; i++){
for (j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0)
entries_per_aggregator++ ;
}
}
if (entries_per_aggregator > 0){
file_offsets_for_agg = (mca_io_ompio_local_io_array *)
malloc(entries_per_aggregator*sizeof(mca_io_ompio_local_io_array));
if (NULL == file_offsets_for_agg) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
sorted_file_offsets = (int *)
malloc (entries_per_aggregator*sizeof(int));
if (NULL == sorted_file_offsets){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
/*Moving file offsets to an IO array!*/
temp_index = 0;
global_count = 0;
for (i=0;i<fh->f_procs_per_group; i++){
for(j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0){
file_offsets_for_agg[temp_index].length =
blocklen_per_process[i][j];
global_count += blocklen_per_process[i][j];
file_offsets_for_agg[temp_index].process_id = i;
file_offsets_for_agg[temp_index].offset =
displs_per_process[i][j];
temp_index++;
}
}
}
}
else{
continue;
}
/* Sort the displacements for each aggregator */
read_heap_sort (file_offsets_for_agg,
entries_per_aggregator,
sorted_file_offsets);
memory_displacements = (MPI_Aint *) malloc
(entries_per_aggregator * sizeof(MPI_Aint));
memory_displacements[sorted_file_offsets[0]] = 0;
for (i=1; i<entries_per_aggregator; i++){
memory_displacements[sorted_file_offsets[i]] =
memory_displacements[sorted_file_offsets[i-1]] +
file_offsets_for_agg[sorted_file_offsets[i-1]].length;
}
/**********************************************************
*** 7e. Create the io array, and pass it to fbtl
*********************************************************/
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
(entries_per_aggregator * sizeof (mca_io_ompio_io_array_t));
if (NULL == fh->f_io_array) {
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
fh->f_num_of_io_entries = 0;
fh->f_io_array[0].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
fh->f_io_array[0].length =
file_offsets_for_agg[sorted_file_offsets[0]].length;
fh->f_io_array[0].memory_address =
global_buf+memory_displacements[sorted_file_offsets[0]];
fh->f_num_of_io_entries++;
for (i=1;i<entries_per_aggregator;i++){
if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
file_offsets_for_agg[sorted_file_offsets[i]].offset){
fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
file_offsets_for_agg[sorted_file_offsets[i]].length;
}
else{
fh->f_io_array[fh->f_num_of_io_entries].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
fh->f_io_array[fh->f_num_of_io_entries].length =
file_offsets_for_agg[sorted_file_offsets[i]].length;
fh->f_io_array[fh->f_num_of_io_entries].memory_address =
global_buf+memory_displacements[sorted_file_offsets[i]];
fh->f_num_of_io_entries++;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_read_time = MPI_Wtime();
#endif
if (fh->f_num_of_io_entries) {
if ( 0 > fh->f_fbtl->fbtl_preadv (fh)) {
opal_output (1, "READ FAILED\n");
ret = OMPI_ERROR;
goto exit;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_read_time = MPI_Wtime();
read_time += end_read_time - start_read_time;
#endif
/**********************************************************
******************** DONE READING ************************
*********************************************************/
temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
if (NULL == temp_disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for (i=0; i<entries_per_aggregator; i++){
temp_index =
file_offsets_for_agg[sorted_file_offsets[i]].process_id;
displs_per_process[temp_index][temp_disp_index[temp_index]] =
memory_displacements[sorted_file_offsets[i]];
if (temp_disp_index[temp_index] < disp_index[temp_index]){
temp_disp_index[temp_index] += 1;
}
else{
printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
temp_index, temp_disp_index[temp_index],
temp_index, disp_index[temp_index]);
}
}
if (NULL != temp_disp_index){
free(temp_disp_index);
temp_disp_index = NULL;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rcomm_time = MPI_Wtime();
#endif
for (i=0;i<fh->f_procs_per_group;i++){
send_req[i] = MPI_REQUEST_NULL;
if ( 0 < disp_index[i] ) {
ompi_datatype_create_hindexed(disp_index[i],
blocklen_per_process[i],
displs_per_process[i],
MPI_BYTE,
&sendtype[i]);
ompi_datatype_commit(&sendtype[i]);
ret = MCA_PML_CALL (isend(global_buf,
1,
sendtype[i],
fh->f_procs_in_group[i],
123,
MCA_PML_BASE_SEND_STANDARD,
fh->f_comm,
&send_req[i]));
if(OMPI_SUCCESS != ret){
goto exit;
}
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
}
/**********************************************************
*** 7f. Scatter the Data from the readers
*********************************************************/
if ( recvbuf_is_contiguous ) {
receive_buf = &((char*)buf)[position];
}
else if (bytes_received) {
/* allocate a receive buffer and copy the data that needs
to be received into it in case the data is non-contigous
in memory */
receive_buf = malloc (bytes_received);
if (NULL == receive_buf) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rcomm_time = MPI_Wtime();
#endif
ret = MCA_PML_CALL(irecv(receive_buf,
bytes_received,
MPI_BYTE,
my_aggregator,
123,
fh->f_comm,
&recv_req));
if (OMPI_SUCCESS != ret){
goto exit;
}
if (my_aggregator == fh->f_rank){
ret = ompi_request_wait_all (fh->f_procs_per_group,
send_req,
MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
ret = ompi_request_wait (&recv_req, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
position += bytes_received;
/* If data is not contigous in memory, copy the data from the
receive buffer into the buffer passed in */
if (!recvbuf_is_contiguous ) {
OPAL_PTRDIFF_TYPE mem_address;
size_t remaining = 0;
size_t temp_position = 0;
remaining = bytes_received;
while (remaining) {
mem_address = (OPAL_PTRDIFF_TYPE)
(decoded_iov[iov_index].iov_base) + current_position;
if (remaining >=
(decoded_iov[iov_index].iov_len - current_position)) {
memcpy ((IOVBASE_TYPE *) mem_address,
receive_buf+temp_position,
decoded_iov[iov_index].iov_len - current_position);
remaining = remaining -
(decoded_iov[iov_index].iov_len - current_position);
temp_position = temp_position +
(decoded_iov[iov_index].iov_len - current_position);
iov_index = iov_index + 1;
current_position = 0;
}
else {
memcpy ((IOVBASE_TYPE *) mem_address,
receive_buf+temp_position,
remaining);
current_position = current_position + remaining;
remaining = 0;
}
}
if (NULL != receive_buf) {
free (receive_buf);
receive_buf = NULL;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
} /* end for (index=0; index < cycles; index ++) */
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rexch = MPI_Wtime();
read_exch += end_rexch - start_rexch;
nentry.time[0] = read_time;
nentry.time[1] = rcomm_time;
nentry.time[2] = read_exch;
if (my_aggregator == fh->f_rank)
nentry.aggregator = 1;
else
nentry.aggregator = 0;
nentry.nprocs_for_coll = dynamic_gen2_num_io_procs;
if (!fh->f_full_print_queue(READ_PRINT_QUEUE)){
fh->f_register_print_entry(READ_PRINT_QUEUE,
nentry);
}
#endif
exit:
if (!recvbuf_is_contiguous) {
if (NULL != receive_buf) {
free (receive_buf);
receive_buf = NULL;
}
}
if (NULL != global_buf) {
free (global_buf);
global_buf = NULL;
}
if (NULL != sorted) {
free (sorted);
sorted = NULL;
}
if (NULL != global_iov_array) {
free (global_iov_array);
global_iov_array = NULL;
}
if (NULL != fview_count) {
free (fview_count);
fview_count = NULL;
}
if (NULL != decoded_iov) {
free (decoded_iov);
decoded_iov = NULL;
}
if (NULL != local_iov_array){
free(local_iov_array);
local_iov_array=NULL;
}
if (NULL != displs) {
free (displs);
displs = NULL;
}
if (my_aggregator == fh->f_rank) {
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if (NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements= NULL;
}
if (NULL != sendtype){
for (i = 0; i < fh->f_procs_per_group; i++) {
if ( MPI_DATATYPE_NULL != sendtype[i] ) {
ompi_datatype_destroy(&sendtype[i]);
}
}
free(sendtype);
sendtype=NULL;
}
if (NULL != disp_index){
free(disp_index);
disp_index = NULL;
}
if ( NULL != blocklen_per_process){
for(l=0;l<fh->f_procs_per_group;l++){
if (NULL != blocklen_per_process[l]){
free(blocklen_per_process[l]);
blocklen_per_process[l] = NULL;
}
}
free(blocklen_per_process);
blocklen_per_process = NULL;
}
if (NULL != displs_per_process){
for (l=0; i<fh->f_procs_per_group; l++){
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
}
free(displs_per_process);
displs_per_process = NULL;
}
if ( NULL != send_req ) {
free ( send_req );
send_req = NULL;
}
}
return ret;
}
int
mca_fcoll_dynamic_file_read_all (mca_io_ompio_file_t *fh,
void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
MPI_Aint position = 0;
MPI_Aint total_bytes = 0; /* total bytes to be read */
MPI_Aint bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
MPI_Aint bytes_per_cycle = 0; /* total read in each cycle by each process*/
int index = 0, ret=OMPI_SUCCESS;
int cycles = 0;
int i=0, j=0, l=0;
int n=0; /* current position in total_bytes_per_process array */
MPI_Aint bytes_remaining = 0; /* how many bytes have been read from the current
value from total_bytes_per_process */
int *sorted_file_offsets=NULL, entries_per_aggregator=0;
int bytes_received = 0;
int blocks = 0;
/* iovec structure and count of the buffer passed in */
uint32_t iov_count = 0;
struct iovec *decoded_iov = NULL;
int iov_index = 0;
size_t current_position = 0;
struct iovec *local_iov_array=NULL, *global_iov_array=NULL;
char *receive_buf = NULL;
MPI_Aint *memory_displacements=NULL;
/* global iovec at the readers that contain the iovecs created from
file_set_view */
uint32_t total_fview_count = 0;
int local_count = 0;
int *fview_count = NULL, *disp_index=NULL, *temp_disp_index=NULL;
int current_index=0, temp_index=0;
int **blocklen_per_process=NULL;
MPI_Aint **displs_per_process=NULL;
char *global_buf = NULL;
MPI_Aint global_count = 0;
local_io_array *file_offsets_for_agg=NULL;
/* array that contains the sorted indices of the global_iov */
int *sorted = NULL;
int *displs = NULL;
int dynamic_num_io_procs;
size_t max_data = 0;
int *bytes_per_process = NULL;
MPI_Aint *total_bytes_per_process = NULL;
ompi_datatype_t **sendtype = NULL;
MPI_Request *send_req=NULL, *recv_req=NULL;
#if TIME_BREAKDOWN
double read_time = 0.0, start_read_time = 0.0, end_read_time = 0.0;
double rcomm_time = 0.0, start_rcomm_time = 0.0, end_rcomm_time = 0.0;
double read_exch = 0.0, start_rexch = 0.0, end_rexch = 0.0;
print_entry nentry;
#endif
// if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
// fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
// }
/**************************************************************************
** In case the data is not contigous in memory, decode it into an iovec **
**************************************************************************/
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
ret = fh->f_decode_datatype ((struct mca_io_ompio_file_t *)fh,
datatype,
count,
buf,
&max_data,
&decoded_iov,
&iov_count);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
else {
max_data = count * datatype->super.size;
}
if ( MPI_STATUS_IGNORE != status ) {
status->_ucount = max_data;
}
fh->f_get_num_aggregators ( &dynamic_num_io_procs);
ret = fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *) fh,
dynamic_num_io_procs,
max_data);
if (OMPI_SUCCESS != ret){
goto exit;
}
total_bytes_per_process = (MPI_Aint*)malloc
(fh->f_procs_per_group*sizeof(MPI_Aint));
if (NULL == total_bytes_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ret = fh->f_allgather_array (&max_data,
1,
MPI_LONG,
total_bytes_per_process,
1,
MPI_LONG,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
goto exit;
}
for (i=0 ; i<fh->f_procs_per_group ; i++) {
total_bytes += total_bytes_per_process[i];
}
if (NULL != total_bytes_per_process) {
free (total_bytes_per_process);
total_bytes_per_process = NULL;
}
/*********************************************************************
*** Generate the File offsets/lengths corresponding to this write ***
********************************************************************/
ret = fh->f_generate_current_file_view ((struct mca_io_ompio_file_t *) fh,
max_data,
&local_iov_array,
&local_count);
if (ret != OMPI_SUCCESS){
goto exit;
}
/* #########################################################*/
/*************************************************************
*** ALLGather the File View information at all processes ***
*************************************************************/
fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == fview_count) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ret = fh->f_allgather_array (&local_count,
1,
MPI_INT,
fview_count,
1,
MPI_INT,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
goto exit;
}
displs = (int*)malloc (fh->f_procs_per_group*sizeof(int));
if (NULL == displs) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs[0] = 0;
total_fview_count = fview_count[0];
for (i=1 ; i<fh->f_procs_per_group ; i++) {
total_fview_count += fview_count[i];
displs[i] = displs[i-1] + fview_count[i-1];
}
#if DEBUG_ON
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
for (i=0 ; i<fh->f_procs_per_group ; i++) {
printf ("%d: PROCESS: %d ELEMENTS: %d DISPLS: %d\n",
fh->f_rank,
i,
fview_count[i],
displs[i]);
}
}
#endif
/* allocate the global iovec */
if (0 != total_fview_count) {
global_iov_array = (struct iovec*)malloc (total_fview_count *
sizeof(struct iovec));
if (NULL == global_iov_array) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
ret = fh->f_allgatherv_array (local_iov_array,
local_count,
fh->f_iov_type,
global_iov_array,
fview_count,
displs,
fh->f_iov_type,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
goto exit;
}
/* sort it */
if (0 != total_fview_count) {
sorted = (int *)malloc (total_fview_count * sizeof(int));
if (NULL == sorted) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
fh->f_sort_iovec (global_iov_array, total_fview_count, sorted);
}
if (NULL != local_iov_array) {
free (local_iov_array);
local_iov_array = NULL;
}
#if DEBUG_ON
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
for (i=0 ; i<total_fview_count ; i++) {
printf("%d: OFFSET: %p LENGTH: %d\n",
fh->f_rank,
global_iov_array[sorted[i]].iov_base,
global_iov_array[sorted[i]].iov_len);
}
}
#endif
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
blocklen_per_process = (int **)malloc (fh->f_procs_per_group * sizeof (int*));
if (NULL == blocklen_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process = (MPI_Aint **)malloc (fh->f_procs_per_group * sizeof (MPI_Aint*));
if (NULL == displs_per_process){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for (i=0;i<fh->f_procs_per_group;i++){
blocklen_per_process[i] = NULL;
displs_per_process[i] = NULL;
}
}
/*
* Calculate how many bytes are read in each cycle
*/
fh->f_get_bytes_per_agg ( (int *) &bytes_per_cycle);
cycles = ceil((double)total_bytes/bytes_per_cycle);
n = 0;
bytes_remaining = 0;
current_index = 0;
#if TIME_BREAKDOWN
start_rexch = MPI_Wtime();
#endif
for (index = 0; index < cycles; index++) {
/* Getting ready for next cycle
Initializing and freeing buffers */
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
if (NULL == sendtype){
sendtype = (ompi_datatype_t **)
malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
if (NULL == sendtype) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
for(l=0;l<fh->f_procs_per_group;l++){
disp_index[l] = 1;
if (NULL != blocklen_per_process[l]){
free(blocklen_per_process[l]);
blocklen_per_process[l] = NULL;
}
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
if (NULL == blocklen_per_process[l]) {
opal_output (1, "OUT OF MEMORY for blocklen\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
if (NULL == displs_per_process[l]){
opal_output (1, "OUT OF MEMORY for displs\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if(NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements = NULL;
}
}
if (cycles-1 == index) {
bytes_to_read_in_cycle = total_bytes - bytes_per_cycle*index;
}
else {
bytes_to_read_in_cycle = bytes_per_cycle;
}
#if DEBUG_ON
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
printf ("****%d: CYCLE %d Bytes %d**********\n",
fh->f_rank,
index,
bytes_to_write_in_cycle);
}
#endif
/* Calculate how much data will be contributed in this cycle
by each process*/
bytes_received = 0;
while (bytes_to_read_in_cycle) {
blocks = fview_count[0];
for (j=0 ; j<fh->f_procs_per_group ; j++) {
if (sorted[current_index] < blocks) {
n = j;
break;
}
else {
blocks += fview_count[j+1];
}
}
if (bytes_remaining) {
if (bytes_remaining <= bytes_to_read_in_cycle) {
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_remaining;
displs_per_process[n][disp_index[n] - 1] =
(OPAL_PTRDIFF_TYPE)global_iov_array[sorted[current_index]].iov_base +
(global_iov_array[sorted[current_index]].iov_len - bytes_remaining);
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_received += bytes_remaining;
}
current_index ++;
bytes_to_read_in_cycle -= bytes_remaining;
bytes_remaining = 0;
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
fh->f_rank) {
blocklen_per_process[n] = (int *) realloc
((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
displs_per_process[n] = (MPI_Aint *) realloc
((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
blocklen_per_process[n][disp_index[n]] = 0;
displs_per_process[n][disp_index[n]] = 0;
disp_index[n] += 1;
}
continue;
}
else {
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
displs_per_process[n][disp_index[n] - 1] =
(OPAL_PTRDIFF_TYPE)global_iov_array[sorted[current_index]].iov_base +
(global_iov_array[sorted[current_index]].iov_len
- bytes_remaining);
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_received += bytes_to_read_in_cycle;
}
bytes_remaining -= bytes_to_read_in_cycle;
bytes_to_read_in_cycle = 0;
break;
}
}
else {
if (bytes_to_read_in_cycle <
(MPI_Aint) global_iov_array[sorted[current_index]].iov_len) {
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_to_read_in_cycle;
displs_per_process[n][disp_index[n] - 1] =
(OPAL_PTRDIFF_TYPE)global_iov_array[sorted[current_index]].iov_base ;
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_received += bytes_to_read_in_cycle;
}
bytes_remaining = global_iov_array[sorted[current_index]].iov_len -
bytes_to_read_in_cycle;
bytes_to_read_in_cycle = 0;
break;
}
else {
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] =
global_iov_array[sorted[current_index]].iov_len;
displs_per_process[n][disp_index[n] - 1] = (OPAL_PTRDIFF_TYPE)
global_iov_array[sorted[current_index]].iov_base;
blocklen_per_process[n] =
(int *) realloc ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
displs_per_process[n] = (MPI_Aint *)realloc
((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
blocklen_per_process[n][disp_index[n]] = 0;
displs_per_process[n][disp_index[n]] = 0;
disp_index[n] += 1;
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_received +=
global_iov_array[sorted[current_index]].iov_len;
}
bytes_to_read_in_cycle -=
global_iov_array[sorted[current_index]].iov_len;
current_index ++;
continue;
}
}
}
/* Calculate the displacement on where to put the data and allocate
the recieve buffer (global_buf) */
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
entries_per_aggregator=0;
for (i=0;i<fh->f_procs_per_group; i++){
for (j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0)
entries_per_aggregator++ ;
}
}
if (entries_per_aggregator > 0){
file_offsets_for_agg = (local_io_array *)
malloc(entries_per_aggregator*sizeof(local_io_array));
if (NULL == file_offsets_for_agg) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
sorted_file_offsets = (int *)
malloc (entries_per_aggregator*sizeof(int));
if (NULL == sorted_file_offsets){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
/*Moving file offsets to an IO array!*/
temp_index = 0;
global_count = 0;
for (i=0;i<fh->f_procs_per_group; i++){
for(j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0){
file_offsets_for_agg[temp_index].length =
blocklen_per_process[i][j];
global_count += blocklen_per_process[i][j];
file_offsets_for_agg[temp_index].process_id = i;
file_offsets_for_agg[temp_index].offset =
displs_per_process[i][j];
temp_index++;
}
}
}
}
else{
continue;
}
read_heap_sort (file_offsets_for_agg,
entries_per_aggregator,
sorted_file_offsets);
memory_displacements = (MPI_Aint *) malloc
(entries_per_aggregator * sizeof(MPI_Aint));
memory_displacements[sorted_file_offsets[0]] = 0;
for (i=1; i<entries_per_aggregator; i++){
memory_displacements[sorted_file_offsets[i]] =
memory_displacements[sorted_file_offsets[i-1]] +
file_offsets_for_agg[sorted_file_offsets[i-1]].length;
}
global_buf = (char *) malloc (global_count * sizeof(char));
if (NULL == global_buf){
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
(entries_per_aggregator * sizeof (mca_io_ompio_io_array_t));
if (NULL == fh->f_io_array) {
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
fh->f_num_of_io_entries = 0;
fh->f_io_array[fh->f_num_of_io_entries].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
fh->f_io_array[fh->f_num_of_io_entries].length =
file_offsets_for_agg[sorted_file_offsets[0]].length;
fh->f_io_array[fh->f_num_of_io_entries].memory_address =
global_buf+memory_displacements[sorted_file_offsets[0]];
fh->f_num_of_io_entries++;
for (i=1;i<entries_per_aggregator;i++){
if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
file_offsets_for_agg[sorted_file_offsets[i]].offset){
fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
file_offsets_for_agg[sorted_file_offsets[i]].length;
}
else{
fh->f_io_array[fh->f_num_of_io_entries].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
fh->f_io_array[fh->f_num_of_io_entries].length =
file_offsets_for_agg[sorted_file_offsets[i]].length;
fh->f_io_array[fh->f_num_of_io_entries].memory_address =
global_buf+memory_displacements[sorted_file_offsets[i]];
fh->f_num_of_io_entries++;
}
}
#if TIME_BREAKDOWN
start_read_time = MPI_Wtime();
#endif
if (fh->f_num_of_io_entries) {
if ( 0 > fh->f_fbtl->fbtl_preadv (fh)) {
opal_output (1, "READ FAILED\n");
ret = OMPI_ERROR;
goto exit;
}
}
#if TIME_BREAKDOWN
end_read_time = MPI_Wtime();
read_time += end_read_time - start_read_time;
#endif
/**********************************************************
******************** DONE READING ************************
*********************************************************/
temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
if (NULL == temp_disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for (i=0; i<entries_per_aggregator; i++){
temp_index =
file_offsets_for_agg[sorted_file_offsets[i]].process_id;
displs_per_process[temp_index][temp_disp_index[temp_index]] =
memory_displacements[sorted_file_offsets[i]];
if (temp_disp_index[temp_index] < disp_index[temp_index]){
temp_disp_index[temp_index] += 1;
}
else{
printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
temp_index, temp_disp_index[temp_index],
temp_index, disp_index[temp_index]);
}
}
if (NULL != temp_disp_index){
free(temp_disp_index);
temp_disp_index = NULL;
}
send_req = (MPI_Request *)
malloc (fh->f_procs_per_group * sizeof(MPI_Request));
if (NULL == send_req){
opal_output ( 1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
#if TIME_BREAKDOWN
start_rcomm_time = MPI_Wtime();
#endif
for (i=0;i<fh->f_procs_per_group;i++){
ompi_datatype_create_hindexed(disp_index[i],
blocklen_per_process[i],
displs_per_process[i],
MPI_BYTE,
&sendtype[i]);
ompi_datatype_commit(&sendtype[i]);
ret = MCA_PML_CALL (isend(global_buf,
1,
sendtype[i],
fh->f_procs_in_group[i],
123,
MCA_PML_BASE_SEND_STANDARD,
fh->f_comm,
&send_req[i]));
if(OMPI_SUCCESS != ret){
goto exit;
}
}
#if TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
}
/**********************************************************
********* Scatter the Data from the readers **************
*********************************************************/
if (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY) {
receive_buf = &((char*)buf)[position];
}
else if (bytes_received) {
/* allocate a receive buffer and copy the data that needs
to be received into it in case the data is non-contigous
in memory */
receive_buf = malloc (bytes_received);
if (NULL == receive_buf) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
#if TIME_BREAKDOWN
start_rcomm_time = MPI_Wtime();
#endif
recv_req = (MPI_Request *) malloc (sizeof (MPI_Request));
if (NULL == recv_req){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ret = MCA_PML_CALL(irecv(receive_buf,
bytes_received,
MPI_BYTE,
fh->f_procs_in_group[fh->f_aggregator_index],
123,
fh->f_comm,
recv_req));
if (OMPI_SUCCESS != ret){
goto exit;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank){
ret = ompi_request_wait_all (fh->f_procs_per_group,
send_req,
MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
ret = ompi_request_wait (recv_req, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
position += bytes_received;
/* If data is not contigous in memory, copy the data from the
receive buffer into the buffer passed in */
if (!(fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
OPAL_PTRDIFF_TYPE mem_address;
size_t remaining = 0;
size_t temp_position = 0;
remaining = bytes_received;
while (remaining) {
mem_address = (OPAL_PTRDIFF_TYPE)
(decoded_iov[iov_index].iov_base) + current_position;
if (remaining >=
(decoded_iov[iov_index].iov_len - current_position)) {
memcpy ((IOVBASE_TYPE *) mem_address,
receive_buf+temp_position,
decoded_iov[iov_index].iov_len - current_position);
remaining = remaining -
(decoded_iov[iov_index].iov_len - current_position);
temp_position = temp_position +
(decoded_iov[iov_index].iov_len - current_position);
iov_index = iov_index + 1;
current_position = 0;
}
else {
memcpy ((IOVBASE_TYPE *) mem_address,
receive_buf+temp_position,
remaining);
current_position = current_position + remaining;
remaining = 0;
}
}
if (NULL != receive_buf) {
free (receive_buf);
receive_buf = NULL;
}
}
#if TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
if (NULL != recv_req){
free(recv_req);
recv_req = NULL;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank){
fh->f_num_of_io_entries = 0;
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
if (NULL != global_buf) {
free (global_buf);
global_buf = NULL;
}
for (i = 0; i < fh->f_procs_per_group; i++)
ompi_datatype_destroy(sendtype+i);
if (NULL != sendtype){
free(sendtype);
sendtype=NULL;
}
if (NULL != send_req){
free(send_req);
send_req = NULL;
}
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if (NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != bytes_per_process){
free(bytes_per_process);
bytes_per_process =NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements= NULL;
}
}
}
#if TIME_BREAKDOWN
end_rexch = MPI_Wtime();
read_exch += end_rexch - start_rexch;
nentry.time[0] = read_time;
nentry.time[1] = rcomm_time;
nentry.time[2] = read_exch;
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank)
nentry.aggregator = 1;
else
nentry.aggregator = 0;
nentry.nprocs_for_coll = dynamic_num_io_procs;
if (!fh->f_full_print_queue(READ_PRINT_QUEUE)){
fh->f_register_print_entry(READ_PRINT_QUEUE,
nentry);
}
#endif
exit:
if (NULL != sorted) {
free (sorted);
sorted = NULL;
}
if (NULL != global_iov_array) {
free (global_iov_array);
global_iov_array = NULL;
}
if (NULL != fview_count) {
free (fview_count);
fview_count = NULL;
}
if (NULL != decoded_iov) {
free (decoded_iov);
decoded_iov = NULL;
}
if (NULL != local_iov_array){
free(local_iov_array);
local_iov_array=NULL;
}
if (NULL != displs) {
free (displs);
displs = NULL;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
if (NULL != disp_index){
free(disp_index);
disp_index = NULL;
}
if ( NULL != blocklen_per_process){
for(l=0;l<fh->f_procs_per_group;l++){
if (NULL != blocklen_per_process[l]){
free(blocklen_per_process[l]);
blocklen_per_process[l] = NULL;
}
}
free(blocklen_per_process);
blocklen_per_process = NULL;
}
if (NULL != displs_per_process){
for (l=0; i<fh->f_procs_per_group; l++){
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
}
free(displs_per_process);
displs_per_process = NULL;
}
}
return ret;
}
/*
* reduce_scatter
*
* Function: - reduce then scatter
* Accepts: - same as MPI_Reduce_scatter()
* Returns: - MPI_SUCCESS or error code
*
* Algorithm:
* Cummutative, reasonable sized messages
* recursive halving algorithm
* Others:
* reduce and scatterv (needs to be cleaned
* up at some point)
*
* NOTE: that the recursive halving algorithm should be faster than
* the reduce/scatter for all message sizes. However, the memory
* usage for the recusive halving is msg_size + 2 * comm_size greater
* for the recursive halving, so I've limited where the recursive
* halving is used to be nice to the app memory wise. There are much
* better algorithms for large messages with cummutative operations,
* so this should be investigated further.
*
* NOTE: We default to a simple reduce/scatterv if one of the rcounts
* is zero. This is because the existing algorithms do not currently
* support a count of zero in the array.
*/
int
mca_coll_basic_reduce_scatter_intra(void *sbuf, void *rbuf, int *rcounts,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
int i, rank, size, count, err = OMPI_SUCCESS;
ptrdiff_t true_lb, true_extent, lb, extent, buf_size;
int *disps = NULL;
char *recv_buf = NULL, *recv_buf_free = NULL;
char *result_buf = NULL, *result_buf_free = NULL;
bool zerocounts = false;
/* Initialize */
rank = ompi_comm_rank(comm);
size = ompi_comm_size(comm);
/* Find displacements and the like */
disps = (int*) malloc(sizeof(int) * size);
if (NULL == disps) return OMPI_ERR_OUT_OF_RESOURCE;
disps[0] = 0;
for (i = 0; i < (size - 1); ++i) {
disps[i + 1] = disps[i] + rcounts[i];
if (0 == rcounts[i]) {
zerocounts = true;
}
}
count = disps[size - 1] + rcounts[size - 1];
if (0 == rcounts[size - 1]) {
zerocounts = true;
}
/* short cut the trivial case */
if (0 == count) {
free(disps);
return OMPI_SUCCESS;
}
/* get datatype information */
ompi_datatype_get_extent(dtype, &lb, &extent);
ompi_datatype_get_true_extent(dtype, &true_lb, &true_extent);
buf_size = true_extent + (count - 1) * extent;
/* Handle MPI_IN_PLACE */
if (MPI_IN_PLACE == sbuf) {
sbuf = rbuf;
}
if ((op->o_flags & OMPI_OP_FLAGS_COMMUTE) &&
(buf_size < COMMUTATIVE_LONG_MSG) && (!zerocounts)) {
int tmp_size, remain = 0, tmp_rank;
/* temporary receive buffer. See coll_basic_reduce.c for details on sizing */
recv_buf_free = (char*) malloc(buf_size);
recv_buf = recv_buf_free - lb;
if (NULL == recv_buf_free) {
err = OMPI_ERR_OUT_OF_RESOURCE;
goto cleanup;
}
/* allocate temporary buffer for results */
result_buf_free = (char*) malloc(buf_size);
result_buf = result_buf_free - lb;
/* copy local buffer into the temporary results */
err = ompi_datatype_sndrcv(sbuf, count, dtype, result_buf, count, dtype);
if (OMPI_SUCCESS != err) goto cleanup;
/* figure out power of two mapping: grow until larger than
comm size, then go back one, to get the largest power of
two less than comm size */
tmp_size = opal_next_poweroftwo(size);
tmp_size >>= 1;
remain = size - tmp_size;
/* If comm size is not a power of two, have the first "remain"
procs with an even rank send to rank + 1, leaving a power of
two procs to do the rest of the algorithm */
if (rank < 2 * remain) {
if ((rank & 1) == 0) {
err = MCA_PML_CALL(send(result_buf, count, dtype, rank + 1,
MCA_COLL_BASE_TAG_REDUCE_SCATTER,
MCA_PML_BASE_SEND_STANDARD,
comm));
if (OMPI_SUCCESS != err) goto cleanup;
/* we don't participate from here on out */
tmp_rank = -1;
} else {
err = MCA_PML_CALL(recv(recv_buf, count, dtype, rank - 1,
MCA_COLL_BASE_TAG_REDUCE_SCATTER,
comm, MPI_STATUS_IGNORE));
if (OMPI_SUCCESS != err) goto cleanup;
/* integrate their results into our temp results */
ompi_op_reduce(op, recv_buf, result_buf, count, dtype);
/* adjust rank to be the bottom "remain" ranks */
tmp_rank = rank / 2;
}
} else {
/* just need to adjust rank to show that the bottom "even
remain" ranks dropped out */
tmp_rank = rank - remain;
}
/* For ranks not kicked out by the above code, perform the
recursive halving */
if (tmp_rank >= 0) {
int *tmp_disps = NULL, *tmp_rcounts = NULL;
int mask, send_index, recv_index, last_index;
/* recalculate disps and rcounts to account for the
special "remainder" processes that are no longer doing
anything */
tmp_rcounts = (int*) malloc(tmp_size * sizeof(int));
if (NULL == tmp_rcounts) {
err = OMPI_ERR_OUT_OF_RESOURCE;
goto cleanup;
}
tmp_disps = (int*) malloc(tmp_size * sizeof(int));
if (NULL == tmp_disps) {
free(tmp_rcounts);
err = OMPI_ERR_OUT_OF_RESOURCE;
goto cleanup;
}
for (i = 0 ; i < tmp_size ; ++i) {
if (i < remain) {
/* need to include old neighbor as well */
tmp_rcounts[i] = rcounts[i * 2 + 1] + rcounts[i * 2];
} else {
tmp_rcounts[i] = rcounts[i + remain];
}
}
tmp_disps[0] = 0;
for (i = 0; i < tmp_size - 1; ++i) {
tmp_disps[i + 1] = tmp_disps[i] + tmp_rcounts[i];
}
/* do the recursive halving communication. Don't use the
dimension information on the communicator because I
think the information is invalidated by our "shrinking"
of the communicator */
mask = tmp_size >> 1;
send_index = recv_index = 0;
last_index = tmp_size;
while (mask > 0) {
int tmp_peer, peer, send_count, recv_count;
struct ompi_request_t *request;
tmp_peer = tmp_rank ^ mask;
peer = (tmp_peer < remain) ? tmp_peer * 2 + 1 : tmp_peer + remain;
/* figure out if we're sending, receiving, or both */
send_count = recv_count = 0;
if (tmp_rank < tmp_peer) {
send_index = recv_index + mask;
for (i = send_index ; i < last_index ; ++i) {
send_count += tmp_rcounts[i];
}
for (i = recv_index ; i < send_index ; ++i) {
recv_count += tmp_rcounts[i];
}
} else {
recv_index = send_index + mask;
for (i = send_index ; i < recv_index ; ++i) {
send_count += tmp_rcounts[i];
}
for (i = recv_index ; i < last_index ; ++i) {
recv_count += tmp_rcounts[i];
}
}
/* actual data transfer. Send from result_buf,
receive into recv_buf */
if (send_count > 0 && recv_count != 0) {
err = MCA_PML_CALL(irecv(recv_buf + tmp_disps[recv_index] * extent,
recv_count, dtype, peer,
MCA_COLL_BASE_TAG_REDUCE_SCATTER,
comm, &request));
if (OMPI_SUCCESS != err) {
free(tmp_rcounts);
free(tmp_disps);
goto cleanup;
}
}
if (recv_count > 0 && send_count != 0) {
err = MCA_PML_CALL(send(result_buf + tmp_disps[send_index] * extent,
send_count, dtype, peer,
MCA_COLL_BASE_TAG_REDUCE_SCATTER,
MCA_PML_BASE_SEND_STANDARD,
comm));
if (OMPI_SUCCESS != err) {
free(tmp_rcounts);
free(tmp_disps);
goto cleanup;
}
}
if (send_count > 0 && recv_count != 0) {
err = ompi_request_wait(&request, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != err) {
free(tmp_rcounts);
free(tmp_disps);
goto cleanup;
}
}
/* if we received something on this step, push it into
the results buffer */
if (recv_count > 0) {
ompi_op_reduce(op,
recv_buf + tmp_disps[recv_index] * extent,
result_buf + tmp_disps[recv_index] * extent,
recv_count, dtype);
}
/* update for next iteration */
send_index = recv_index;
last_index = recv_index + mask;
mask >>= 1;
}
/* copy local results from results buffer into real receive buffer */
if (0 != rcounts[rank]) {
err = ompi_datatype_sndrcv(result_buf + disps[rank] * extent,
rcounts[rank], dtype,
rbuf, rcounts[rank], dtype);
if (OMPI_SUCCESS != err) {
free(tmp_rcounts);
free(tmp_disps);
goto cleanup;
}
}
free(tmp_rcounts);
free(tmp_disps);
}
/*
* exscan_intra
*
* Function: - basic exscan operation
* Accepts: - same arguments as MPI_Exscan()
* Returns: - MPI_SUCCESS or error code
*/
int
mca_coll_basic_exscan_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
int size, rank, err;
ptrdiff_t true_lb, true_extent, lb, extent;
char *free_buffer = NULL;
char *reduce_buffer = NULL;
char *source;
MPI_Request req = MPI_REQUEST_NULL;
/* Initialize. */
rank = ompi_comm_rank(comm);
size = ompi_comm_size(comm);
/* If we're rank 0, then we send our sbuf to the next rank */
if (0 == rank) {
return MCA_PML_CALL(send(sbuf, count, dtype, rank + 1,
MCA_COLL_BASE_TAG_EXSCAN,
MCA_PML_BASE_SEND_STANDARD, comm));
}
/* If we're the last rank, then just receive the result from the
* prior rank */
else if ((size - 1) == rank) {
return MCA_PML_CALL(recv(rbuf, count, dtype, rank - 1,
MCA_COLL_BASE_TAG_EXSCAN, comm,
MPI_STATUS_IGNORE));
}
/* Otherwise, get the result from the prior rank, combine it with my
* data, and send it to the next rank */
/* Start the receive for the prior rank's answer */
err = MCA_PML_CALL(irecv(rbuf, count, dtype, rank - 1,
MCA_COLL_BASE_TAG_EXSCAN, comm, &req));
if (MPI_SUCCESS != err) {
goto error;
}
/* Get a temporary buffer to perform the reduction into. Rationale
* for malloc'ing this size is provided in coll_basic_reduce.c. */
ompi_ddt_get_extent(dtype, &lb, &extent);
ompi_ddt_get_true_extent(dtype, &true_lb, &true_extent);
free_buffer = (char*)malloc(true_extent + (count - 1) * extent);
if (NULL == free_buffer) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
reduce_buffer = free_buffer - lb;
if (ompi_op_is_commute(op)) {
/* If we're commutative, we can copy my sbuf into the reduction
* buffer before the receive completes */
err = ompi_ddt_copy_content_same_ddt(dtype, count,
reduce_buffer, (char*)sbuf);
if (MPI_SUCCESS != err) {
goto error;
}
/* Now setup the reduction */
source = (char*)rbuf;
/* Finally, wait for the receive to complete (so that we can do
* the reduction). */
err = ompi_request_wait(&req, MPI_STATUS_IGNORE);
if (MPI_SUCCESS != err) {
goto error;
}
} else {
/* Setup the reduction */
source = (char*)sbuf;
/* If we're not commutative, we have to wait for the receive to
* complete and then copy it into the reduce buffer */
err = ompi_request_wait(&req, MPI_STATUS_IGNORE);
if (MPI_SUCCESS != err) {
goto error;
}
err = ompi_ddt_copy_content_same_ddt(dtype, count,
reduce_buffer, (char*)rbuf);
if (MPI_SUCCESS != err) {
goto error;
}
}
/* Now reduce the received answer with my source into the answer
* that we send off to the next rank */
ompi_op_reduce(op, source, reduce_buffer, count, dtype);
/* Send my result off to the next rank */
err = MCA_PML_CALL(send(reduce_buffer, count, dtype, rank + 1,
MCA_COLL_BASE_TAG_EXSCAN,
MCA_PML_BASE_SEND_STANDARD, comm));
/* Error */
error:
free(free_buffer);
if (MPI_REQUEST_NULL != req) {
ompi_request_cancel(req);
ompi_request_wait(&req, MPI_STATUS_IGNORE);
}
/* All done */
return err;
}
int
ompi_coll_tuned_bcast_intra_generic( void* buffer,
int original_count,
struct ompi_datatype_t* datatype,
int root,
struct ompi_communicator_t* comm,
mca_coll_base_module_t *module,
uint32_t count_by_segment,
ompi_coll_tree_t* tree )
{
int err = 0, line, i, rank, size, segindex, req_index;
int num_segments; /* Number of segments */
int sendcount; /* number of elements sent in this segment */
size_t realsegsize, type_size;
char *tmpbuf;
ptrdiff_t extent, lb;
ompi_request_t *recv_reqs[2] = {MPI_REQUEST_NULL, MPI_REQUEST_NULL};
#if !defined(COLL_TUNED_BCAST_USE_BLOCKING)
ompi_request_t **send_reqs = NULL;
#endif
size = ompi_comm_size(comm);
rank = ompi_comm_rank(comm);
assert( size > 1 );
ompi_datatype_get_extent (datatype, &lb, &extent);
ompi_datatype_type_size( datatype, &type_size );
num_segments = (original_count + count_by_segment - 1) / count_by_segment;
realsegsize = (ptrdiff_t)count_by_segment * extent;
/* Set the buffer pointers */
tmpbuf = (char *) buffer;
#if !defined(COLL_TUNED_BCAST_USE_BLOCKING)
if( tree->tree_nextsize != 0 ) {
send_reqs = (ompi_request_t**)malloc( (ptrdiff_t)tree->tree_nextsize *
sizeof(ompi_request_t*) );
}
#endif
/* Root code */
if( rank == root ) {
/*
For each segment:
- send segment to all children.
The last segment may have less elements than other segments.
*/
sendcount = count_by_segment;
for( segindex = 0; segindex < num_segments; segindex++ ) {
if( segindex == (num_segments - 1) ) {
sendcount = original_count - segindex * count_by_segment;
}
for( i = 0; i < tree->tree_nextsize; i++ ) {
#if defined(COLL_TUNED_BCAST_USE_BLOCKING)
err = MCA_PML_CALL(send(tmpbuf, sendcount, datatype,
tree->tree_next[i],
MCA_COLL_BASE_TAG_BCAST,
MCA_PML_BASE_SEND_STANDARD, comm));
#else
err = MCA_PML_CALL(isend(tmpbuf, sendcount, datatype,
tree->tree_next[i],
MCA_COLL_BASE_TAG_BCAST,
MCA_PML_BASE_SEND_STANDARD, comm,
&send_reqs[i]));
#endif /* COLL_TUNED_BCAST_USE_BLOCKING */
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
}
#if !defined(COLL_TUNED_BCAST_USE_BLOCKING)
/* complete the sends before starting the next sends */
err = ompi_request_wait_all( tree->tree_nextsize, send_reqs,
MPI_STATUSES_IGNORE );
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
#endif /* not COLL_TUNED_BCAST_USE_BLOCKING */
/* update tmp buffer */
tmpbuf += realsegsize;
}
}
/* Intermediate nodes code */
else if( tree->tree_nextsize > 0 ) {
/*
Create the pipeline.
1) Post the first receive
2) For segments 1 .. num_segments
- post new receive
- wait on the previous receive to complete
- send this data to children
3) Wait on the last segment
4) Compute number of elements in last segment.
5) Send the last segment to children
*/
req_index = 0;
err = MCA_PML_CALL(irecv(tmpbuf, count_by_segment, datatype,
tree->tree_prev, MCA_COLL_BASE_TAG_BCAST,
comm, &recv_reqs[req_index]));
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
for( segindex = 1; segindex < num_segments; segindex++ ) {
req_index = req_index ^ 0x1;
/* post new irecv */
err = MCA_PML_CALL(irecv( tmpbuf + realsegsize, count_by_segment,
datatype, tree->tree_prev,
MCA_COLL_BASE_TAG_BCAST,
comm, &recv_reqs[req_index]));
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
/* wait for and forward the previous segment to children */
err = ompi_request_wait( &recv_reqs[req_index ^ 0x1],
MPI_STATUSES_IGNORE );
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
for( i = 0; i < tree->tree_nextsize; i++ ) {
#if defined(COLL_TUNED_BCAST_USE_BLOCKING)
err = MCA_PML_CALL(send(tmpbuf, count_by_segment, datatype,
tree->tree_next[i],
MCA_COLL_BASE_TAG_BCAST,
MCA_PML_BASE_SEND_STANDARD, comm));
#else
err = MCA_PML_CALL(isend(tmpbuf, count_by_segment, datatype,
tree->tree_next[i],
MCA_COLL_BASE_TAG_BCAST,
MCA_PML_BASE_SEND_STANDARD, comm,
&send_reqs[i]));
#endif /* COLL_TUNED_BCAST_USE_BLOCKING */
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
}
#if !defined(COLL_TUNED_BCAST_USE_BLOCKING)
/* complete the sends before starting the next iteration */
err = ompi_request_wait_all( tree->tree_nextsize, send_reqs,
MPI_STATUSES_IGNORE );
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
#endif /* COLL_TUNED_BCAST_USE_BLOCKING */
/* Update the receive buffer */
tmpbuf += realsegsize;
}
/* Process the last segment */
err = ompi_request_wait( &recv_reqs[req_index], MPI_STATUSES_IGNORE );
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
sendcount = original_count - (ptrdiff_t)(num_segments - 1) * count_by_segment;
for( i = 0; i < tree->tree_nextsize; i++ ) {
#if defined(COLL_TUNED_BCAST_USE_BLOCKING)
err = MCA_PML_CALL(send(tmpbuf, sendcount, datatype,
tree->tree_next[i],
MCA_COLL_BASE_TAG_BCAST,
MCA_PML_BASE_SEND_STANDARD, comm));
#else
err = MCA_PML_CALL(isend(tmpbuf, sendcount, datatype,
tree->tree_next[i],
MCA_COLL_BASE_TAG_BCAST,
MCA_PML_BASE_SEND_STANDARD, comm,
&send_reqs[i]));
#endif /* COLL_TUNED_BCAST_USE_BLOCKING */
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
}
#if !defined(COLL_TUNED_BCAST_USE_BLOCKING)
err = ompi_request_wait_all( tree->tree_nextsize, send_reqs,
MPI_STATUSES_IGNORE );
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
#endif /* COLL_TUNED_BCAST_USE_BLOCKING */
}
/* Leaf nodes */
else {
/*
Receive all segments from parent in a loop:
1) post irecv for the first segment
2) for segments 1 .. num_segments
- post irecv for the next segment
- wait on the previous segment to arrive
3) wait for the last segment
*/
req_index = 0;
err = MCA_PML_CALL(irecv(tmpbuf, count_by_segment, datatype,
tree->tree_prev, MCA_COLL_BASE_TAG_BCAST,
comm, &recv_reqs[req_index]));
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
for( segindex = 1; segindex < num_segments; segindex++ ) {
req_index = req_index ^ 0x1;
tmpbuf += realsegsize;
/* post receive for the next segment */
err = MCA_PML_CALL(irecv(tmpbuf, count_by_segment, datatype,
tree->tree_prev, MCA_COLL_BASE_TAG_BCAST,
comm, &recv_reqs[req_index]));
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
/* wait on the previous segment */
err = ompi_request_wait( &recv_reqs[req_index ^ 0x1],
MPI_STATUS_IGNORE );
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
}
err = ompi_request_wait( &recv_reqs[req_index], MPI_STATUS_IGNORE );
if (err != MPI_SUCCESS) {
line = __LINE__;
goto error_hndl;
}
}
#if !defined(COLL_TUNED_BCAST_USE_BLOCKING)
if( NULL != send_reqs ) free(send_reqs);
#endif
return (MPI_SUCCESS);
error_hndl:
OPAL_OUTPUT( (ompi_coll_tuned_stream,"%s:%4d\tError occurred %d, rank %2d",
__FILE__, line, err, rank) );
#if !defined(COLL_TUNED_BCAST_USE_BLOCKING)
if( NULL != send_reqs ) free(send_reqs);
#endif
return (err);
}
int
mca_fcoll_dynamic_file_write_all (mca_io_ompio_file_t *fh,
void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
MPI_Aint total_bytes_written = 0; /* total bytes that have been written*/
MPI_Aint total_bytes = 0; /* total bytes to be written */
MPI_Aint bytes_to_write_in_cycle = 0; /* left to be written in a cycle*/
MPI_Aint bytes_per_cycle = 0; /* total written in each cycle by each process*/
int index = 0;
int cycles = 0;
int i=0, j=0, l=0;
int n=0; /* current position in total_bytes_per_process array */
MPI_Aint bytes_remaining = 0; /* how many bytes have been written from the current
value from total_bytes_per_process */
int bytes_sent = 0, ret =0;
int blocks=0, entries_per_aggregator=0;
/* iovec structure and count of the buffer passed in */
uint32_t iov_count = 0;
struct iovec *decoded_iov = NULL;
int iov_index = 0;
char *send_buf = NULL;
size_t current_position = 0;
struct iovec *local_iov_array=NULL, *global_iov_array=NULL;
local_io_array *file_offsets_for_agg=NULL;
/* global iovec at the writers that contain the iovecs created from
file_set_view */
uint32_t total_fview_count = 0;
int local_count = 0, temp_pindex;
int *fview_count = NULL, *disp_index=NULL, *temp_disp_index=NULL;
int current_index = 0, temp_index=0;
char *global_buf = NULL;
MPI_Aint global_count = 0;
/* array that contains the sorted indices of the global_iov */
int *sorted = NULL, *sorted_file_offsets=NULL;
int *displs = NULL;
size_t max_data = 0;
int **blocklen_per_process=NULL;
MPI_Aint **displs_per_process=NULL, *memory_displacements=NULL;
ompi_datatype_t **recvtype = NULL;
MPI_Aint *total_bytes_per_process = NULL;
MPI_Request *send_req=NULL, *recv_req=NULL;
int datatype_size, recv_req_count=0;
#if TIME_BREAKDOWN
double start_time=0.0, end_time=0.0, start_time2=0.0, end_time2=0.0;
double total=0.0 , total_io=0.0, max_io=0.0; /* max_pp=0.0;*/
double start_ptime=0.0, end_ptime=0.0, tpw=0.0; /* max_tpw=0.0;*/
double start_cio_array=0.0, end_cio_array=0.0, tcio_array=0.0;/* max_cio=0.0;*/
double start_sr=0.0, end_sr=0.0, tsr=0.0;/* max_sr=0.0;*/
double comm_time = 0.0, max_comm_time=0.0;
double write_time = 0.0, max_write_time=0.0;
#endif
#if TIME_BREAKDOWN
start_time = MPI_Wtime();
#endif
if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
}
/**************************************************************************
** In case the data is not contigous in memory, decode it into an iovec **
**************************************************************************/
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
ret = ompi_io_ompio_decode_datatype (fh,
datatype,
count,
buf,
&max_data,
&decoded_iov,
&iov_count);
if (OMPI_SUCCESS != ret ){
goto exit;
}
}
else {
max_data = count * datatype->super.size;
}
if ( MPI_STATUS_IGNORE != status ) {
status->_ucount = max_data;
}
if (! (fh->f_flags & OMPIO_AGGREGATOR_IS_SET)) {
ret = ompi_io_ompio_set_aggregator_props (fh,
mca_fcoll_dynamic_num_io_procs,
max_data);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
if (-1 == mca_fcoll_dynamic_num_io_procs) {
mca_fcoll_dynamic_num_io_procs = 1;
}
#if TIME_BREAKDOWN
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
start_time = MPI_Wtime();
}
#endif
total_bytes_per_process = (MPI_Aint*)malloc
(fh->f_procs_per_group*sizeof(MPI_Aint));
if (NULL == total_bytes_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ret = ompi_io_ompio_allgather_array (&max_data,
1,
MPI_LONG,
total_bytes_per_process,
1,
MPI_LONG,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if( OMPI_SUCCESS != ret){
goto exit;
}
for (i=0 ; i<fh->f_procs_per_group ; i++) {
total_bytes += total_bytes_per_process[i];
}
if (NULL != total_bytes_per_process) {
free (total_bytes_per_process);
total_bytes_per_process = NULL;
}
/*********************************************************************
*** Generate the File offsets/lengths corresponding to this write ***
********************************************************************/
ret = ompi_io_ompio_generate_current_file_view(fh,
max_data,
&local_iov_array,
&local_count);
if (ret != OMPI_SUCCESS){
goto exit;
}
#if DEBUG_ON
for (i=0 ; i<local_count ; i++) {
printf("Local offset-length pair for rank:%d \n",
fh->f_rank);
printf("%d: OFFSET: %p LENGTH: %lld\n",
fh->f_rank,
iov[i].iov_base,
iov[i].iov_len);
}
#endif
/*************************************************************
*** ALLGather the File View information at all processes ***
*************************************************************/
fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == fview_count) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ret = ompi_io_ompio_allgather_array (&local_count,
1,
MPI_INT,
fview_count,
1,
MPI_INT,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if( OMPI_SUCCESS != ret){
goto exit;
}
displs = (int*) malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == displs) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs[0] = 0;
total_fview_count = fview_count[0];
for (i=1 ; i<fh->f_procs_per_group ; i++) {
total_fview_count += fview_count[i];
displs[i] = displs[i-1] + fview_count[i-1];
}
#if DEBUG_ON
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
for (i=0 ; i<fh->f_procs_per_group ; i++) {
printf ("%d: PROCESS: %d ELEMENTS: %d DISPLS: %d\n",
fh->f_rank,
i,
fview_count[i],
displs[i]);
}
}
#endif
/* allocate the global iovec */
if (0 != total_fview_count) {
global_iov_array = (struct iovec*) malloc (total_fview_count *
sizeof(struct iovec));
if (NULL == global_iov_array){
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
if (fh->f_flags & OMPIO_UNIFORM_FVIEW) {
ret = ompi_io_ompio_allgather_array (local_iov_array,
local_count,
fh->f_iov_type,
global_iov_array,
local_count,
fh->f_iov_type,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
else {
ret = ompi_io_ompio_allgatherv_array (local_iov_array,
local_count,
fh->f_iov_type,
global_iov_array,
fview_count,
displs,
fh->f_iov_type,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
/* sort it */
if (0 != total_fview_count) {
sorted = (int *)malloc (total_fview_count * sizeof(int));
if (NULL == sorted) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ompi_io_ompio_sort_iovec (global_iov_array, total_fview_count, sorted);
}
if (NULL != local_iov_array){
free(local_iov_array);
local_iov_array = NULL;
}
if (NULL != displs){
free(displs);
displs=NULL;
}
#if DEBUG_ON
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
uint32_t tv=0;
for (tv=0 ; tv<total_fview_count ; tv++) {
printf("%d: OFFSET: %lld LENGTH: %ld\n",
fh->f_rank,
global_iov_array[sorted[tv]].offset,
global_iov_array[sorted[tv]].length);
}
}
#endif
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
blocklen_per_process = (int **)malloc (fh->f_procs_per_group * sizeof (int*));
if (NULL == blocklen_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process = (MPI_Aint **)malloc (fh->f_procs_per_group * sizeof (MPI_Aint*));
if (NULL == displs_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for(i=0;i<fh->f_procs_per_group;i++){
blocklen_per_process[i] = NULL;
displs_per_process[i] = NULL;
}
}
bytes_per_cycle = mca_fcoll_dynamic_cycle_buffer_size;
cycles = ceil((double)total_bytes/bytes_per_cycle);
n = 0;
bytes_remaining = 0;
current_index = 0;
#if TIME_BREAKDOWN
end_time = MPI_Wtime();
total = end_time-start_time;
start_time2 = MPI_Wtime();
#endif
for (index = 0; index < cycles; index++) {
/* Getting ready for next cycle
Initializing and freeing buffers*/
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
if (NULL == recvtype){
recvtype = (ompi_datatype_t **)
malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
if (NULL == recvtype) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
for(l=0;l<fh->f_procs_per_group;l++){
disp_index[l] = 1;
if (NULL != blocklen_per_process[l]){
free(blocklen_per_process[l]);
blocklen_per_process[l] = NULL;
}
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
if (NULL == blocklen_per_process[l]) {
opal_output (1, "OUT OF MEMORY for blocklen\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
if (NULL == displs_per_process[l]){
opal_output (1, "OUT OF MEMORY for displs\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if(NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements = NULL;
}
}
if (cycles-1 == index) {
bytes_to_write_in_cycle = total_bytes - bytes_per_cycle*index;
}
else {
bytes_to_write_in_cycle = bytes_per_cycle;
}
#if DEBUG_ON
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
printf ("****%d: CYCLE %d Bytes %lld**********\n",
fh->f_rank,
index,
bytes_to_write_in_cycle);
}
#endif
/**********************************************************
**Gather the Data from all the processes at the writers **
*********************************************************/
/* Calculate how much data will be contributed in this cycle
by each process*/
bytes_sent = 0;
#if DEBUG_ON
printf("bytes_to_write_in_cycle: %ld, cycle : %d\n", bytes_to_write_in_cycle,
index);
#endif
/* The blocklen and displs calculation only done at aggregators!*/
#if TIME_BREAKDOWN
start_cio_array = MPI_Wtime();
#endif
while (bytes_to_write_in_cycle) {
blocks = fview_count[0];
for (j=0 ; j<fh->f_procs_per_group ; j++) {
if (sorted[current_index] < blocks) {
n = j;
break;
}
else {
blocks += fview_count[j+1];
}
}
if (bytes_remaining) {
if (bytes_remaining <= bytes_to_write_in_cycle) {
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_remaining;
displs_per_process[n][disp_index[n] - 1] =
(OPAL_PTRDIFF_TYPE)global_iov_array[sorted[current_index]].iov_base +
(global_iov_array[sorted[current_index]].iov_len
- bytes_remaining);
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_sent += bytes_remaining;
}
current_index ++;
bytes_to_write_in_cycle -= bytes_remaining;
bytes_remaining = 0;
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
fh->f_rank) {
/* In this cases the length is consumed so allocating for
next displacement and blocklength*/
blocklen_per_process[n] = (int *) realloc
((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
displs_per_process[n] = (MPI_Aint *) realloc
((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
blocklen_per_process[n][disp_index[n]] = 0;
displs_per_process[n][disp_index[n]] = 0;
disp_index[n] += 1;
}
continue;
}
else {
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_to_write_in_cycle;
displs_per_process[n][disp_index[n] - 1] =
(OPAL_PTRDIFF_TYPE)global_iov_array[sorted[current_index]].iov_base +
(global_iov_array[sorted[current_index]].iov_len
- bytes_remaining);
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_sent += bytes_to_write_in_cycle;
}
bytes_remaining -= bytes_to_write_in_cycle;
bytes_to_write_in_cycle = 0;
break;
}
}
else {
if (bytes_to_write_in_cycle <
(MPI_Aint) global_iov_array[sorted[current_index]].iov_len) {
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_to_write_in_cycle;
displs_per_process[n][disp_index[n] - 1] =
(OPAL_PTRDIFF_TYPE)global_iov_array[sorted[current_index]].iov_base ;
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_sent += bytes_to_write_in_cycle;
}
bytes_remaining = global_iov_array[sorted[current_index]].iov_len -
bytes_to_write_in_cycle;
bytes_to_write_in_cycle = 0;
break;
}
else {
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] =
global_iov_array[sorted[current_index]].iov_len;
displs_per_process[n][disp_index[n] - 1] = (OPAL_PTRDIFF_TYPE)
global_iov_array[sorted[current_index]].iov_base;
blocklen_per_process[n] =
(int *) realloc ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
displs_per_process[n] = (MPI_Aint *)realloc
((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
blocklen_per_process[n][disp_index[n]] = 0;
displs_per_process[n][disp_index[n]] = 0;
disp_index[n] += 1;
/*realloc for next blocklength
and assign this displacement and check for next displs as
the total length of this entry has been consumed!*/
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_sent += global_iov_array[sorted[current_index]].iov_len;
}
bytes_to_write_in_cycle -=
global_iov_array[sorted[current_index]].iov_len;
current_index ++;
continue;
}
}
}
#if TIME_BREAKDOWN
start_sr = MPI_Wtime();
#endif
/* Calculate the displacement on where to put the data and allocate
the recieve buffer (global_buf) */
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
entries_per_aggregator=0;
for (i=0;i<fh->f_procs_per_group; i++){
for (j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0)
entries_per_aggregator++ ;
}
}
#if DEBUG_ON
printf("%d: cycle: %d, bytes_sent: %d\n ",fh->f_rank,index,
bytes_sent);
printf("%d : Entries per aggregator : %d\n",fh->f_rank,entries_per_aggregator);
#endif
if (entries_per_aggregator > 0){
file_offsets_for_agg = (local_io_array *)
malloc(entries_per_aggregator*sizeof(local_io_array));
if (NULL == file_offsets_for_agg) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
sorted_file_offsets = (int *)
malloc (entries_per_aggregator*sizeof(int));
if (NULL == sorted_file_offsets){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
/*Moving file offsets to an IO array!*/
temp_index = 0;
for (i=0;i<fh->f_procs_per_group; i++){
for(j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0){
file_offsets_for_agg[temp_index].length =
blocklen_per_process[i][j];
file_offsets_for_agg[temp_index].process_id = i;
file_offsets_for_agg[temp_index].offset =
displs_per_process[i][j];
temp_index++;
#if DEBUG_ON
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
printf("%d sends blocklen[%d]: %d, disp[%d]: %ld to %d\n",
fh->f_procs_in_group[i],j,
blocklen_per_process[i][j],j,
displs_per_process[i][j],
fh->f_rank);
#endif
}
}
}
}
else{
continue;
}
/* Sort the displacements for each aggregator*/
local_heap_sort (file_offsets_for_agg,
entries_per_aggregator,
sorted_file_offsets);
/*create contiguous memory displacements
based on blocklens on the same displs array
and map it to this aggregator's actual
file-displacements (this is in the io-array created above)*/
memory_displacements = (MPI_Aint *) malloc
(entries_per_aggregator * sizeof(MPI_Aint));
memory_displacements[sorted_file_offsets[0]] = 0;
for (i=1; i<entries_per_aggregator; i++){
memory_displacements[sorted_file_offsets[i]] =
memory_displacements[sorted_file_offsets[i-1]] +
file_offsets_for_agg[sorted_file_offsets[i-1]].length;
}
temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
if (NULL == temp_disp_index) {
opal_output (1, "OUT OF MEMORY\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
/*Now update the displacements array with memory offsets*/
global_count = 0;
for (i=0;i<entries_per_aggregator;i++){
temp_pindex =
file_offsets_for_agg[sorted_file_offsets[i]].process_id;
displs_per_process[temp_pindex][temp_disp_index[temp_pindex]] =
memory_displacements[sorted_file_offsets[i]];
if (temp_disp_index[temp_pindex] < disp_index[temp_pindex])
temp_disp_index[temp_pindex] += 1;
else{
printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
temp_pindex, temp_disp_index[temp_pindex],
temp_pindex, disp_index[temp_pindex]);
}
global_count +=
file_offsets_for_agg[sorted_file_offsets[i]].length;
}
if (NULL != temp_disp_index){
free(temp_disp_index);
temp_disp_index = NULL;
}
#if DEBUG_ON
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
for (i=0;i<fh->f_procs_per_group; i++){
for(j=0;j<disp_index[i];j++){
if (blocklen[i][j] > 0){
printf("%d sends blocklen[%d]: %d, disp[%d]: %ld to %d\n",
fh->f_procs_in_group[i],j,
blocklen_per_process[i][j],j,
displs_per_process[i][j],
fh->f_rank);
}
}
}
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
for (i=0; i<entries_per_aggregator;i++){
printf("%d: OFFSET: %lld LENGTH: %ld, Mem-offset: %ld\n",
file_offsets_for_agg[sorted_file_offsets[i]].process_id,
file_offsets_for_agg[sorted_file_offsets[i]].offset,
file_offsets_for_agg[sorted_file_offsets[i]].length,
memory_displacements[sorted_file_offsets[i]]);
}
printf("%d : global_count : %ld, bytes_sent : %d\n",
fh->f_rank,global_count, bytes_sent);
#endif
global_buf = (char *) malloc (global_count);
if (NULL == global_buf){
opal_output(1, "OUT OF MEMORY");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
recv_req_count = 0;
for (i=0;i<fh->f_procs_per_group; i++){
ompi_datatype_create_hindexed(disp_index[i],
blocklen_per_process[i],
displs_per_process[i],
MPI_BYTE,
&recvtype[i]);
ompi_datatype_commit(&recvtype[i]);
MPI_Type_size (recvtype[i],
&datatype_size);
if (datatype_size){
recv_req = (MPI_Request *)realloc
((void *)recv_req, (recv_req_count + 1)*sizeof(MPI_Request));
ret = MCA_PML_CALL(irecv(global_buf,
1,
recvtype[i],
fh->f_procs_in_group[i],
123,
fh->f_comm,
&recv_req[recv_req_count]));
recv_req_count++;
if (OMPI_SUCCESS != ret){
goto exit;
}
}
}
}
#if TIME_BREAKDOWN
end_cio_array = MPI_Wtime();
tcio_array = end_cio_array - start_cio_array;
#endif
if (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY) {
send_buf = &((char*)buf)[total_bytes_written];
}
else if (bytes_sent) {
/* allocate a send buffer and copy the data that needs
to be sent into it in case the data is non-contigous
in memory */
OPAL_PTRDIFF_TYPE mem_address;
size_t remaining = 0;
size_t temp_position = 0;
send_buf = malloc (bytes_sent);
if (NULL == send_buf) {
opal_output (1, "OUT OF MEMORY\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
remaining = bytes_sent;
while (remaining) {
mem_address = (OPAL_PTRDIFF_TYPE)
(decoded_iov[iov_index].iov_base) + current_position;
if (remaining >=
(decoded_iov[iov_index].iov_len - current_position)) {
memcpy (send_buf+temp_position,
(IOVBASE_TYPE *)mem_address,
decoded_iov[iov_index].iov_len - current_position);
remaining = remaining -
(decoded_iov[iov_index].iov_len - current_position);
temp_position = temp_position +
(decoded_iov[iov_index].iov_len - current_position);
iov_index = iov_index + 1;
current_position = 0;
}
else {
memcpy (send_buf+temp_position,
(IOVBASE_TYPE *) mem_address,
remaining);
current_position = current_position + remaining;
remaining = 0;
}
}
}
total_bytes_written += bytes_sent;
/* Gather the sendbuf from each process in appropritate locations in
aggregators*/
send_req = (MPI_Request *) malloc (sizeof(MPI_Request));
if (NULL == send_req){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
if (bytes_sent){
ret = MCA_PML_CALL(isend(send_buf,
bytes_sent,
MPI_BYTE,
fh->f_procs_in_group[fh->f_aggregator_index],
123,
MCA_PML_BASE_SEND_STANDARD,
fh->f_comm,
send_req));
if ( OMPI_SUCCESS != ret ){
goto exit;
}
ret = ompi_request_wait(send_req, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
ret = ompi_request_wait_all (recv_req_count,
recv_req,
MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
#if DEBUG_ON
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank){
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
for (i=0 ; i<global_count/4 ; i++)
printf (" RECV %d \n",((int *)global_buf)[i]);
}
#endif
#if TIME_BREAKDOWN
end_sr = MPI_Wtime();
tsr = end_sr - start_sr;
comm_time += tsr;
#endif
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
if (NULL != send_buf) {
free (send_buf);
send_buf = NULL;
}
}
/**********************************************************
**************** DONE GATHERING OF DATA ******************
*********************************************************/
/**********************************************************
******* Create the io array, and pass it to fbtl *********
*********************************************************/
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
(entries_per_aggregator * sizeof (mca_io_ompio_io_array_t));
if (NULL == fh->f_io_array) {
opal_output(1, "OUT OF MEMORY\n");
return OMPI_ERR_OUT_OF_RESOURCE;
}
fh->f_num_of_io_entries = 0;
/*First entry for every aggregator*/
fh->f_io_array[fh->f_num_of_io_entries].offset =
(IOVBASE_TYPE *)file_offsets_for_agg[sorted_file_offsets[0]].offset;
fh->f_io_array[fh->f_num_of_io_entries].length =
file_offsets_for_agg[sorted_file_offsets[0]].length;
fh->f_io_array[fh->f_num_of_io_entries].memory_address =
global_buf+memory_displacements[sorted_file_offsets[0]];
fh->f_num_of_io_entries++;
for (i=1;i<entries_per_aggregator;i++){
/* If the enrties are contiguous merge them,
else make a new entry */
if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
file_offsets_for_agg[sorted_file_offsets[i]].offset){
fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
file_offsets_for_agg[sorted_file_offsets[i]].length;
}
else {
fh->f_io_array[fh->f_num_of_io_entries].offset =
(IOVBASE_TYPE *)file_offsets_for_agg[sorted_file_offsets[i]].offset;
fh->f_io_array[fh->f_num_of_io_entries].length =
file_offsets_for_agg[sorted_file_offsets[i]].length;
fh->f_io_array[fh->f_num_of_io_entries].memory_address =
global_buf+memory_displacements[sorted_file_offsets[i]];
fh->f_num_of_io_entries++;
}
}
#if DEBUG_ON
printf("*************************** %d\n", fh->f_num_of_io_entries);
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
printf(" ADDRESS: %p OFFSET: %ld LENGTH: %ld\n",
fh->f_io_array[i].memory_address,
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].offset,
fh->f_io_array[i].length);
}
#endif
#if TIME_BREAKDOWN
start_ptime = MPI_Wtime();
#endif
if (fh->f_num_of_io_entries) {
if (OMPI_SUCCESS != fh->f_fbtl->fbtl_pwritev (fh, NULL)) {
opal_output (1, "WRITE FAILED\n");
ret = OMPI_ERROR;
goto exit;
}
}
#if TIME_BREAKDOWN
end_ptime = MPI_Wtime();
tpw = end_ptime - start_ptime;
write_time += tpw;
#endif
}
if (NULL != send_req){
free(send_req);
send_req = NULL;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
fh->f_num_of_io_entries = 0;
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
if (NULL != recvtype){
free(recvtype);
recvtype=NULL;
}
if (NULL != recv_req){
free(recv_req);
recv_req = NULL;
}
if (NULL != global_buf) {
free (global_buf);
global_buf = NULL;
}
}
}
#if TIME_BREAKDOWN
end_time2 = MPI_Wtime();
total_io = end_time2-start_time2;
fh->f_comm->c_coll.coll_allreduce (&total_io,
&max_io,
1,
MPI_DOUBLE,
MPI_MAX,
fh->f_comm,
fh->f_comm->c_coll.coll_allreduce_module);
fh->f_comm->c_coll.coll_allreduce (&comm_time,
&max_comm_time,
1,
MPI_DOUBLE,
MPI_SUM,
fh->f_comm,
fh->f_comm->c_coll.coll_allreduce_module);
fh->f_comm->c_coll.coll_allreduce (&write_time,
&max_write_time,
1,
MPI_DOUBLE,
MPI_SUM,
fh->f_comm,
fh->f_comm->c_coll.coll_allreduce_module);
if (0 == fh->f_rank){
printf ("Max Exchange and write ---- %f\n", max_io);
printf ("AVG pwrite time : %f \n", max_write_time/mca_fcoll_dynamic_num_io_procs);
printf ("AVG communication time : %f\n", max_comm_time/fh->f_size);
}
fh->f_comm->c_coll.coll_allreduce (&comm_time,
&max_comm_time,
1,
MPI_DOUBLE,
MPI_MAX,
fh->f_comm,
fh->f_comm->c_coll.coll_allreduce_module);
fh->f_comm->c_coll.coll_allreduce (&write_time,
&max_write_time,
1,
MPI_DOUBLE,
MPI_MAX,
fh->f_comm,
fh->f_comm->c_coll.coll_allreduce_module);
if (0 == fh->f_rank){
printf ("MAX pwrite time : %f \n", max_write_time);
printf ("MAX communication time : %f\n", max_comm_time);
}
fh->f_comm->c_coll.coll_allreduce (&comm_time,
&max_comm_time,
1,
MPI_DOUBLE,
MPI_MIN,
fh->f_comm,
fh->f_comm->c_coll.coll_allreduce_module);
if (0 == fh->f_rank){
printf ("MIN communication time : %f\n", max_comm_time);
}
#endif
exit :
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
if (NULL != disp_index){
free(disp_index);
disp_index = NULL;
}
if (NULL != recvtype){
free(recvtype);
recvtype=NULL;
}
if (NULL != recv_req){
free(recv_req);
recv_req = NULL;
}
if (NULL != global_buf) {
free (global_buf);
global_buf = NULL;
}
for(l=0;l<fh->f_procs_per_group;l++){
if (NULL != blocklen_per_process[l]){
free(blocklen_per_process[l]);
blocklen_per_process[l] = NULL;
}
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
}
if (NULL != blocklen_per_process){
free(blocklen_per_process);
blocklen_per_process = NULL;
}
if (NULL != displs_per_process){
free(displs_per_process);
displs_per_process = NULL;
}
}
if (NULL != sorted) {
free (sorted);
sorted = NULL;
}
if (NULL != global_iov_array) {
free (global_iov_array);
global_iov_array = NULL;
}
if (NULL != fview_count) {
free (fview_count);
fview_count = NULL;
}
if (NULL != decoded_iov) {
free (decoded_iov);
decoded_iov = NULL;
}
if (NULL != send_req){
free(send_req);
send_req = NULL;
}
return OMPI_SUCCESS;
}
/*
* gather_intra_linear_sync
*
* Function: - synchronized gather operation with
* Accepts: - same arguments as MPI_Gather(), first segment size
* Returns: - MPI_SUCCESS or error code
*/
int
ompi_coll_base_gather_intra_linear_sync(const void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void *rbuf, int rcount,
struct ompi_datatype_t *rdtype,
int root,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module,
int first_segment_size)
{
int i, ret, line, rank, size, first_segment_count;
ompi_request_t **reqs = NULL;
MPI_Aint extent, lb;
size_t typelng;
size = ompi_comm_size(comm);
rank = ompi_comm_rank(comm);
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
"ompi_coll_base_gather_intra_linear_sync rank %d, segment %d", rank, first_segment_size));
if (rank != root) {
/* Non-root processes:
- receive zero byte message from the root,
- send the first segment of the data synchronously,
- send the second segment of the data.
*/
ompi_datatype_type_size(sdtype, &typelng);
ompi_datatype_get_extent(sdtype, &lb, &extent);
first_segment_count = scount;
COLL_BASE_COMPUTED_SEGCOUNT( (size_t) first_segment_size, typelng,
first_segment_count );
ret = MCA_PML_CALL(recv(sbuf, 0, MPI_BYTE, root,
MCA_COLL_BASE_TAG_GATHER,
comm, MPI_STATUS_IGNORE));
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
ret = MCA_PML_CALL(send(sbuf, first_segment_count, sdtype, root,
MCA_COLL_BASE_TAG_GATHER,
MCA_PML_BASE_SEND_STANDARD, comm));
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
ret = MCA_PML_CALL(send((char*)sbuf + extent * first_segment_count,
(scount - first_segment_count), sdtype,
root, MCA_COLL_BASE_TAG_GATHER,
MCA_PML_BASE_SEND_STANDARD, comm));
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
} else {
/* Root process,
- For every non-root node:
- post irecv for the first segment of the message
- send zero byte message to signal node to send the message
- post irecv for the second segment of the message
- wait for the first segment to complete
- Copy local data if necessary
- Waitall for all the second segments to complete.
*/
char *ptmp;
ompi_request_t *first_segment_req;
reqs = (ompi_request_t**) calloc(size, sizeof(ompi_request_t*));
if (NULL == reqs) { ret = -1; line = __LINE__; goto error_hndl; }
ompi_datatype_type_size(rdtype, &typelng);
ompi_datatype_get_extent(rdtype, &lb, &extent);
first_segment_count = rcount;
COLL_BASE_COMPUTED_SEGCOUNT( (size_t)first_segment_size, typelng,
first_segment_count );
ptmp = (char *) rbuf;
for (i = 0; i < size; ++i) {
if (i == rank) {
/* skip myself */
reqs[i] = MPI_REQUEST_NULL;
continue;
}
/* irecv for the first segment from i */
ptmp = (char*)rbuf + (ptrdiff_t)i * (ptrdiff_t)rcount * extent;
ret = MCA_PML_CALL(irecv(ptmp, first_segment_count, rdtype, i,
MCA_COLL_BASE_TAG_GATHER, comm,
&first_segment_req));
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
/* send sync message */
ret = MCA_PML_CALL(send(rbuf, 0, MPI_BYTE, i,
MCA_COLL_BASE_TAG_GATHER,
MCA_PML_BASE_SEND_STANDARD, comm));
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
/* irecv for the second segment */
ptmp = (char*)rbuf + ((ptrdiff_t)i * (ptrdiff_t)rcount + first_segment_count) * extent;
ret = MCA_PML_CALL(irecv(ptmp, (rcount - first_segment_count),
rdtype, i, MCA_COLL_BASE_TAG_GATHER, comm,
&reqs[i]));
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
/* wait on the first segment to complete */
ret = ompi_request_wait(&first_segment_req, MPI_STATUS_IGNORE);
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
}
/* copy local data if necessary */
if (MPI_IN_PLACE != sbuf) {
ret = ompi_datatype_sndrcv((void *)sbuf, scount, sdtype,
(char*)rbuf + (ptrdiff_t)rank * (ptrdiff_t)rcount * extent,
rcount, rdtype);
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
}
/* wait all second segments to complete */
ret = ompi_request_wait_all(size, reqs, MPI_STATUSES_IGNORE);
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
free(reqs);
}
/* All done */
return MPI_SUCCESS;
error_hndl:
if (NULL != reqs) {
free(reqs);
}
OPAL_OUTPUT (( ompi_coll_base_framework.framework_output,
"ERROR_HNDL: node %d file %s line %d error %d\n",
rank, __FILE__, line, ret ));
return ret;
}
int mca_io_ompio_finalize_initial_grouping(mca_io_ompio_file_t *fh,
int num_groups,
contg *contg_groups)
{
int z = 0;
int y = 0;
int r = 0;
MPI_Request *sendreq = NULL , *req = NULL;
req = (MPI_Request *)malloc (2* sizeof(MPI_Request));
if (NULL == req) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
fh->f_init_num_aggrs = num_groups;
fh->f_init_aggr_list = (int*)malloc (fh->f_init_num_aggrs * sizeof(int));
if (NULL == fh->f_init_aggr_list) {
opal_output (1, "OUT OF MEMORY\n");
free(req);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if(OMPIO_ROOT == fh->f_rank){
sendreq = (MPI_Request *)malloc ( 2 *fh->f_size * sizeof(MPI_Request));
if (NULL == sendreq) {
free(req);
return OMPI_ERR_OUT_OF_RESOURCE;
}
for( z = 0 ;z < num_groups; z++){
for( y = 0; y < contg_groups[z].procs_per_contg_group; y++){
MCA_PML_CALL(isend(&contg_groups[z].procs_per_contg_group,
1,
MPI_INT,
contg_groups[z].procs_in_contg_group[y],
OMPIO_PROCS_PER_GROUP_TAG,
MCA_PML_BASE_SEND_STANDARD,
fh->f_comm,
&sendreq[r++]));
//send initial grouping distribution to all processes in the group
MCA_PML_CALL(isend(contg_groups[z].procs_in_contg_group,
contg_groups[z].procs_per_contg_group,
MPI_INT,
contg_groups[z].procs_in_contg_group[y],
OMPIO_PROCS_IN_GROUP_TAG,
MCA_PML_BASE_SEND_STANDARD,
fh->f_comm,
&sendreq[r++]));
}
}
}
//All processes receive initial procs per group from OMPIO_ROOT
MCA_PML_CALL(irecv(&fh->f_init_procs_per_group,
1,
MPI_INT,
OMPIO_ROOT,
OMPIO_PROCS_PER_GROUP_TAG,
fh->f_comm,
&req[0]));
ompi_request_wait (&req[0], MPI_STATUS_IGNORE);
fh->f_init_procs_in_group = (int*)malloc (fh->f_init_procs_per_group * sizeof(int));
if (NULL == fh->f_init_procs_in_group) {
opal_output (1, "OUT OF MEMORY\n");
free(req);
if (NULL != sendreq) {
free(sendreq);
}
return OMPI_ERR_OUT_OF_RESOURCE;
}
//All processes receive initial process distribution from OMPIO_ROOT
MCA_PML_CALL(irecv(fh->f_init_procs_in_group,
fh->f_init_procs_per_group,
MPI_INT,
OMPIO_ROOT,
OMPIO_PROCS_IN_GROUP_TAG,
fh->f_comm,
&req[1]));
ompi_request_wait (&req[1], MPI_STATUS_IGNORE);
free (req);
if(OMPIO_ROOT == fh->f_rank){
ompi_request_wait_all (r, sendreq, MPI_STATUSES_IGNORE);
free (sendreq);
}
/*set initial aggregator list */
//OMPIO_ROOT broadcasts aggr list
if(OMPIO_ROOT == fh->f_rank){
for( z = 0 ;z < num_groups; z++){
fh->f_init_aggr_list[z] = contg_groups[z].procs_in_contg_group[0];
}
}
fh->f_comm->c_coll.coll_bcast (fh->f_init_aggr_list,
num_groups,
MPI_INT,
OMPIO_ROOT,
fh->f_comm,
fh->f_comm->c_coll.coll_bcast_module);
return OMPI_SUCCESS;
}
int
mca_fcoll_dynamic_file_write_all (ompio_file_t *fh,
const void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
MPI_Aint total_bytes_written = 0; /* total bytes that have been written*/
MPI_Aint total_bytes = 0; /* total bytes to be written */
MPI_Aint bytes_to_write_in_cycle = 0; /* left to be written in a cycle*/
MPI_Aint bytes_per_cycle = 0; /* total written in each cycle by each process*/
int index = 0;
int cycles = 0;
int i=0, j=0, l=0;
int n=0; /* current position in total_bytes_per_process array */
MPI_Aint bytes_remaining = 0; /* how many bytes have been written from the current
value from total_bytes_per_process */
int bytes_sent = 0, ret =0;
int blocks=0, entries_per_aggregator=0;
/* iovec structure and count of the buffer passed in */
uint32_t iov_count = 0;
struct iovec *decoded_iov = NULL;
int iov_index = 0;
char *send_buf = NULL;
size_t current_position = 0;
struct iovec *local_iov_array=NULL, *global_iov_array=NULL;
mca_io_ompio_local_io_array *file_offsets_for_agg=NULL;
/* global iovec at the writers that contain the iovecs created from
file_set_view */
uint32_t total_fview_count = 0;
int local_count = 0, temp_pindex;
int *fview_count = NULL, *disp_index=NULL, *temp_disp_index=NULL;
int current_index = 0, temp_index=0;
char *global_buf = NULL;
MPI_Aint global_count = 0;
/* array that contains the sorted indices of the global_iov */
int *sorted = NULL, *sorted_file_offsets=NULL;
int *displs = NULL;
int dynamic_num_io_procs;
size_t max_data = 0, datatype_size = 0;
int **blocklen_per_process=NULL;
MPI_Aint **displs_per_process=NULL, *memory_displacements=NULL;
ompi_datatype_t **recvtype = NULL;
MPI_Aint *total_bytes_per_process = NULL;
MPI_Request send_req=NULL, *recv_req=NULL;
int my_aggregator=-1;
bool sendbuf_is_contiguous = false;
size_t ftype_size;
ptrdiff_t ftype_extent, lb;
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
double write_time = 0.0, start_write_time = 0.0, end_write_time = 0.0;
double comm_time = 0.0, start_comm_time = 0.0, end_comm_time = 0.0;
double exch_write = 0.0, start_exch = 0.0, end_exch = 0.0;
mca_common_ompio_print_entry nentry;
#endif
opal_datatype_type_size ( &datatype->super, &ftype_size );
opal_datatype_get_extent ( &datatype->super, &lb, &ftype_extent );
/**************************************************************************
** 1. In case the data is not contigous in memory, decode it into an iovec
**************************************************************************/
if ( ( ftype_extent == (ptrdiff_t) ftype_size) &&
opal_datatype_is_contiguous_memory_layout(&datatype->super,1) &&
0 == lb ) {
sendbuf_is_contiguous = true;
}
if (! sendbuf_is_contiguous ) {
ret = mca_common_ompio_decode_datatype ((struct ompio_file_t *) fh,
datatype,
count,
buf,
&max_data,
&decoded_iov,
&iov_count);
if (OMPI_SUCCESS != ret ){
goto exit;
}
}
else {
max_data = count * datatype->super.size;
}
if ( MPI_STATUS_IGNORE != status ) {
status->_ucount = max_data;
}
dynamic_num_io_procs = fh->f_get_mca_parameter_value ( "num_aggregators", strlen ("num_aggregators"));
if ( OMPI_ERR_MAX == dynamic_num_io_procs ) {
ret = OMPI_ERROR;
goto exit;
}
ret = mca_common_ompio_set_aggregator_props ((struct ompio_file_t *) fh,
dynamic_num_io_procs,
max_data);
if (OMPI_SUCCESS != ret){
goto exit;
}
my_aggregator = fh->f_procs_in_group[0];
/**************************************************************************
** 2. Determine the total amount of data to be written
**************************************************************************/
total_bytes_per_process = (MPI_Aint*)malloc
(fh->f_procs_per_group*sizeof(MPI_Aint));
if (NULL == total_bytes_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_comm_time = MPI_Wtime();
#endif
ret = ompi_fcoll_base_coll_allgather_array (&max_data,
1,
MPI_LONG,
total_bytes_per_process,
1,
MPI_LONG,
0,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if( OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_comm_time = MPI_Wtime();
comm_time += (end_comm_time - start_comm_time);
#endif
for (i=0 ; i<fh->f_procs_per_group ; i++) {
total_bytes += total_bytes_per_process[i];
}
if (NULL != total_bytes_per_process) {
free (total_bytes_per_process);
total_bytes_per_process = NULL;
}
/*********************************************************************
*** 3. Generate the local offsets/lengths array corresponding to
*** this write operation
********************************************************************/
ret = fh->f_generate_current_file_view( (struct ompio_file_t *) fh,
max_data,
&local_iov_array,
&local_count);
if (ret != OMPI_SUCCESS){
goto exit;
}
#if DEBUG_ON
for (i=0 ; i<local_count ; i++) {
printf("%d: OFFSET: %d LENGTH: %ld\n",
fh->f_rank,
local_iov_array[i].iov_base,
local_iov_array[i].iov_len);
}
#endif
/*************************************************************
*** 4. Allgather the offset/lengths array from all processes
*************************************************************/
fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == fview_count) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_comm_time = MPI_Wtime();
#endif
ret = ompi_fcoll_base_coll_allgather_array (&local_count,
1,
MPI_INT,
fview_count,
1,
MPI_INT,
0,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if( OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_comm_time = MPI_Wtime();
comm_time += (end_comm_time - start_comm_time);
#endif
displs = (int*) malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == displs) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs[0] = 0;
total_fview_count = fview_count[0];
for (i=1 ; i<fh->f_procs_per_group ; i++) {
total_fview_count += fview_count[i];
displs[i] = displs[i-1] + fview_count[i-1];
}
#if DEBUG_ON
printf("total_fview_count : %d\n", total_fview_count);
if (my_aggregator == fh->f_rank) {
for (i=0 ; i<fh->f_procs_per_group ; i++) {
printf ("%d: PROCESS: %d ELEMENTS: %d DISPLS: %d\n",
fh->f_rank,
i,
fview_count[i],
displs[i]);
}
}
#endif
/* allocate the global iovec */
if (0 != total_fview_count) {
global_iov_array = (struct iovec*) malloc (total_fview_count *
sizeof(struct iovec));
if (NULL == global_iov_array){
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_comm_time = MPI_Wtime();
#endif
ret = ompi_fcoll_base_coll_allgatherv_array (local_iov_array,
local_count,
fh->f_iov_type,
global_iov_array,
fview_count,
displs,
fh->f_iov_type,
0,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_comm_time = MPI_Wtime();
comm_time += (end_comm_time - start_comm_time);
#endif
/****************************************************************************************
*** 5. Sort the global offset/lengths list based on the offsets.
*** The result of the sort operation is the 'sorted', an integer array,
*** which contains the indexes of the global_iov_array based on the offset.
*** For example, if global_iov_array[x].offset is followed by global_iov_array[y].offset
*** in the file, and that one is followed by global_iov_array[z].offset, than
*** sorted[0] = x, sorted[1]=y and sorted[2]=z;
******************************************************************************************/
if (0 != total_fview_count) {
sorted = (int *)malloc (total_fview_count * sizeof(int));
if (NULL == sorted) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ompi_fcoll_base_sort_iovec (global_iov_array, total_fview_count, sorted);
}
if (NULL != local_iov_array){
free(local_iov_array);
local_iov_array = NULL;
}
if (NULL != displs){
free(displs);
displs=NULL;
}
#if DEBUG_ON
if (my_aggregator == fh->f_rank) {
uint32_t tv=0;
for (tv=0 ; tv<total_fview_count ; tv++) {
printf("%d: OFFSET: %lld LENGTH: %ld\n",
fh->f_rank,
global_iov_array[sorted[tv]].iov_base,
global_iov_array[sorted[tv]].iov_len);
}
}
#endif
/*************************************************************
*** 6. Determine the number of cycles required to execute this
*** operation
*************************************************************/
bytes_per_cycle = fh->f_bytes_per_agg;
cycles = ceil((double)total_bytes/bytes_per_cycle);
if (my_aggregator == fh->f_rank) {
disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
blocklen_per_process = (int **)calloc (fh->f_procs_per_group, sizeof (int*));
if (NULL == blocklen_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process = (MPI_Aint **)calloc (fh->f_procs_per_group, sizeof (MPI_Aint*));
if (NULL == displs_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
recv_req = (MPI_Request *)malloc ((fh->f_procs_per_group)*sizeof(MPI_Request));
if ( NULL == recv_req ) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
global_buf = (char *) malloc (bytes_per_cycle);
if (NULL == global_buf){
opal_output(1, "OUT OF MEMORY");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
recvtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
if (NULL == recvtype) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for(l=0;l<fh->f_procs_per_group;l++){
recvtype[l] = MPI_DATATYPE_NULL;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_exch = MPI_Wtime();
#endif
n = 0;
bytes_remaining = 0;
current_index = 0;
for (index = 0; index < cycles; index++) {
/**********************************************************************
*** 7a. Getting ready for next cycle: initializing and freeing buffers
**********************************************************************/
if (my_aggregator == fh->f_rank) {
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
fh->f_num_of_io_entries = 0;
if (NULL != recvtype){
for (i =0; i< fh->f_procs_per_group; i++) {
if ( MPI_DATATYPE_NULL != recvtype[i] ) {
ompi_datatype_destroy(&recvtype[i]);
recvtype[i] = MPI_DATATYPE_NULL;
}
}
}
for(l=0;l<fh->f_procs_per_group;l++){
disp_index[l] = 1;
free(blocklen_per_process[l]);
free(displs_per_process[l]);
blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
if (NULL == displs_per_process[l] || NULL == blocklen_per_process[l]){
opal_output (1, "OUT OF MEMORY for displs\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if(NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements = NULL;
}
} /* (my_aggregator == fh->f_rank */
/**************************************************************************
*** 7b. Determine the number of bytes to be actually written in this cycle
**************************************************************************/
if (cycles-1 == index) {
bytes_to_write_in_cycle = total_bytes - bytes_per_cycle*index;
}
else {
bytes_to_write_in_cycle = bytes_per_cycle;
}
#if DEBUG_ON
if (my_aggregator == fh->f_rank) {
printf ("****%d: CYCLE %d Bytes %lld**********\n",
fh->f_rank,
index,
bytes_to_write_in_cycle);
}
#endif
/**********************************************************
**Gather the Data from all the processes at the writers **
*********************************************************/
#if DEBUG_ON
printf("bytes_to_write_in_cycle: %ld, cycle : %d\n", bytes_to_write_in_cycle,
index);
#endif
/*****************************************************************
*** 7c. Calculate how much data will be contributed in this cycle
*** by each process
*****************************************************************/
bytes_sent = 0;
/* The blocklen and displs calculation only done at aggregators!*/
while (bytes_to_write_in_cycle) {
/* This next block identifies which process is the holder
** of the sorted[current_index] element;
*/
blocks = fview_count[0];
for (j=0 ; j<fh->f_procs_per_group ; j++) {
if (sorted[current_index] < blocks) {
n = j;
break;
}
else {
blocks += fview_count[j+1];
}
}
if (bytes_remaining) {
/* Finish up a partially used buffer from the previous cycle */
if (bytes_remaining <= bytes_to_write_in_cycle) {
/* The data fits completely into the block */
if (my_aggregator == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_remaining;
displs_per_process[n][disp_index[n] - 1] =
(ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
(global_iov_array[sorted[current_index]].iov_len
- bytes_remaining);
/* In this cases the length is consumed so allocating for
next displacement and blocklength*/
blocklen_per_process[n] = (int *) realloc
((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
displs_per_process[n] = (MPI_Aint *) realloc
((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
blocklen_per_process[n][disp_index[n]] = 0;
displs_per_process[n][disp_index[n]] = 0;
disp_index[n] += 1;
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_sent += bytes_remaining;
}
current_index ++;
bytes_to_write_in_cycle -= bytes_remaining;
bytes_remaining = 0;
continue;
}
else {
/* the remaining data from the previous cycle is larger than the
bytes_to_write_in_cycle, so we have to segment again */
if (my_aggregator == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_to_write_in_cycle;
displs_per_process[n][disp_index[n] - 1] =
(ptrdiff_t)global_iov_array[sorted[current_index]].iov_base +
(global_iov_array[sorted[current_index]].iov_len
- bytes_remaining);
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_sent += bytes_to_write_in_cycle;
}
bytes_remaining -= bytes_to_write_in_cycle;
bytes_to_write_in_cycle = 0;
break;
}
}
else {
/* No partially used entry available, have to start a new one */
if (bytes_to_write_in_cycle <
(MPI_Aint) global_iov_array[sorted[current_index]].iov_len) {
/* This entry has more data than we can sendin one cycle */
if (my_aggregator == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] = bytes_to_write_in_cycle;
displs_per_process[n][disp_index[n] - 1] =
(ptrdiff_t)global_iov_array[sorted[current_index]].iov_base ;
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_sent += bytes_to_write_in_cycle;
}
bytes_remaining = global_iov_array[sorted[current_index]].iov_len -
bytes_to_write_in_cycle;
bytes_to_write_in_cycle = 0;
break;
}
else {
/* Next data entry is less than bytes_to_write_in_cycle */
if (my_aggregator == fh->f_rank) {
blocklen_per_process[n][disp_index[n] - 1] =
global_iov_array[sorted[current_index]].iov_len;
displs_per_process[n][disp_index[n] - 1] = (ptrdiff_t)
global_iov_array[sorted[current_index]].iov_base;
/*realloc for next blocklength
and assign this displacement and check for next displs as
the total length of this entry has been consumed!*/
blocklen_per_process[n] =
(int *) realloc ((void *)blocklen_per_process[n], (disp_index[n]+1)*sizeof(int));
displs_per_process[n] = (MPI_Aint *)realloc
((void *)displs_per_process[n], (disp_index[n]+1)*sizeof(MPI_Aint));
blocklen_per_process[n][disp_index[n]] = 0;
displs_per_process[n][disp_index[n]] = 0;
disp_index[n] += 1;
}
if (fh->f_procs_in_group[n] == fh->f_rank) {
bytes_sent += global_iov_array[sorted[current_index]].iov_len;
}
bytes_to_write_in_cycle -=
global_iov_array[sorted[current_index]].iov_len;
current_index ++;
continue;
}
}
}
/*************************************************************************
*** 7d. Calculate the displacement on where to put the data and allocate
*** the recieve buffer (global_buf)
*************************************************************************/
if (my_aggregator == fh->f_rank) {
entries_per_aggregator=0;
for (i=0;i<fh->f_procs_per_group; i++){
for (j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0)
entries_per_aggregator++ ;
}
}
#if DEBUG_ON
printf("%d: cycle: %d, bytes_sent: %d\n ",fh->f_rank,index,
bytes_sent);
printf("%d : Entries per aggregator : %d\n",fh->f_rank,entries_per_aggregator);
#endif
if (entries_per_aggregator > 0){
file_offsets_for_agg = (mca_io_ompio_local_io_array *)
malloc(entries_per_aggregator*sizeof(mca_io_ompio_local_io_array));
if (NULL == file_offsets_for_agg) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
sorted_file_offsets = (int *)
malloc (entries_per_aggregator*sizeof(int));
if (NULL == sorted_file_offsets){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
/*Moving file offsets to an IO array!*/
temp_index = 0;
for (i=0;i<fh->f_procs_per_group; i++){
for(j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0){
file_offsets_for_agg[temp_index].length =
blocklen_per_process[i][j];
file_offsets_for_agg[temp_index].process_id = i;
file_offsets_for_agg[temp_index].offset =
displs_per_process[i][j];
temp_index++;
#if DEBUG_ON
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
printf("%d sends blocklen[%d]: %d, disp[%d]: %ld to %d\n",
fh->f_procs_in_group[i],j,
blocklen_per_process[i][j],j,
displs_per_process[i][j],
fh->f_rank);
#endif
}
}
}
}
else{
continue;
}
/* Sort the displacements for each aggregator*/
local_heap_sort (file_offsets_for_agg,
entries_per_aggregator,
sorted_file_offsets);
/*create contiguous memory displacements
based on blocklens on the same displs array
and map it to this aggregator's actual
file-displacements (this is in the io-array created above)*/
memory_displacements = (MPI_Aint *) malloc
(entries_per_aggregator * sizeof(MPI_Aint));
memory_displacements[sorted_file_offsets[0]] = 0;
for (i=1; i<entries_per_aggregator; i++){
memory_displacements[sorted_file_offsets[i]] =
memory_displacements[sorted_file_offsets[i-1]] +
file_offsets_for_agg[sorted_file_offsets[i-1]].length;
}
temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
if (NULL == temp_disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
/*Now update the displacements array with memory offsets*/
global_count = 0;
for (i=0;i<entries_per_aggregator;i++){
temp_pindex =
file_offsets_for_agg[sorted_file_offsets[i]].process_id;
displs_per_process[temp_pindex][temp_disp_index[temp_pindex]] =
memory_displacements[sorted_file_offsets[i]];
if (temp_disp_index[temp_pindex] < disp_index[temp_pindex])
temp_disp_index[temp_pindex] += 1;
else{
printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
temp_pindex, temp_disp_index[temp_pindex],
temp_pindex, disp_index[temp_pindex]);
}
global_count +=
file_offsets_for_agg[sorted_file_offsets[i]].length;
}
if (NULL != temp_disp_index){
free(temp_disp_index);
temp_disp_index = NULL;
}
#if DEBUG_ON
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
for (i=0;i<fh->f_procs_per_group; i++){
for(j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0){
printf("%d sends blocklen[%d]: %d, disp[%d]: %ld to %d\n",
fh->f_procs_in_group[i],j,
blocklen_per_process[i][j],j,
displs_per_process[i][j],
fh->f_rank);
}
}
}
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
for (i=0; i<entries_per_aggregator;i++){
printf("%d: OFFSET: %lld LENGTH: %ld, Mem-offset: %ld\n",
file_offsets_for_agg[sorted_file_offsets[i]].process_id,
file_offsets_for_agg[sorted_file_offsets[i]].offset,
file_offsets_for_agg[sorted_file_offsets[i]].length,
memory_displacements[sorted_file_offsets[i]]);
}
printf("%d : global_count : %ld, bytes_sent : %d\n",
fh->f_rank,global_count, bytes_sent);
#endif
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_comm_time = MPI_Wtime();
#endif
/*************************************************************************
*** 7e. Perform the actual communication
*************************************************************************/
for (i=0;i<fh->f_procs_per_group; i++) {
recv_req[i] = MPI_REQUEST_NULL;
if ( 0 < disp_index[i] ) {
ompi_datatype_create_hindexed(disp_index[i],
blocklen_per_process[i],
displs_per_process[i],
MPI_BYTE,
&recvtype[i]);
ompi_datatype_commit(&recvtype[i]);
opal_datatype_type_size(&recvtype[i]->super, &datatype_size);
if (datatype_size){
ret = MCA_PML_CALL(irecv(global_buf,
1,
recvtype[i],
fh->f_procs_in_group[i],
123,
fh->f_comm,
&recv_req[i]));
if (OMPI_SUCCESS != ret){
goto exit;
}
}
}
}
} /* end if (my_aggregator == fh->f_rank ) */
if ( sendbuf_is_contiguous ) {
send_buf = &((char*)buf)[total_bytes_written];
}
else if (bytes_sent) {
/* allocate a send buffer and copy the data that needs
to be sent into it in case the data is non-contigous
in memory */
ptrdiff_t mem_address;
size_t remaining = 0;
size_t temp_position = 0;
send_buf = malloc (bytes_sent);
if (NULL == send_buf) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
remaining = bytes_sent;
while (remaining) {
mem_address = (ptrdiff_t)
(decoded_iov[iov_index].iov_base) + current_position;
if (remaining >=
(decoded_iov[iov_index].iov_len - current_position)) {
memcpy (send_buf+temp_position,
(IOVBASE_TYPE *)mem_address,
decoded_iov[iov_index].iov_len - current_position);
remaining = remaining -
(decoded_iov[iov_index].iov_len - current_position);
temp_position = temp_position +
(decoded_iov[iov_index].iov_len - current_position);
iov_index = iov_index + 1;
current_position = 0;
}
else {
memcpy (send_buf+temp_position,
(IOVBASE_TYPE *) mem_address,
remaining);
current_position = current_position + remaining;
remaining = 0;
}
}
}
total_bytes_written += bytes_sent;
/* Gather the sendbuf from each process in appropritate locations in
aggregators*/
if (bytes_sent){
ret = MCA_PML_CALL(isend(send_buf,
bytes_sent,
MPI_BYTE,
my_aggregator,
123,
MCA_PML_BASE_SEND_STANDARD,
fh->f_comm,
&send_req));
if ( OMPI_SUCCESS != ret ){
goto exit;
}
ret = ompi_request_wait(&send_req, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
if (my_aggregator == fh->f_rank) {
ret = ompi_request_wait_all (fh->f_procs_per_group,
recv_req,
MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
}
#if DEBUG_ON
if (my_aggregator == fh->f_rank){
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
for (i=0 ; i<global_count/4 ; i++)
printf (" RECV %d \n",((int *)global_buf)[i]);
}
#endif
if (! sendbuf_is_contiguous) {
if (NULL != send_buf) {
free (send_buf);
send_buf = NULL;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_comm_time = MPI_Wtime();
comm_time += (end_comm_time - start_comm_time);
#endif
/**********************************************************
*** 7f. Create the io array, and pass it to fbtl
*********************************************************/
if (my_aggregator == fh->f_rank) {
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_write_time = MPI_Wtime();
#endif
fh->f_io_array = (mca_common_ompio_io_array_t *) malloc
(entries_per_aggregator * sizeof (mca_common_ompio_io_array_t));
if (NULL == fh->f_io_array) {
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
fh->f_num_of_io_entries = 0;
/*First entry for every aggregator*/
fh->f_io_array[0].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
fh->f_io_array[0].length =
file_offsets_for_agg[sorted_file_offsets[0]].length;
fh->f_io_array[0].memory_address =
global_buf+memory_displacements[sorted_file_offsets[0]];
fh->f_num_of_io_entries++;
for (i=1;i<entries_per_aggregator;i++){
/* If the enrties are contiguous merge them,
else make a new entry */
if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
file_offsets_for_agg[sorted_file_offsets[i]].offset){
fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
file_offsets_for_agg[sorted_file_offsets[i]].length;
}
else {
fh->f_io_array[fh->f_num_of_io_entries].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
fh->f_io_array[fh->f_num_of_io_entries].length =
file_offsets_for_agg[sorted_file_offsets[i]].length;
fh->f_io_array[fh->f_num_of_io_entries].memory_address =
global_buf+memory_displacements[sorted_file_offsets[i]];
fh->f_num_of_io_entries++;
}
}
#if DEBUG_ON
printf("*************************** %d\n", fh->f_num_of_io_entries);
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
printf(" ADDRESS: %p OFFSET: %ld LENGTH: %ld\n",
fh->f_io_array[i].memory_address,
(ptrdiff_t)fh->f_io_array[i].offset,
fh->f_io_array[i].length);
}
#endif
if (fh->f_num_of_io_entries) {
if ( 0 > fh->f_fbtl->fbtl_pwritev (fh)) {
opal_output (1, "WRITE FAILED\n");
ret = OMPI_ERROR;
goto exit;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_write_time = MPI_Wtime();
write_time += end_write_time - start_write_time;
#endif
} /* end if (my_aggregator == fh->f_rank) */
} /* end for (index = 0; index < cycles; index++) */
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_exch = MPI_Wtime();
exch_write += end_exch - start_exch;
nentry.time[0] = write_time;
nentry.time[1] = comm_time;
nentry.time[2] = exch_write;
if (my_aggregator == fh->f_rank)
nentry.aggregator = 1;
else
nentry.aggregator = 0;
nentry.nprocs_for_coll = dynamic_num_io_procs;
if (!mca_common_ompio_full_print_queue(fh->f_coll_write_time)){
mca_common_ompio_register_print_entry(fh->f_coll_write_time,
nentry);
}
#endif
exit :
if (my_aggregator == fh->f_rank) {
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if(NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements = NULL;
}
if (NULL != recvtype){
for (i =0; i< fh->f_procs_per_group; i++) {
if ( MPI_DATATYPE_NULL != recvtype[i] ) {
ompi_datatype_destroy(&recvtype[i]);
}
}
free(recvtype);
recvtype=NULL;
}
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
if (NULL != disp_index){
free(disp_index);
disp_index = NULL;
}
if (NULL != recvtype){
free(recvtype);
recvtype=NULL;
}
if (NULL != recv_req){
free(recv_req);
recv_req = NULL;
}
if (NULL != global_buf) {
free (global_buf);
global_buf = NULL;
}
for(l=0;l<fh->f_procs_per_group;l++){
if (NULL != blocklen_per_process){
free(blocklen_per_process[l]);
}
if (NULL != displs_per_process){
free(displs_per_process[l]);
}
}
free(blocklen_per_process);
free(displs_per_process);
}
if (NULL != displs){
free(displs);
displs=NULL;
}
if (! sendbuf_is_contiguous) {
if (NULL != send_buf) {
free (send_buf);
send_buf = NULL;
}
}
if (NULL != global_buf) {
free (global_buf);
global_buf = NULL;
}
if (NULL != sorted) {
free (sorted);
sorted = NULL;
}
if (NULL != global_iov_array) {
free (global_iov_array);
global_iov_array = NULL;
}
if (NULL != fview_count) {
free (fview_count);
fview_count = NULL;
}
if (NULL != decoded_iov) {
free (decoded_iov);
decoded_iov = NULL;
}
return OMPI_SUCCESS;
}
/*
* allgather_inter
*
* Function: - allgather using other MPI collections
* Accepts: - same as MPI_Allgather()
* Returns: - MPI_SUCCESS or error code
*/
int
mca_coll_basic_allgather_inter(void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void *rbuf, int rcount,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
int rank, root = 0, size, rsize, err, i;
char *tmpbuf = NULL, *ptmp;
ptrdiff_t rlb, slb, rextent, sextent, incr;
ompi_request_t *req;
mca_coll_basic_module_t *basic_module = (mca_coll_basic_module_t*) module;
ompi_request_t **reqs = basic_module->mccb_reqs;
rank = ompi_comm_rank(comm);
size = ompi_comm_size(comm);
rsize = ompi_comm_remote_size(comm);
/* Algorithm:
* - a gather to the root in remote group (simultaniously executed,
* thats why we cannot use coll_gather).
* - exchange the temp-results between two roots
* - inter-bcast (again simultanious).
*/
/* Step one: gather operations: */
if (rank != root) {
/* send your data to root */
err = MCA_PML_CALL(send(sbuf, scount, sdtype, root,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_PML_BASE_SEND_STANDARD, comm));
if (OMPI_SUCCESS != err) {
return err;
}
} else {
/* receive a msg. from all other procs. */
err = ompi_datatype_get_extent(rdtype, &rlb, &rextent);
if (OMPI_SUCCESS != err) {
return err;
}
err = ompi_datatype_get_extent(sdtype, &slb, &sextent);
if (OMPI_SUCCESS != err) {
return err;
}
/* Do a send-recv between the two root procs. to avoid deadlock */
err = MCA_PML_CALL(isend(sbuf, scount, sdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_PML_BASE_SEND_STANDARD,
comm, &reqs[rsize]));
if (OMPI_SUCCESS != err) {
return err;
}
err = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER, comm,
&reqs[0]));
if (OMPI_SUCCESS != err) {
return err;
}
incr = rextent * rcount;
ptmp = (char *) rbuf + incr;
for (i = 1; i < rsize; ++i, ptmp += incr) {
err = MCA_PML_CALL(irecv(ptmp, rcount, rdtype, i,
MCA_COLL_BASE_TAG_ALLGATHER,
comm, &reqs[i]));
if (MPI_SUCCESS != err) {
return err;
}
}
err = ompi_request_wait_all(rsize + 1, reqs, MPI_STATUSES_IGNORE);
if (OMPI_SUCCESS != err) {
return err;
}
/* Step 2: exchange the resuts between the root processes */
tmpbuf = (char *) malloc(scount * size * sextent);
if (NULL == tmpbuf) {
return err;
}
err = MCA_PML_CALL(isend(rbuf, rsize * rcount, rdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_PML_BASE_SEND_STANDARD, comm, &req));
if (OMPI_SUCCESS != err) {
goto exit;
}
err = MCA_PML_CALL(recv(tmpbuf, size * scount, sdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER, comm,
MPI_STATUS_IGNORE));
if (OMPI_SUCCESS != err) {
goto exit;
}
err = ompi_request_wait( &req, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != err) {
goto exit;
}
}
/* Step 3: bcast the data to the remote group. This
* happens in both groups simultaniously, thus we can
* not use coll_bcast (this would deadlock).
*/
if (rank != root) {
/* post the recv */
err = MCA_PML_CALL(recv(rbuf, rsize * rcount, rdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER, comm,
MPI_STATUS_IGNORE));
if (OMPI_SUCCESS != err) {
goto exit;
}
} else {
/* Send the data to every other process in the remote group
* except to rank zero. which has it already. */
for (i = 1; i < rsize; i++) {
err = MCA_PML_CALL(isend(tmpbuf, size * scount, sdtype, i,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_PML_BASE_SEND_STANDARD,
comm, &reqs[i - 1]));
if (OMPI_SUCCESS != err) {
goto exit;
}
}
err = ompi_request_wait_all(rsize - 1, reqs, MPI_STATUSES_IGNORE);
if (OMPI_SUCCESS != err) {
goto exit;
}
}
exit:
if (NULL != tmpbuf) {
free(tmpbuf);
}
return err;
}
/*
* reduce_scatter_block_inter
*
* Function: - reduce/scatter operation
* Accepts: - same arguments as MPI_Reduce_scatter()
* Returns: - MPI_SUCCESS or error code
*/
int
mca_coll_basic_reduce_scatter_block_inter(const void *sbuf, void *rbuf, int rcount,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
int err, i, rank, root = 0, rsize, lsize;
int totalcounts;
ptrdiff_t gap, span;
char *tmpbuf = NULL, *tmpbuf2 = NULL;
char *lbuf = NULL, *buf;
ompi_request_t *req;
rank = ompi_comm_rank(comm);
rsize = ompi_comm_remote_size(comm);
lsize = ompi_comm_size(comm);
totalcounts = lsize * rcount;
/*
* The following code basically does an interreduce followed by a
* intrascatter. This is implemented by having the roots of each
* group exchange their sbuf. Then, the roots receive the data
* from each of the remote ranks and execute the reduce. When
* this is complete, they have the reduced data available to them
* for doing the scatter. They do this on the local communicator
* associated with the intercommunicator.
*
* Note: There are other ways to implement MPI_Reduce_scatter_block on
* intercommunicators. For example, one could do a MPI_Reduce locally,
* then send the results to the other root which could scatter it.
*
*/
if (rank == root) {
span = opal_datatype_span(&dtype->super, totalcounts, &gap);
tmpbuf = (char *) malloc(span);
tmpbuf2 = (char *) malloc(span);
if (NULL == tmpbuf || NULL == tmpbuf2) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
lbuf = tmpbuf - gap;
buf = tmpbuf2 - gap;
/* Do a send-recv between the two root procs. to avoid deadlock */
err = MCA_PML_CALL(isend(sbuf, totalcounts, dtype, 0,
MCA_COLL_BASE_TAG_REDUCE_SCATTER,
MCA_PML_BASE_SEND_STANDARD, comm, &req));
if (OMPI_SUCCESS != err) {
goto exit;
}
err = MCA_PML_CALL(recv(lbuf, totalcounts, dtype, 0,
MCA_COLL_BASE_TAG_REDUCE_SCATTER, comm,
MPI_STATUS_IGNORE));
if (OMPI_SUCCESS != err) {
goto exit;
}
err = ompi_request_wait( &req, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != err) {
goto exit;
}
/* Loop receiving and calling reduction function (C or Fortran)
* The result of this reduction operations is then in
* tmpbuf2.
*/
for (i = 1; i < rsize; i++) {
char *tbuf;
err = MCA_PML_CALL(recv(buf, totalcounts, dtype, i,
MCA_COLL_BASE_TAG_REDUCE_SCATTER, comm,
MPI_STATUS_IGNORE));
if (MPI_SUCCESS != err) {
goto exit;
}
/* Perform the reduction */
ompi_op_reduce(op, lbuf, buf, totalcounts, dtype);
/* swap the buffers */
tbuf = lbuf; lbuf = buf; buf = tbuf;
}
} else {
/* If not root, send data to the root. */
err = MCA_PML_CALL(send(sbuf, totalcounts, dtype, root,
MCA_COLL_BASE_TAG_REDUCE_SCATTER,
MCA_PML_BASE_SEND_STANDARD, comm));
if (OMPI_SUCCESS != err) {
goto exit;
}
}
/* Now do a scatterv on the local communicator */
err = comm->c_local_comm->c_coll->coll_scatter(lbuf, rcount, dtype,
rbuf, rcount, dtype, 0,
comm->c_local_comm,
comm->c_local_comm->c_coll->coll_scatter_module);
exit:
if (NULL != tmpbuf) {
free(tmpbuf);
}
if (NULL != tmpbuf2) {
free(tmpbuf2);
}
return err;
}
/*
* allgather_inter
*
* Function: - allgather using other MPI collections
* Accepts: - same as MPI_Allgather()
* Returns: - MPI_SUCCESS or error code
*/
int
mca_coll_basic_allgather_inter(const void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void *rbuf, int rcount,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
int rank, root = 0, size, rsize, err, i, line;
char *tmpbuf_free = NULL, *tmpbuf, *ptmp;
ptrdiff_t rlb, rextent, incr;
ptrdiff_t gap, span;
ompi_request_t *req;
ompi_request_t **reqs = NULL;
rank = ompi_comm_rank(comm);
size = ompi_comm_size(comm);
rsize = ompi_comm_remote_size(comm);
/* Algorithm:
* - a gather to the root in remote group (simultaniously executed,
* thats why we cannot use coll_gather).
* - exchange the temp-results between two roots
* - inter-bcast (again simultanious).
*/
/* Step one: gather operations: */
if (rank != root) {
/* send your data to root */
err = MCA_PML_CALL(send(sbuf, scount, sdtype, root,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_PML_BASE_SEND_STANDARD, comm));
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
} else {
/* receive a msg. from all other procs. */
err = ompi_datatype_get_extent(rdtype, &rlb, &rextent);
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
/* Get a requests arrays of the right size */
reqs = ompi_coll_base_comm_get_reqs(module->base_data, rsize + 1);
if( NULL == reqs ) { line = __LINE__; err = OMPI_ERR_OUT_OF_RESOURCE; goto exit; }
/* Do a send-recv between the two root procs. to avoid deadlock */
err = MCA_PML_CALL(isend(sbuf, scount, sdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_PML_BASE_SEND_STANDARD,
comm, &reqs[rsize]));
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
err = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER, comm,
&reqs[0]));
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
incr = rextent * rcount;
ptmp = (char *) rbuf + incr;
for (i = 1; i < rsize; ++i, ptmp += incr) {
err = MCA_PML_CALL(irecv(ptmp, rcount, rdtype, i,
MCA_COLL_BASE_TAG_ALLGATHER,
comm, &reqs[i]));
if (MPI_SUCCESS != err) { line = __LINE__; goto exit; }
}
err = ompi_request_wait_all(rsize + 1, reqs, MPI_STATUSES_IGNORE);
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
/* Step 2: exchange the resuts between the root processes */
span = opal_datatype_span(&sdtype->super, (int64_t)scount * (int64_t)size, &gap);
tmpbuf_free = (char *) malloc(span);
if (NULL == tmpbuf_free) { line = __LINE__; err = OMPI_ERR_OUT_OF_RESOURCE; goto exit; }
tmpbuf = tmpbuf_free - gap;
err = MCA_PML_CALL(isend(rbuf, rsize * rcount, rdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_PML_BASE_SEND_STANDARD, comm, &req));
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
err = MCA_PML_CALL(recv(tmpbuf, size * scount, sdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER, comm,
MPI_STATUS_IGNORE));
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
err = ompi_request_wait( &req, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
}
/* Step 3: bcast the data to the remote group. This
* happens in both groups simultaneously, thus we can
* not use coll_bcast (this would deadlock).
*/
if (rank != root) {
/* post the recv */
err = MCA_PML_CALL(recv(rbuf, rsize * rcount, rdtype, 0,
MCA_COLL_BASE_TAG_ALLGATHER, comm,
MPI_STATUS_IGNORE));
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
} else {
/* Send the data to every other process in the remote group
* except to rank zero. which has it already. */
for (i = 1; i < rsize; i++) {
err = MCA_PML_CALL(isend(tmpbuf, size * scount, sdtype, i,
MCA_COLL_BASE_TAG_ALLGATHER,
MCA_PML_BASE_SEND_STANDARD,
comm, &reqs[i - 1]));
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
}
err = ompi_request_wait_all(rsize - 1, reqs, MPI_STATUSES_IGNORE);
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
}
exit:
if( MPI_SUCCESS != err ) {
OPAL_OUTPUT( (ompi_coll_base_framework.framework_output,"%s:%4d\tError occurred %d, rank %2d",
__FILE__, line, err, rank) );
(void)line; // silence compiler warning
if( NULL != reqs ) ompi_coll_base_free_reqs(reqs, rsize+1);
}
if (NULL != tmpbuf_free) {
free(tmpbuf_free);
}
return err;
}
static int mca_coll_hierarch_bcast_intra_seg3 (void *buff,
int count,
struct ompi_datatype_t *datatype,
int root,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module,
int segsize )
{
struct ompi_communicator_t *llcomm=NULL;
struct ompi_communicator_t *lcomm=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
int lroot=MPI_UNDEFINED, llroot=MPI_UNDEFINED;
int llrank=MPI_UNDEFINED, llsize=0, rank=0, ret=OMPI_SUCCESS;
int lsize=0, lrank=MPI_UNDEFINED;
MPI_Aint ub=0, typeext=0;
size_t typesize=0;
int i, realsegsize=0, remaining_count=0;
int num_segments=0, segcount=0, segindex=0;
char* tmpbuf = (char *) buff;
ompi_request_t **sreq=NULL, **sreq1=NULL;
ompi_request_t *rreq=MPI_REQUEST_NULL, *rreq1=MPI_REQUEST_NULL;
rank = ompi_comm_rank ( comm );
lcomm = hierarch_module->hier_lcomm;
if ( mca_coll_hierarch_verbose_param ) {
printf("%s:%d: executing hierarchical seg3 bcast with cnt=%d root=%d segsize=%d\n",
comm->c_name, rank, count, root, segsize );
}
/*
* This function returns the local leader communicator
* which *always* contains the root of this operation.
* This might involve creating a new communicator. This is
* also the reason, that *every* process in comm has to call
* this function
*/
llcomm = mca_coll_hierarch_get_llcomm ( root, hierarch_module, &llroot, &lroot);
ompi_datatype_type_size ( datatype, &typesize);
ompi_datatype_get_extent ( datatype, &ub, &typeext);
/* Determine number of segments and number of elements per segment */
if ((typesize > 0) && (segsize % typesize != 0)) {
/* segment size must be a multiple of typesize */
segsize = typesize * (segsize / typesize);
}
if ((segsize == 0) || (count == 0) || (typesize == 0)) {
segcount = count;
num_segments = 1;
} else {
segcount = segsize/typesize;
num_segments = count/segcount;
if ( (count % segcount) != 0 ) num_segments++;
if (num_segments == 1) segcount = count;
}
realsegsize = segcount*typeext;
remaining_count = segcount;
if ( MPI_COMM_NULL != lcomm ) {
lsize = ompi_comm_size ( lcomm );
lrank = ompi_comm_rank ( lcomm );
sreq1 = (ompi_request_t **)malloc ( lsize * sizeof(ompi_request_t *));
if ( NULL == sreq1 ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for(i=0; i<lsize; i++) {
sreq1[i] = MPI_REQUEST_NULL;
}
}
if ( MPI_COMM_NULL != llcomm ) {
llsize = ompi_comm_size (llcomm);
llrank = ompi_comm_rank ( llcomm );
sreq = hierarch_module->hier_reqs;
for(i=0; i<llsize; i++) {
sreq[i] = MPI_REQUEST_NULL;
}
}
/* Broadcasting the first segment in the upper level*/
if ( MPI_UNDEFINED != llroot ) {
ret = llcomm->c_coll.coll_bcast(tmpbuf, remaining_count, datatype,
llroot, llcomm,
llcomm->c_coll.coll_bcast_module);
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
}
for (segindex = 1; segindex < num_segments; segindex++) {
/* determine how many elements are being sent in this round */
if( segindex == (num_segments - 1) ) {
remaining_count = count - segindex*segcount;
}
tmpbuf += realsegsize;
/* Broadcasting the next segment in the upper level*/
if ( MPI_COMM_NULL != llcomm ) {
if(llrank == llroot) {
for(i = 0; i < llsize; i++) {
if( i != llroot) {
ret = MCA_PML_CALL(isend(tmpbuf, remaining_count, datatype, i,
MCA_COLL_BASE_TAG_BCAST,
MCA_PML_BASE_SEND_STANDARD,
llcomm, (sreq+i) ));
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
}
}
}
else {
ret = MCA_PML_CALL(irecv(tmpbuf, remaining_count, datatype, llroot,
MCA_COLL_BASE_TAG_BCAST,
llcomm, &rreq ));
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
}
}
/* broadcasting the before segment among the lower level processes
* once the local leaders got the data from the root, they can distribute
* it to the processes in their local, low-level communicator.
*/
if ( MPI_COMM_NULL != lcomm ) {
if( lrank == lroot) {
for( i = 0; i < lsize; i++) {
if( i != lroot) {
ret = MCA_PML_CALL(isend(tmpbuf-realsegsize, segcount, datatype, i,
MCA_COLL_BASE_TAG_BCAST,
MCA_PML_BASE_SEND_STANDARD,
lcomm, (sreq1+i) ));
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
}
}
}
else {
ret = MCA_PML_CALL(irecv(tmpbuf-realsegsize, segcount, datatype, lroot,
MCA_COLL_BASE_TAG_BCAST , lcomm, &rreq1 ));
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
}
}
/* Wait for the upper level bcast to complete*/
if ( MPI_COMM_NULL != llcomm ) {
if ( llrank == llroot ) {
ret = ompi_request_wait_all(llsize, sreq, MPI_STATUSES_IGNORE);
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
}
else {
ret = ompi_request_wait( &rreq, MPI_STATUS_IGNORE );
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
}
}
/*Wait for the lower level bcast to complete */
if ( MPI_COMM_NULL != lcomm ) {
if ( lrank == lroot ) {
ret = ompi_request_wait_all(lsize, sreq1, MPI_STATUSES_IGNORE);
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
}
else {
ret = ompi_request_wait( &rreq1, MPI_STATUS_IGNORE);
if ( OMPI_SUCCESS != ret ) {
goto exit;
}
}
}
}
/*Bcasting the last segment among the lower level processes
* once the local leaders got the data from the root, they can distribute
* it to the processes in their local, low-level communicator.
*/
if ( MPI_COMM_NULL != lcomm ) {
ret = lcomm->c_coll.coll_bcast(tmpbuf, remaining_count, datatype,
lroot, lcomm,
lcomm->c_coll.coll_bcast_module);
}
exit:
if ( NULL != sreq1 ) {
free ( sreq1 );
}
return ret;
}
/**
* This is a generic implementation of the reduce protocol. It used the tree
* provided as an argument and execute all operations using a segment of
* count times a datatype.
* For the last communication it will update the count in order to limit
* the number of datatype to the original count (original_count)
*
* Note that for non-commutative operations we cannot save memory copy
* for the first block: thus we must copy sendbuf to accumbuf on intermediate
* to keep the optimized loop happy.
*/
int ompi_coll_tuned_reduce_generic( void* sendbuf, void* recvbuf, int original_count,
ompi_datatype_t* datatype, ompi_op_t* op,
int root, ompi_communicator_t* comm,
mca_coll_base_module_t *module,
ompi_coll_tree_t* tree, int count_by_segment,
int max_outstanding_reqs )
{
char *inbuf[2] = {NULL, NULL}, *inbuf_free[2] = {NULL, NULL};
char *accumbuf = NULL, *accumbuf_free = NULL;
char *local_op_buffer = NULL, *sendtmpbuf = NULL;
ptrdiff_t extent, lower_bound, segment_increment;
size_t typelng;
ompi_request_t* reqs[2] = {MPI_REQUEST_NULL, MPI_REQUEST_NULL};
int num_segments, line, ret, segindex, i, rank;
int recvcount, prevcount, inbi;
/**
* Determine number of segments and number of elements
* sent per operation
*/
ompi_datatype_get_extent( datatype, &lower_bound, &extent );
ompi_datatype_type_size( datatype, &typelng );
num_segments = (original_count + count_by_segment - 1) / count_by_segment;
segment_increment = (ptrdiff_t)count_by_segment * extent;
sendtmpbuf = (char*) sendbuf;
if( sendbuf == MPI_IN_PLACE ) {
sendtmpbuf = (char *)recvbuf;
}
OPAL_OUTPUT((ompi_coll_tuned_stream, "coll:tuned:reduce_generic count %d, msg size %ld, segsize %ld, max_requests %d",
original_count, (unsigned long)((ptrdiff_t)num_segments * (ptrdiff_t)segment_increment),
(unsigned long)segment_increment, max_outstanding_reqs));
rank = ompi_comm_rank(comm);
/*printf("Tree of rank %d - ", rank);
printf("Parent : %d - ", tree->tree_prev);
printf("Child : ");
for (i = 0; i < tree->tree_nextsize; i++)
printf("%d ", tree->tree_next[i]);
printf("\n");*/
/* non-leaf nodes - wait for children to send me data & forward up
(if needed) */
if( tree->tree_nextsize > 0 ) {
ptrdiff_t true_lower_bound, true_extent, real_segment_size;
ompi_datatype_get_true_extent( datatype, &true_lower_bound,
&true_extent );
/* handle non existant recv buffer (i.e. its NULL) and
protect the recv buffer on non-root nodes */
accumbuf = (char*)recvbuf;
if( (NULL == accumbuf) || (root != rank) ) {
/* Allocate temporary accumulator buffer. */
accumbuf_free = (char*)malloc(true_extent +
(ptrdiff_t)(original_count - 1) * extent);
if (accumbuf_free == NULL) {
line = __LINE__; ret = -1; goto error_hndl;
}
accumbuf = accumbuf_free - lower_bound;
}
/* If this is a non-commutative operation we must copy
sendbuf to the accumbuf, in order to simplfy the loops */
if (!ompi_op_is_commute(op)) {
ompi_datatype_copy_content_same_ddt(datatype, original_count,
(char*)accumbuf,
(char*)sendtmpbuf);
}
/* Allocate two buffers for incoming segments */
real_segment_size = true_extent + (ptrdiff_t)(count_by_segment - 1) * extent;
inbuf_free[0] = (char*) malloc(real_segment_size);
if( inbuf_free[0] == NULL ) {
line = __LINE__; ret = -1; goto error_hndl;
}
inbuf[0] = inbuf_free[0] - lower_bound;
/* if there is chance to overlap communication -
allocate second buffer */
if( (num_segments > 1) || (tree->tree_nextsize > 1) ) {
inbuf_free[1] = (char*) malloc(real_segment_size);
if( inbuf_free[1] == NULL ) {
line = __LINE__; ret = -1; goto error_hndl;
}
inbuf[1] = inbuf_free[1] - lower_bound;
}
/* reset input buffer index and receive count */
inbi = 0;
recvcount = 0;
/* for each segment */
for( segindex = 0; segindex <= num_segments; segindex++ ) {
prevcount = recvcount;
/* recvcount - number of elements in current segment */
recvcount = count_by_segment;
if( segindex == (num_segments-1) )
recvcount = original_count - (ptrdiff_t)count_by_segment * (ptrdiff_t)segindex;
/* for each child */
for( i = 0; i < tree->tree_nextsize; i++ ) {
/**
* We try to overlap communication:
* either with next segment or with the next child
*/
/* post irecv for current segindex on current child */
if( segindex < num_segments ) {
void* local_recvbuf = inbuf[inbi];
if( 0 == i ) {
/* for the first step (1st child per segment) and
* commutative operations we might be able to irecv
* directly into the accumulate buffer so that we can
* reduce(op) this with our sendbuf in one step as
* ompi_op_reduce only has two buffer pointers,
* this avoids an extra memory copy.
*
* BUT if the operation is non-commutative or
* we are root and are USING MPI_IN_PLACE this is wrong!
*/
if( (ompi_op_is_commute(op)) &&
!((MPI_IN_PLACE == sendbuf) && (rank == tree->tree_root)) ) {
local_recvbuf = accumbuf + (ptrdiff_t)segindex * (ptrdiff_t)segment_increment;
}
}
ret = MCA_PML_CALL(irecv(local_recvbuf, recvcount, datatype,
tree->tree_next[i],
MCA_COLL_BASE_TAG_REDUCE, comm,
&reqs[inbi]));
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl;}
}
/* wait for previous req to complete, if any.
if there are no requests reqs[inbi ^1] will be
MPI_REQUEST_NULL. */
/* wait on data from last child for previous segment */
ret = ompi_request_wait_all( 1, &reqs[inbi ^ 1],
MPI_STATUSES_IGNORE );
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
local_op_buffer = inbuf[inbi ^ 1];
if( i > 0 ) {
/* our first operation is to combine our own [sendbuf] data
* with the data we recvd from down stream (but only
* the operation is commutative and if we are not root and
* not using MPI_IN_PLACE)
*/
if( 1 == i ) {
if( (ompi_op_is_commute(op)) &&
!((MPI_IN_PLACE == sendbuf) && (rank == tree->tree_root)) ) {
local_op_buffer = sendtmpbuf + (ptrdiff_t)segindex * (ptrdiff_t)segment_increment;
}
}
/* apply operation */
ompi_op_reduce(op, local_op_buffer,
accumbuf + (ptrdiff_t)segindex * (ptrdiff_t)segment_increment,
recvcount, datatype );
} else if ( segindex > 0 ) {
void* accumulator = accumbuf + (ptrdiff_t)(segindex-1) * (ptrdiff_t)segment_increment;
if( tree->tree_nextsize <= 1 ) {
if( (ompi_op_is_commute(op)) &&
!((MPI_IN_PLACE == sendbuf) && (rank == tree->tree_root)) ) {
local_op_buffer = sendtmpbuf + (ptrdiff_t)(segindex-1) * (ptrdiff_t)segment_increment;
}
}
ompi_op_reduce(op, local_op_buffer, accumulator, prevcount,
datatype );
/* all reduced on available data this step (i) complete,
* pass to the next process unless you are the root.
*/
if (rank != tree->tree_root) {
/* send combined/accumulated data to parent */
ret = MCA_PML_CALL( send( accumulator, prevcount,
datatype, tree->tree_prev,
MCA_COLL_BASE_TAG_REDUCE,
MCA_PML_BASE_SEND_STANDARD,
comm) );
if (ret != MPI_SUCCESS) {
line = __LINE__; goto error_hndl;
}
}
/* we stop when segindex = number of segments
(i.e. we do num_segment+1 steps for pipelining */
if (segindex == num_segments) break;
}
/* update input buffer index */
inbi = inbi ^ 1;
} /* end of for each child */
} /* end of for each segment */
/* clean up */
if( inbuf_free[0] != NULL) free(inbuf_free[0]);
if( inbuf_free[1] != NULL) free(inbuf_free[1]);
if( accumbuf_free != NULL ) free(accumbuf_free);
}
/* leaf nodes
Depending on the value of max_outstanding_reqs and
the number of segments we have two options:
- send all segments using blocking send to the parent, or
- avoid overflooding the parent nodes by limiting the number of
outstanding requests to max_oustanding_reqs.
TODO/POSSIBLE IMPROVEMENT: If there is a way to determine the eager size
for the current communication, synchronization should be used only
when the message/segment size is smaller than the eager size.
*/
else {
/* If the number of segments is less than a maximum number of oustanding
requests or there is no limit on the maximum number of outstanding
requests, we send data to the parent using blocking send */
if ((0 == max_outstanding_reqs) ||
(num_segments <= max_outstanding_reqs)) {
segindex = 0;
while ( original_count > 0) {
if (original_count < count_by_segment) {
count_by_segment = original_count;
}
ret = MCA_PML_CALL( send((char*)sendbuf +
(ptrdiff_t)segindex * (ptrdiff_t)segment_increment,
count_by_segment, datatype,
tree->tree_prev,
MCA_COLL_BASE_TAG_REDUCE,
MCA_PML_BASE_SEND_STANDARD,
comm) );
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
segindex++;
original_count -= count_by_segment;
}
}
/* Otherwise, introduce flow control:
- post max_outstanding_reqs non-blocking synchronous send,
- for remaining segments
- wait for a ssend to complete, and post the next one.
- wait for all outstanding sends to complete.
*/
else {
int creq = 0;
ompi_request_t **sreq = NULL;
sreq = (ompi_request_t**) calloc( max_outstanding_reqs,
sizeof(ompi_request_t*) );
if (NULL == sreq) { line = __LINE__; ret = -1; goto error_hndl; }
/* post first group of requests */
for (segindex = 0; segindex < max_outstanding_reqs; segindex++) {
ret = MCA_PML_CALL( isend((char*)sendbuf +
(ptrdiff_t)segindex * (ptrdiff_t)segment_increment,
count_by_segment, datatype,
tree->tree_prev,
MCA_COLL_BASE_TAG_REDUCE,
MCA_PML_BASE_SEND_SYNCHRONOUS, comm,
&sreq[segindex]) );
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
original_count -= count_by_segment;
}
creq = 0;
while ( original_count > 0 ) {
/* wait on a posted request to complete */
ret = ompi_request_wait(&sreq[creq], MPI_STATUS_IGNORE);
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
sreq[creq] = MPI_REQUEST_NULL;
if( original_count < count_by_segment ) {
count_by_segment = original_count;
}
ret = MCA_PML_CALL( isend((char*)sendbuf +
(ptrdiff_t)segindex * (ptrdiff_t)segment_increment,
count_by_segment, datatype,
tree->tree_prev,
MCA_COLL_BASE_TAG_REDUCE,
MCA_PML_BASE_SEND_SYNCHRONOUS, comm,
&sreq[creq]) );
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
creq = (creq + 1) % max_outstanding_reqs;
segindex++;
original_count -= count_by_segment;
}
/* Wait on the remaining request to complete */
ret = ompi_request_wait_all( max_outstanding_reqs, sreq,
MPI_STATUSES_IGNORE );
if (ret != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
/* free requests */
free(sreq);
}
}
return OMPI_SUCCESS;
error_hndl: /* error handler */
OPAL_OUTPUT (( ompi_coll_tuned_stream,
"ERROR_HNDL: node %d file %s line %d error %d\n",
rank, __FILE__, line, ret ));
if( inbuf_free[0] != NULL ) free(inbuf_free[0]);
if( inbuf_free[1] != NULL ) free(inbuf_free[1]);
if( accumbuf_free != NULL ) free(accumbuf);
return ret;
}
/* Arguments not used in this implementation:
* - bridgecomm
* - local_leader
* - remote_leader
* - send_first
*/
static int ompi_comm_allreduce_inter ( int *inbuf, int *outbuf,
int count, struct ompi_op_t *op,
ompi_communicator_t *intercomm,
ompi_communicator_t *bridgecomm,
void* local_leader,
void* remote_leader,
int send_first )
{
int local_rank, rsize;
int i, rc;
int *sbuf;
int *tmpbuf=NULL;
int *rcounts=NULL, scount=0;
int *rdisps=NULL;
if ( &ompi_mpi_op_sum.op != op && &ompi_mpi_op_prod.op != op &&
&ompi_mpi_op_max.op != op && &ompi_mpi_op_min.op != op ) {
return MPI_ERR_OP;
}
if ( !OMPI_COMM_IS_INTER (intercomm)) {
return MPI_ERR_COMM;
}
/* Allocate temporary arrays */
rsize = ompi_comm_remote_size (intercomm);
local_rank = ompi_comm_rank ( intercomm );
tmpbuf = (int *) malloc ( count * sizeof(int));
rdisps = (int *) calloc ( rsize, sizeof(int));
rcounts = (int *) calloc ( rsize, sizeof(int) );
if ( OPAL_UNLIKELY (NULL == tmpbuf || NULL == rdisps || NULL == rcounts)) {
rc = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
/* Execute the inter-allreduce: the result of our group will
be in the buffer of the remote group */
rc = intercomm->c_coll.coll_allreduce ( inbuf, tmpbuf, count, MPI_INT,
op, intercomm,
intercomm->c_coll.coll_allreduce_module);
if ( OMPI_SUCCESS != rc ) {
goto exit;
}
if ( 0 == local_rank ) {
MPI_Request req;
/* for the allgatherv later */
scount = count;
/* local leader exchange their data and determine the overall result
for both groups */
rc = MCA_PML_CALL(irecv (outbuf, count, MPI_INT, 0,
OMPI_COMM_ALLREDUCE_TAG,
intercomm, &req));
if ( OMPI_SUCCESS != rc ) {
goto exit;
}
rc = MCA_PML_CALL(send (tmpbuf, count, MPI_INT, 0,
OMPI_COMM_ALLREDUCE_TAG,
MCA_PML_BASE_SEND_STANDARD,
intercomm));
if ( OMPI_SUCCESS != rc ) {
goto exit;
}
rc = ompi_request_wait ( &req, MPI_STATUS_IGNORE );
if ( OMPI_SUCCESS != rc ) {
goto exit;
}
if ( &ompi_mpi_op_max.op == op ) {
for ( i = 0 ; i < count; i++ ) {
if (tmpbuf[i] > outbuf[i]) outbuf[i] = tmpbuf[i];
}
}
else if ( &ompi_mpi_op_min.op == op ) {
for ( i = 0 ; i < count; i++ ) {
if (tmpbuf[i] < outbuf[i]) outbuf[i] = tmpbuf[i];
}
}
else if ( &ompi_mpi_op_sum.op == op ) {
for ( i = 0 ; i < count; i++ ) {
outbuf[i] += tmpbuf[i];
}
}
else if ( &ompi_mpi_op_prod.op == op ) {
for ( i = 0 ; i < count; i++ ) {
outbuf[i] *= tmpbuf[i];
}
}
}
/* distribute the overall result to all processes in the other group.
Instead of using bcast, we are using here allgatherv, to avoid the
possible deadlock. Else, we need an algorithm to determine,
which group sends first in the inter-bcast and which receives
the result first.
*/
rcounts[0] = count;
sbuf = outbuf;
rc = intercomm->c_coll.coll_allgatherv (sbuf, scount, MPI_INT, outbuf,
rcounts, rdisps, MPI_INT,
intercomm,
intercomm->c_coll.coll_allgatherv_module);
exit:
if ( NULL != tmpbuf ) {
free ( tmpbuf );
}
if ( NULL != rcounts ) {
free ( rcounts );
}
if ( NULL != rdisps ) {
free ( rdisps );
}
return (rc);
}
int
mca_fcoll_static_file_write_all (mca_io_ompio_file_t *fh,
void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
size_t max_data = 0, bytes_per_cycle=0;
struct iovec *iov=NULL, *decoded_iov=NULL;
uint32_t iov_count=0, iov_index=0;
int i=0,j=0,l=0, temp_index;
int ret=OMPI_SUCCESS, cycles, local_cycles, *bytes_per_process=NULL;
int index, *disp_index=NULL, **blocklen_per_process=NULL;
int *iovec_count_per_process=NULL, *displs=NULL;
size_t total_bytes_written=0;
MPI_Aint **displs_per_process=NULL, *memory_displacements=NULL;
MPI_Aint bytes_to_write_in_cycle=0, global_iov_count=0, global_count=0;
local_io_array *local_iov_array =NULL, *global_iov_array=NULL;
local_io_array *file_offsets_for_agg=NULL;
int *sorted=NULL, *sorted_file_offsets=NULL, temp_pindex, *temp_disp_index=NULL;
char *send_buf=NULL, *global_buf=NULL;
int iov_size=0, current_position=0, *current_index=NULL;
int *bytes_remaining=NULL, entries_per_aggregator=0;
ompi_datatype_t **recvtype = NULL;
MPI_Request *send_req=NULL, *recv_req=NULL;
/* For creating datatype of type io_array */
int blocklen[3] = {1, 1, 1};
int static_num_io_procs=1;
OPAL_PTRDIFF_TYPE d[3], base;
ompi_datatype_t *types[3];
ompi_datatype_t *io_array_type=MPI_DATATYPE_NULL;
/*----------------------------------------------*/
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
double write_time = 0.0, start_write_time = 0.0, end_write_time = 0.0;
double comm_time = 0.0, start_comm_time = 0.0, end_comm_time = 0.0;
double exch_write = 0.0, start_exch = 0.0, end_exch = 0.0;
mca_io_ompio_print_entry nentry;
#endif
#if DEBUG_ON
MPI_Aint gc_in;
#endif
// if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
// fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
// }
/* In case the data is not contigous in memory, decode it into an iovec */
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
fh->f_decode_datatype ((struct mca_io_ompio_file_t *)fh,
datatype,
count,
buf,
&max_data,
&decoded_iov,
&iov_count);
}
else {
max_data = count * datatype->super.size;
}
if ( MPI_STATUS_IGNORE != status ) {
status->_ucount = max_data;
}
fh->f_get_num_aggregators ( & static_num_io_procs );
fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *)fh,
static_num_io_procs,
max_data);
/* io_array datatype for using in communication*/
types[0] = &ompi_mpi_long.dt;
types[1] = &ompi_mpi_long.dt;
types[2] = &ompi_mpi_int.dt;
d[0] = (OPAL_PTRDIFF_TYPE)&local_iov_array[0];
d[1] = (OPAL_PTRDIFF_TYPE)&local_iov_array[0].length;
d[2] = (OPAL_PTRDIFF_TYPE)&local_iov_array[0].process_id;
base = d[0];
for (i=0 ; i<3 ; i++) {
d[i] -= base;
}
ompi_datatype_create_struct (3,
blocklen,
d,
types,
&io_array_type);
ompi_datatype_commit (&io_array_type);
/* #########################################################*/
ret = fh->f_generate_current_file_view((struct mca_io_ompio_file_t *)fh,
max_data,
&iov,
&iov_size);
if (ret != OMPI_SUCCESS){
fprintf(stderr,"Current File View Generation Error\n");
goto exit;
}
if (0 == iov_size){
iov_size = 1;
}
local_iov_array = (local_io_array *)malloc (iov_size * sizeof(local_io_array));
if ( NULL == local_iov_array){
fprintf(stderr,"local_iov_array allocation error\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for (j=0; j < iov_size; j++){
local_iov_array[j].offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)
iov[j].iov_base;
local_iov_array[j].length = (size_t)iov[j].iov_len;
local_iov_array[j].process_id = fh->f_rank;
}
fh->f_get_bytes_per_agg ( (int *) &bytes_per_cycle);
local_cycles = ceil((double)max_data/bytes_per_cycle);
ret = fh->f_comm->c_coll.coll_allreduce (&local_cycles,
&cycles,
1,
MPI_INT,
MPI_MAX,
fh->f_comm,
fh->f_comm->c_coll.coll_allreduce_module);
if (OMPI_SUCCESS != ret){
fprintf(stderr,"local cycles allreduce!\n");
goto exit;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
disp_index = (int *)malloc (fh->f_procs_per_group * sizeof (int));
if (NULL == disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
bytes_per_process = (int *) malloc (fh->f_procs_per_group * sizeof(int ));
if (NULL == bytes_per_process){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
bytes_remaining = (int *) malloc (fh->f_procs_per_group * sizeof(int));
if (NULL == bytes_remaining){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
current_index = (int *) malloc (fh->f_procs_per_group * sizeof(int));
if (NULL == current_index){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
blocklen_per_process = (int **)malloc (fh->f_procs_per_group * sizeof (int*));
if (NULL == blocklen_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process = (MPI_Aint **)
malloc (fh->f_procs_per_group * sizeof (MPI_Aint*));
if (NULL == displs_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for(i=0;i<fh->f_procs_per_group;i++){
current_index[i] = 0;
bytes_remaining[i] =0;
blocklen_per_process[i] = NULL;
displs_per_process[i] = NULL;
}
}
iovec_count_per_process = (int *) malloc (fh->f_procs_per_group * sizeof(int));
if (NULL == iovec_count_per_process){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs = (int *) malloc (fh->f_procs_per_group * sizeof(int));
if (NULL == displs){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ret = fh->f_allgather_array (&iov_size,
1,
MPI_INT,
iovec_count_per_process,
1,
MPI_INT,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if( OMPI_SUCCESS != ret){
fprintf(stderr,"iov size allgatherv array!\n");
goto exit;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
displs[0] = 0;
global_iov_count = iovec_count_per_process[0];
for (i=1 ; i<fh->f_procs_per_group ; i++) {
global_iov_count += iovec_count_per_process[i];
displs[i] = displs[i-1] + iovec_count_per_process[i-1];
}
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
global_iov_array = (local_io_array *) malloc (global_iov_count *
sizeof(local_io_array));
if (NULL == global_iov_array){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
ret = fh->f_gatherv_array (local_iov_array,
iov_size,
io_array_type,
global_iov_array,
iovec_count_per_process,
displs,
io_array_type,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
fprintf(stderr,"global_iov_array gather error!\n");
goto exit;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
if ( 0 == global_iov_count){
global_iov_count = 1;
}
sorted = (int *)malloc (global_iov_count * sizeof(int));
if (NULL == sorted) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
local_heap_sort (global_iov_array, global_iov_count, sorted);
}
#if DEBUG_ON
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
for (gc_in=0; gc_in<global_iov_count; gc_in++){
printf("%d: Offset[%ld]: %lld, Length[%ld]: %ld\n",
global_iov_array[gc_in].process_id,
gc_in, global_iov_array[gc_in].offset,
gc_in, global_iov_array[gc_in].length);
}
}
#endif
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_exch = MPI_Wtime();
#endif
for (index = 0; index < cycles; index++){
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
if (NULL == recvtype){
recvtype = (ompi_datatype_t **)
malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
if (NULL == recvtype) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
for(l=0;l<fh->f_procs_per_group;l++){
disp_index[l] = 1;
if (NULL != blocklen_per_process[l]){
free(blocklen_per_process[l]);
blocklen_per_process[l] = NULL;
}
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
if (NULL == blocklen_per_process[l]) {
opal_output (1, "OUT OF MEMORY for blocklen\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
if (NULL == displs_per_process[l]){
opal_output (1, "OUT OF MEMORY for displs\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if(NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements = NULL;
}
}
if (local_cycles > index) {
if ((index == local_cycles-1) && (max_data % bytes_per_cycle)) {
bytes_to_write_in_cycle = max_data % bytes_per_cycle;
}
else if (max_data <= bytes_per_cycle) {
bytes_to_write_in_cycle = max_data;
}
else {
bytes_to_write_in_cycle = bytes_per_cycle;
}
}
else {
bytes_to_write_in_cycle = 0;
}
#if DEBUG_ON
/* if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {*/
printf ("***%d: CYCLE %d Bytes %ld**********\n",
fh->f_rank,
index,
bytes_to_write_in_cycle);
/* }*/
#endif
/**********************************************************
**Gather the Data from all the processes at the writers **
*********************************************************/
/* gather from each process how many bytes each will be sending */
fh->f_gather_array (&bytes_to_write_in_cycle,
1,
MPI_INT,
bytes_per_process,
1,
MPI_INT,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
/*
For each aggregator
it needs to get bytes_to_write_in_cycle from each process
in group which adds up to bytes_per_cycle
*/
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
for (i=0;i<fh->f_procs_per_group; i++){
/* printf("bytes_per_process[%d]: %d\n", i, bytes_per_process[i]);
*/
#if DEBUG_ON
printf ("%d : bytes_per_process : %d\n",
fh->f_procs_in_group[i],
bytes_per_process[i]);
#endif
while (bytes_per_process[i] > 0){
if (get_process_id(global_iov_array[sorted[current_index[i]]].process_id,
fh) == i){ /* current id owns this entry!*/
/*Add and subtract length and create
blocklength and displs array*/
if (bytes_remaining[i]){ /*Remaining bytes in the current entry of
the global offset array*/
if (bytes_remaining[i] <= bytes_per_process[i]){
blocklen_per_process[i][disp_index[i] - 1] = bytes_remaining[i];
displs_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].offset +
(global_iov_array[sorted[current_index[i]]].length
- bytes_remaining[i]);
blocklen_per_process[i] = (int *) realloc
((void *)blocklen_per_process[i], (disp_index[i]+1)*sizeof(int));
displs_per_process[i] = (MPI_Aint *)realloc
((void *)displs_per_process[i], (disp_index[i]+1)*sizeof(MPI_Aint));
bytes_per_process[i] -= bytes_remaining[i];
blocklen_per_process[i][disp_index[i]] = 0;
displs_per_process[i][disp_index[i]] = 0;
bytes_remaining[i] = 0;
disp_index[i] += 1;
/* This entry has been used up, we need to move to the
next entry of this process and make current_index point there*/
current_index[i] = find_next_index(i,
current_index[i],
fh,
global_iov_array,
global_iov_count,
sorted);
if (current_index[i] == -1){
/* No more entries left, so Its all done! exit!*/
break;
}
continue;
}
else{
blocklen_per_process[i][disp_index[i] - 1] = bytes_per_process[i];
displs_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].offset +
(global_iov_array[sorted[current_index[i]]].length
- bytes_remaining[i]);
bytes_remaining[i] -= bytes_per_process[i];
bytes_per_process[i] = 0;
break;
}
}
else{
if (bytes_per_process[i] <
global_iov_array[sorted[current_index[i]]].length){
blocklen_per_process[i][disp_index[i] - 1] =
bytes_per_process[i];
displs_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].offset;
bytes_remaining[i] =
global_iov_array[sorted[current_index[i]]].length -
bytes_per_process[i];
bytes_per_process[i] = 0;
break;
}
else {
blocklen_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].length;
displs_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].offset;
blocklen_per_process[i] =
(int *) realloc ((void *)blocklen_per_process[i], (disp_index[i]+1)*sizeof(int));
displs_per_process[i] = (MPI_Aint *)realloc
((void *)displs_per_process[i], (disp_index[i]+1)*sizeof(MPI_Aint));
blocklen_per_process[i][disp_index[i]] = 0;
displs_per_process[i][disp_index[i]] = 0;
disp_index[i] += 1;
bytes_per_process[i] -=
global_iov_array[sorted[current_index[i]]].length;
current_index[i] = find_next_index(i,
current_index[i],
fh,
global_iov_array,
global_iov_count,
sorted);
if (current_index[i] == -1){
break;
}
}
}
}
else{
current_index[i] = find_next_index(i,
current_index[i],
fh,
global_iov_array,
global_iov_count,
sorted);
if (current_index[i] == -1){
bytes_per_process[i] = 0; /* no more entries left
to service this request*/
continue;
}
}
}
}
entries_per_aggregator=0;
for (i=0;i<fh->f_procs_per_group;i++){
for (j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0){
entries_per_aggregator++;
#if DEBUG_ON
printf("%d sends blocklen[%d]: %d, disp[%d]: %ld to %d\n",
fh->f_procs_in_group[i],j,
blocklen_per_process[i][j],j,
displs_per_process[i][j],
fh->f_rank);
#endif
}
}
}
if (entries_per_aggregator > 0){
file_offsets_for_agg = (local_io_array *)
malloc(entries_per_aggregator*sizeof(local_io_array));
if (NULL == file_offsets_for_agg) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
sorted_file_offsets = (int *)
malloc (entries_per_aggregator*sizeof(int));
if (NULL == sorted_file_offsets){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
temp_index = 0;
for (i=0;i<fh->f_procs_per_group; i++){
for(j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0){
file_offsets_for_agg[temp_index].length =
blocklen_per_process[i][j];
file_offsets_for_agg[temp_index].process_id = i;
file_offsets_for_agg[temp_index].offset =
displs_per_process[i][j];
temp_index++;
}
}
}
}
else{
continue;
}
local_heap_sort (file_offsets_for_agg,
entries_per_aggregator,
sorted_file_offsets);
memory_displacements = (MPI_Aint *) malloc
(entries_per_aggregator * sizeof(MPI_Aint));
memory_displacements[sorted_file_offsets[0]] = 0;
for (i=1; i<entries_per_aggregator; i++){
memory_displacements[sorted_file_offsets[i]] =
memory_displacements[sorted_file_offsets[i-1]] +
file_offsets_for_agg[sorted_file_offsets[i-1]].length;
}
temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
if (NULL == temp_disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
global_count = 0;
for (i=0;i<entries_per_aggregator;i++){
temp_pindex =
file_offsets_for_agg[sorted_file_offsets[i]].process_id;
displs_per_process[temp_pindex][temp_disp_index[temp_pindex]] =
memory_displacements[sorted_file_offsets[i]];
if (temp_disp_index[temp_pindex] < disp_index[temp_pindex])
temp_disp_index[temp_pindex] += 1;
else{
printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
temp_pindex, temp_disp_index[temp_pindex],
temp_pindex, disp_index[temp_pindex]);
}
global_count +=
file_offsets_for_agg[sorted_file_offsets[i]].length;
}
if (NULL != temp_disp_index){
free(temp_disp_index);
temp_disp_index = NULL;
}
#if DEBUG_ON
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
for (i=0; i<entries_per_aggregator;i++){
printf("%d: OFFSET: %lld LENGTH: %ld, Mem-offset: %ld, disp : %d\n",
file_offsets_for_agg[sorted_file_offsets[i]].process_id,
file_offsets_for_agg[sorted_file_offsets[i]].offset,
file_offsets_for_agg[sorted_file_offsets[i]].length,
memory_displacements[sorted_file_offsets[i]],
disp_index[ file_offsets_for_agg[sorted_file_offsets[i]].process_id]);
}
#endif
#if DEBUG_ON
printf("%d: global_count : %ld, bytes_to_write_in_cycle : %ld, procs_per_group: %d\n",
fh->f_rank,
global_count,
bytes_to_write_in_cycle,
fh->f_procs_per_group);
#endif
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_comm_time = MPI_Wtime();
#endif
global_buf = (char *) malloc (global_count);
if (NULL == global_buf){
opal_output(1, "OUT OF MEMORY");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
recv_req = (MPI_Request *)
malloc (fh->f_procs_per_group * sizeof(MPI_Request));
if (NULL == recv_req){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for (i=0;i<fh->f_procs_per_group; i++){
ompi_datatype_create_hindexed(disp_index[i],
blocklen_per_process[i],
displs_per_process[i],
MPI_BYTE,
&recvtype[i]);
ompi_datatype_commit(&recvtype[i]);
ret = MCA_PML_CALL(irecv(global_buf,
1,
recvtype[i],
fh->f_procs_in_group[i],
123,
fh->f_comm,
&recv_req[i]));
if (OMPI_SUCCESS != ret){
fprintf(stderr,"irecv Error!\n");
goto exit;
}
}
}
if (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY) {
send_buf = &((char*)buf)[total_bytes_written];
}
else if (bytes_to_write_in_cycle) {
/* allocate a send buffer and copy the data that needs
to be sent into it in case the data is non-contigous
in memory */
OPAL_PTRDIFF_TYPE mem_address;
size_t remaining = 0;
size_t temp_position = 0;
send_buf = malloc (bytes_to_write_in_cycle);
if (NULL == send_buf) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
remaining = bytes_to_write_in_cycle;
while (remaining) {
mem_address = (OPAL_PTRDIFF_TYPE)
(decoded_iov[iov_index].iov_base) + current_position;
if (remaining >=
(decoded_iov[iov_index].iov_len - current_position)) {
memcpy (send_buf+temp_position,
(IOVBASE_TYPE *)mem_address,
decoded_iov[iov_index].iov_len - current_position);
remaining = remaining -
(decoded_iov[iov_index].iov_len - current_position);
temp_position = temp_position +
(decoded_iov[iov_index].iov_len - current_position);
iov_index = iov_index + 1;
current_position = 0;
}
else {
memcpy (send_buf+temp_position,
(IOVBASE_TYPE *)mem_address,
remaining);
current_position = current_position + remaining;
remaining = 0;
}
}
}
total_bytes_written += bytes_to_write_in_cycle;
send_req = (MPI_Request *) malloc (sizeof(MPI_Request));
if (NULL == send_req){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ret = MCA_PML_CALL(isend(send_buf,
bytes_to_write_in_cycle,
MPI_BYTE,
fh->f_procs_in_group[fh->f_aggregator_index],
123,
MCA_PML_BASE_SEND_STANDARD,
fh->f_comm,
send_req));
if ( OMPI_SUCCESS != ret ){
fprintf(stderr,"isend error!\n");
goto exit;
}
ret = ompi_request_wait (send_req, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
ret = ompi_request_wait_all (fh->f_procs_per_group,
recv_req,
MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
#if DEBUG_ON
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank){
for (i=0 ; i<global_count/4 ; i++)
printf (" RECV %d \n",((int *)global_buf)[i]);
}
#endif
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_comm_time = MPI_Wtime();
comm_time += end_comm_time - start_comm_time;
#endif
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
(entries_per_aggregator * sizeof (mca_io_ompio_io_array_t));
if (NULL == fh->f_io_array) {
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
fh->f_num_of_io_entries = 0;
/*First entry for every aggregator*/
fh->f_io_array[fh->f_num_of_io_entries].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
fh->f_io_array[fh->f_num_of_io_entries].length =
file_offsets_for_agg[sorted_file_offsets[0]].length;
fh->f_io_array[fh->f_num_of_io_entries].memory_address =
global_buf+memory_displacements[sorted_file_offsets[0]];
fh->f_num_of_io_entries++;
for (i=1;i<entries_per_aggregator;i++){
if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
file_offsets_for_agg[sorted_file_offsets[i]].offset){
fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
file_offsets_for_agg[sorted_file_offsets[i]].length;
}
else {
fh->f_io_array[fh->f_num_of_io_entries].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
fh->f_io_array[fh->f_num_of_io_entries].length =
file_offsets_for_agg[sorted_file_offsets[i]].length;
fh->f_io_array[fh->f_num_of_io_entries].memory_address =
global_buf+memory_displacements[sorted_file_offsets[i]];
fh->f_num_of_io_entries++;
}
}
#if DEBUG_ON
printf("*************************** %d\n", fh->f_num_of_io_entries);
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
printf(" ADDRESS: %p OFFSET: %ld LENGTH: %ld\n",
fh->f_io_array[i].memory_address,
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].offset,
fh->f_io_array[i].length);
}
#endif
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_write_time = MPI_Wtime();
#endif
if (fh->f_num_of_io_entries) {
if ( 0 > fh->f_fbtl->fbtl_pwritev (fh)) {
opal_output (1, "WRITE FAILED\n");
ret = OMPI_ERROR;
goto exit;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_write_time = MPI_Wtime();
write_time += end_write_time - start_write_time;
#endif
}
if (NULL != send_req){
free(send_req);
send_req = NULL;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
fh->f_num_of_io_entries = 0;
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
for (i = 0; i < fh->f_procs_per_group; i++)
ompi_datatype_destroy(recvtype+i);
if (NULL != recvtype){
free(recvtype);
recvtype=NULL;
}
if (NULL != recv_req){
free(recv_req);
recv_req = NULL;
}
if (NULL != global_buf) {
free (global_buf);
global_buf = NULL;
}
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_exch = MPI_Wtime();
exch_write += end_exch - start_exch;
nentry.time[0] = write_time;
nentry.time[1] = comm_time;
nentry.time[2] = exch_write;
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank)
nentry.aggregator = 1;
else
nentry.aggregator = 0;
nentry.nprocs_for_coll = static_num_io_procs;
if (!fh->f_full_print_queue(WRITE_PRINT_QUEUE)){
fh->f_register_print_entry(WRITE_PRINT_QUEUE,
nentry);
}
#endif
exit:
if (NULL != decoded_iov){
free(decoded_iov);
decoded_iov = NULL;
}
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
if (NULL != local_iov_array){
free(local_iov_array);
local_iov_array = NULL;
}
for(l=0;l<fh->f_procs_per_group;l++){
if (NULL != blocklen_per_process[l]){
free(blocklen_per_process[l]);
blocklen_per_process[l] = NULL;
}
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
}
}
if (NULL != send_buf){
free(send_buf);
send_buf = NULL;
}
if (NULL != global_buf){
free(global_buf);
global_buf = NULL;
}
if (NULL != recvtype){
free(recvtype);
recvtype = NULL;
}
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if (NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements = NULL;
}
if (NULL != displs_per_process){
free(displs_per_process);
displs_per_process = NULL;
}
if (NULL != blocklen_per_process){
free(blocklen_per_process);
blocklen_per_process = NULL;
}
if(NULL != current_index){
free(current_index);
current_index = NULL;
}
if(NULL != bytes_remaining){
free(bytes_remaining);
bytes_remaining = NULL;
}
if (NULL != disp_index){
free(disp_index);
disp_index = NULL;
}
if (NULL != sorted) {
free(sorted);
sorted = NULL;
}
return ret;
}
/* MPI_IN_PLACE all to all algorithm. TODO: implement a better one. */
int
mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, int rcount,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
int i, j, size, rank, err = MPI_SUCCESS, line;
OPAL_PTRDIFF_TYPE ext, gap;
ompi_request_t *req;
char *allocated_buffer = NULL, *tmp_buffer;
size_t max_size;
/* Initialize. */
size = ompi_comm_size(comm);
rank = ompi_comm_rank(comm);
/* If only one process, we're done. */
if (1 == size) {
return MPI_SUCCESS;
}
/* Find the largest receive amount */
ompi_datatype_type_extent (rdtype, &ext);
max_size = opal_datatype_span(&rdtype->super, rcount, &gap);
/* Initiate all send/recv to/from others. */
/* Allocate a temporary buffer */
allocated_buffer = calloc (max_size, 1);
if( NULL == allocated_buffer) { err = OMPI_ERR_OUT_OF_RESOURCE; line = __LINE__; goto error_hndl; }
tmp_buffer = allocated_buffer - gap;
max_size = ext * rcount;
/* in-place alltoall slow algorithm (but works) */
for (i = 0 ; i < size ; ++i) {
for (j = i+1 ; j < size ; ++j) {
if (i == rank) {
/* Copy the data into the temporary buffer */
err = ompi_datatype_copy_content_same_ddt (rdtype, rcount, tmp_buffer,
(char *) rbuf + j * max_size);
if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
/* Exchange data with the peer */
err = MCA_PML_CALL(irecv ((char *) rbuf + max_size * j, rcount, rdtype,
j, MCA_COLL_BASE_TAG_ALLTOALL, comm, &req));
if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
err = MCA_PML_CALL(send ((char *) tmp_buffer, rcount, rdtype,
j, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
comm));
if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
} else if (j == rank) {
/* Copy the data into the temporary buffer */
err = ompi_datatype_copy_content_same_ddt (rdtype, rcount, tmp_buffer,
(char *) rbuf + i * max_size);
if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
/* Exchange data with the peer */
err = MCA_PML_CALL(irecv ((char *) rbuf + max_size * i, rcount, rdtype,
i, MCA_COLL_BASE_TAG_ALLTOALL, comm, &req));
if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
err = MCA_PML_CALL(send ((char *) tmp_buffer, rcount, rdtype,
i, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
comm));
if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
} else {
continue;
}
/* Wait for the requests to complete */
err = ompi_request_wait ( &req, MPI_STATUSES_IGNORE);
if (MPI_SUCCESS != err) { line = __LINE__; goto error_hndl; }
}
}
error_hndl:
/* Free the temporary buffer */
if( NULL != allocated_buffer )
free (allocated_buffer);
if( MPI_SUCCESS != err ) {
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
"%s:%4d\tError occurred %d, rank %2d", __FILE__, line, err,
rank));
(void)line; // silence compiler warning
}
/* All done */
return err;
}
int
mca_fcoll_static_file_read_all (mca_io_ompio_file_t *fh,
void *buf,
int count,
struct ompi_datatype_t *datatype,
ompi_status_public_t *status)
{
int ret = OMPI_SUCCESS, iov_size=0, *bytes_remaining=NULL;
int i, j, l,cycles=0, local_cycles=0, *current_index=NULL;
int index, *disp_index=NULL, *bytes_per_process=NULL, current_position=0;
int **blocklen_per_process=NULL, *iovec_count_per_process=NULL;
int *displs=NULL, *sorted=NULL ,entries_per_aggregator=0;
int *sorted_file_offsets=NULL, temp_index=0, position=0, *temp_disp_index=NULL;
MPI_Aint **displs_per_process=NULL, global_iov_count=0, global_count=0;
MPI_Aint *memory_displacements=NULL;
int bytes_to_read_in_cycle=0;
size_t max_data=0, bytes_per_cycle=0;
uint32_t iov_count=0, iov_index=0;
struct iovec *decoded_iov=NULL, *iov=NULL;
mca_fcoll_static_local_io_array *local_iov_array=NULL, *global_iov_array=NULL;
mca_fcoll_static_local_io_array *file_offsets_for_agg=NULL;
char *global_buf=NULL, *receive_buf=NULL;
int blocklen[3] = {1, 1, 1};
int static_num_io_procs=1;
OPAL_PTRDIFF_TYPE d[3], base;
ompi_datatype_t *types[3];
ompi_datatype_t *io_array_type=MPI_DATATYPE_NULL;
ompi_datatype_t **sendtype = NULL;
MPI_Request *send_req=NULL, recv_req=NULL;
int my_aggregator=-1;
bool recvbuf_is_contiguous=false;
size_t ftype_size;
OPAL_PTRDIFF_TYPE ftype_extent, lb;
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
double read_time = 0.0, start_read_time = 0.0, end_read_time = 0.0;
double rcomm_time = 0.0, start_rcomm_time = 0.0, end_rcomm_time = 0.0;
double read_exch = 0.0, start_rexch = 0.0, end_rexch = 0.0;
mca_common_ompio_print_entry nentry;
#endif
#if DEBUG_ON
MPI_Aint gc_in;
#endif
opal_datatype_type_size ( &datatype->super, &ftype_size );
opal_datatype_get_extent ( &datatype->super, &lb, &ftype_extent );
/**************************************************************************
** 1. In case the data is not contigous in memory, decode it into an iovec
**************************************************************************/
if ( ( ftype_extent == (OPAL_PTRDIFF_TYPE) ftype_size) &&
opal_datatype_is_contiguous_memory_layout(&datatype->super,1) &&
0 == lb ) {
recvbuf_is_contiguous = true;
}
/* In case the data is not contigous in memory, decode it into an iovec */
if (!recvbuf_is_contiguous ) {
fh->f_decode_datatype ( (struct mca_io_ompio_file_t *)fh,
datatype,
count,
buf,
&max_data,
&decoded_iov,
&iov_count);
}
else {
max_data = count * datatype->super.size;
}
if ( MPI_STATUS_IGNORE != status ) {
status->_ucount = max_data;
}
fh->f_get_num_aggregators ( &static_num_io_procs );
fh->f_set_aggregator_props ((struct mca_io_ompio_file_t *) fh,
static_num_io_procs,
max_data);
my_aggregator = fh->f_procs_in_group[fh->f_aggregator_index];
/* printf("max_data %ld\n", max_data); */
ret = fh->f_generate_current_file_view((struct mca_io_ompio_file_t *)fh,
max_data,
&iov,
&iov_size);
if (ret != OMPI_SUCCESS){
goto exit;
}
if ( iov_size > 0 ) {
local_iov_array = (mca_fcoll_static_local_io_array *)malloc (iov_size * sizeof(mca_fcoll_static_local_io_array));
if ( NULL == local_iov_array){
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for (j=0; j < iov_size; j++){
local_iov_array[j].offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t)
iov[j].iov_base;
local_iov_array[j].length = (size_t)iov[j].iov_len;
local_iov_array[j].process_id = fh->f_rank;
}
}
else {
/* Allocate at least one element to correctly create the derived
data type */
local_iov_array = (mca_fcoll_static_local_io_array *)malloc (sizeof(mca_fcoll_static_local_io_array));
if ( NULL == local_iov_array){
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
local_iov_array[0].offset = (OMPI_MPI_OFFSET_TYPE)(intptr_t) 0;
local_iov_array[0].length = (size_t) 0;
local_iov_array[0].process_id = fh->f_rank;
}
d[0] = (OPAL_PTRDIFF_TYPE)&local_iov_array[0];
d[1] = (OPAL_PTRDIFF_TYPE)&local_iov_array[0].length;
d[2] = (OPAL_PTRDIFF_TYPE)&local_iov_array[0].process_id;
base = d[0];
for (i=0 ; i<3 ; i++) {
d[i] -= base;
}
/* io_array datatype for using in communication*/
types[0] = &ompi_mpi_long.dt;
types[1] = &ompi_mpi_long.dt;
types[2] = &ompi_mpi_int.dt;
ompi_datatype_create_struct (3,
blocklen,
d,
types,
&io_array_type);
ompi_datatype_commit (&io_array_type);
/* #########################################################*/
fh->f_get_bytes_per_agg ( (int*) &bytes_per_cycle);
local_cycles = ceil((double)max_data*fh->f_procs_per_group/bytes_per_cycle);
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rexch = MPI_Wtime();
#endif
ret = fh->f_comm->c_coll.coll_allreduce (&local_cycles,
&cycles,
1,
MPI_INT,
MPI_MAX,
fh->f_comm,
fh->f_comm->c_coll.coll_allreduce_module);
if (OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
if (my_aggregator == fh->f_rank) {
disp_index = (int *) malloc (fh->f_procs_per_group * sizeof(int));
if (NULL == disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
bytes_per_process = (int *) malloc (fh->f_procs_per_group * sizeof(int ));
if (NULL == bytes_per_process){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
bytes_remaining = (int *) calloc (fh->f_procs_per_group, sizeof(int));
if (NULL == bytes_remaining){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
current_index = (int *) calloc (fh->f_procs_per_group, sizeof(int));
if (NULL == current_index){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
blocklen_per_process = (int **)calloc (fh->f_procs_per_group, sizeof (int*));
if (NULL == blocklen_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process = (MPI_Aint **)calloc (fh->f_procs_per_group, sizeof (MPI_Aint*));
if (NULL == displs_per_process) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
iovec_count_per_process = (int *) calloc (fh->f_procs_per_group, sizeof(int));
if (NULL == iovec_count_per_process){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs = (int *) calloc (fh->f_procs_per_group, sizeof(int));
if (NULL == displs){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rexch = MPI_Wtime();
#endif
ret = fcoll_base_coll_allgather_array (&iov_size,
1,
MPI_INT,
iovec_count_per_process,
1,
MPI_INT,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if( OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
if (my_aggregator == fh->f_rank) {
displs[0] = 0;
global_iov_count = iovec_count_per_process[0];
for (i=1 ; i<fh->f_procs_per_group ; i++) {
global_iov_count += iovec_count_per_process[i];
displs[i] = displs[i-1] + iovec_count_per_process[i-1];
}
}
if ( (my_aggregator == fh->f_rank) &&
(global_iov_count > 0 )) {
global_iov_array = (mca_fcoll_static_local_io_array *) malloc (global_iov_count *
sizeof(mca_fcoll_static_local_io_array));
if (NULL == global_iov_array){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rexch = MPI_Wtime();
#endif
ret = fcoll_base_coll_gatherv_array (local_iov_array,
iov_size,
io_array_type,
global_iov_array,
iovec_count_per_process,
displs,
io_array_type,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
if (OMPI_SUCCESS != ret){
fprintf(stderr,"global_iov_array gather error!\n");
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
if (NULL != local_iov_array){
free(local_iov_array);
local_iov_array = NULL;
}
if ( ( my_aggregator == fh->f_rank) &&
( global_iov_count > 0 )) {
sorted = (int *)malloc (global_iov_count * sizeof(int));
if (NULL == sorted) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
read_local_heap_sort (global_iov_array, global_iov_count, sorted);
send_req = (MPI_Request *) malloc (fh->f_procs_per_group * sizeof(MPI_Request));
if (NULL == send_req){
opal_output ( 1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
sendtype = (ompi_datatype_t **) malloc (fh->f_procs_per_group * sizeof(ompi_datatype_t *));
if (NULL == sendtype) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for ( i=0; i<fh->f_procs_per_group; i++ ) {
sendtype[i] = MPI_DATATYPE_NULL;
}
if (NULL == bytes_per_process){
bytes_per_process = (int *) malloc (fh->f_procs_per_group * sizeof(int));
if (NULL == bytes_per_process){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
}
#if DEBUG_ON
if (my_aggregator == fh->f_rank) {
for (gc_in=0; gc_in<global_iov_count; gc_in++){
printf("%d: Offset[%ld]: %lld, Length[%ld]: %ld\n",
global_iov_array[sorted[gc_in]].process_id,
gc_in, global_iov_array[sorted[gc_in]].offset,
gc_in, global_iov_array[sorted[gc_in]].length);
}
}
#endif
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rexch = MPI_Wtime();
#endif
for (index = 0; index < cycles; index++){
if (my_aggregator == fh->f_rank) {
fh->f_num_of_io_entries = 0;
if (NULL != fh->f_io_array) {
free (fh->f_io_array);
fh->f_io_array = NULL;
}
if (NULL != global_buf) {
free (global_buf);
global_buf = NULL;
}
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if (NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements= NULL;
}
if ( NULL != sendtype ) {
for ( i=0; i<fh->f_procs_per_group; i++ ) {
if ( MPI_DATATYPE_NULL != sendtype[i] ) {
ompi_datatype_destroy (&sendtype[i] );
sendtype[i] = MPI_DATATYPE_NULL;
}
}
}
for(l=0;l<fh->f_procs_per_group;l++){
disp_index[l] = 1;
if (NULL != blocklen_per_process[l]){
free(blocklen_per_process[l]);
blocklen_per_process[l] = NULL;
}
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
if (NULL == blocklen_per_process[l]) {
opal_output (1, "OUT OF MEMORY for blocklen\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
if (NULL == displs_per_process[l]){
opal_output (1, "OUT OF MEMORY for displs\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
}
if (index < local_cycles ) {
if ((index == local_cycles-1) && (max_data % (bytes_per_cycle/fh->f_procs_per_group))) {
bytes_to_read_in_cycle = max_data - position;
}
else if (max_data <= bytes_per_cycle/fh->f_procs_per_group) {
bytes_to_read_in_cycle = max_data;
}
else {
bytes_to_read_in_cycle = bytes_per_cycle/fh->f_procs_per_group;
}
}
else {
bytes_to_read_in_cycle = 0;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rexch = MPI_Wtime();
#endif
fcoll_base_coll_gather_array (&bytes_to_read_in_cycle,
1,
MPI_INT,
bytes_per_process,
1,
MPI_INT,
fh->f_aggregator_index,
fh->f_procs_in_group,
fh->f_procs_per_group,
fh->f_comm);
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
if (recvbuf_is_contiguous ) {
receive_buf = &((char*)buf)[position];
}
else if (bytes_to_read_in_cycle) {
receive_buf = (char *) malloc (bytes_to_read_in_cycle * sizeof(char));
if ( NULL == receive_buf){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rcomm_time = MPI_Wtime();
#endif
ret = MCA_PML_CALL(irecv(receive_buf,
bytes_to_read_in_cycle,
MPI_BYTE,
my_aggregator,
123,
fh->f_comm,
&recv_req));
if (OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
if (my_aggregator == fh->f_rank) {
for (i=0;i<fh->f_procs_per_group; i++){
while (bytes_per_process[i] > 0){
/*printf("%d: bytes_per_process[%d]: %d, bytes_remaining[%d]: %d\n",
index, i, bytes_per_process[i], i, bytes_remaining[i]);*/
if (read_get_process_id(global_iov_array[sorted[current_index[i]]].process_id,
fh) == i){ /* current id owns this entry!*/
if (bytes_remaining[i]){ /*Remaining bytes in the current entry of
the global offset array*/
if (bytes_remaining[i] <= bytes_per_process[i]){
blocklen_per_process[i][disp_index[i] - 1] = bytes_remaining[i];
displs_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].offset +
(global_iov_array[sorted[current_index[i]]].length
- bytes_remaining[i]);
blocklen_per_process[i] = (int *) realloc
((void *)blocklen_per_process[i], (disp_index[i]+1)*sizeof(int));
displs_per_process[i] = (MPI_Aint *)realloc
((void *)displs_per_process[i], (disp_index[i]+1)*sizeof(MPI_Aint));
bytes_per_process[i] -= bytes_remaining[i];
blocklen_per_process[i][disp_index[i]] = 0;
displs_per_process[i][disp_index[i]] = 0;
disp_index[i] += 1;
bytes_remaining[i] = 0;
/* This entry has been used up, we need to move to the
next entry of this process and make current_index point there*/
current_index[i] = read_find_next_index(i,
current_index[i],
fh,
global_iov_array,
global_iov_count,
sorted);
if (current_index[i] == -1){
break;
}
continue;
}
else{
blocklen_per_process[i][disp_index[i] - 1] = bytes_per_process[i];
displs_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].offset +
(global_iov_array[sorted[current_index[i]]].length
- bytes_remaining[i]);
bytes_remaining[i] -= bytes_per_process[i];
bytes_per_process[i] = 0;
break;
}
}
else{
if (bytes_per_process[i] <
global_iov_array[sorted[current_index[i]]].length){
blocklen_per_process[i][disp_index[i] - 1] =
bytes_per_process[i];
displs_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].offset;
bytes_remaining[i] =
global_iov_array[sorted[current_index[i]]].length -
bytes_per_process[i];
bytes_per_process[i] = 0;
break;
}
else {
blocklen_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].length;
displs_per_process[i][disp_index[i] - 1] =
global_iov_array[sorted[current_index[i]]].offset;
blocklen_per_process[i] =
(int *) realloc ((void *)blocklen_per_process[i], (disp_index[i]+1)*sizeof(int));
displs_per_process[i] = (MPI_Aint *)realloc
((void *)displs_per_process[i], (disp_index[i]+1)*sizeof(MPI_Aint));
blocklen_per_process[i][disp_index[i]] = 0;
displs_per_process[i][disp_index[i]] = 0;
disp_index[i] += 1;
bytes_per_process[i] -=
global_iov_array[sorted[current_index[i]]].length;
current_index[i] = read_find_next_index(i,
current_index[i],
fh,
global_iov_array,
global_iov_count,
sorted);
if (current_index[i] == -1){
break;
}
}
}
}
else{
current_index[i] = read_find_next_index(i,
current_index[i],
fh,
global_iov_array,
global_iov_count,
sorted);
if (current_index[i] == -1){
bytes_per_process[i] = 0; /* no more entries left
to service this request*/
continue;
}
}
}
}
entries_per_aggregator=0;
for (i=0;i<fh->f_procs_per_group;i++){
for (j=0;j<disp_index[i];j++){
if (blocklen_per_process[i][j] > 0){
entries_per_aggregator++;
#if DEBUG_ON
printf("%d sends blocklen[%d]: %d, disp[%d]: %ld to %d\n",
fh->f_procs_in_group[i],j,
blocklen_per_process[i][j],j,
displs_per_process[i][j],
fh->f_rank);
#endif
}
}
}
if (entries_per_aggregator > 0){
file_offsets_for_agg = (mca_fcoll_static_local_io_array *)
malloc(entries_per_aggregator*sizeof(mca_fcoll_static_local_io_array));
if (NULL == file_offsets_for_agg) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
sorted_file_offsets = (int *) malloc (entries_per_aggregator * sizeof(int));
if (NULL == sorted_file_offsets){
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
temp_index=0;
global_count = 0;
for (i=0;i<fh->f_procs_per_group; i++){
for(j=0;j<disp_index[i]; j++){
if (blocklen_per_process[i][j] > 0){
file_offsets_for_agg[temp_index].length =
blocklen_per_process[i][j];
global_count += blocklen_per_process[i][j];
file_offsets_for_agg[temp_index].process_id = i;
file_offsets_for_agg[temp_index].offset =
displs_per_process[i][j];
temp_index++;
}
}
}
}
else{
continue;
}
read_local_heap_sort (file_offsets_for_agg,
entries_per_aggregator,
sorted_file_offsets);
memory_displacements = (MPI_Aint *) malloc
(entries_per_aggregator * sizeof(MPI_Aint));
memory_displacements[sorted_file_offsets[0]] = 0;
for (i=1; i<entries_per_aggregator; i++){
memory_displacements[sorted_file_offsets[i]] =
memory_displacements[sorted_file_offsets[i-1]] +
file_offsets_for_agg[sorted_file_offsets[i-1]].length;
}
global_buf = (char *) malloc (global_count * sizeof(char));
if (NULL == global_buf){
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
#if DEBUG_ON
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
for (i=0; i<entries_per_aggregator;i++){
printf("%d: OFFSET: %lld LENGTH: %ld, Mem-offset: %ld, disp_index :%d\n",
file_offsets_for_agg[sorted_file_offsets[i]].process_id,
file_offsets_for_agg[sorted_file_offsets[i]].offset,
file_offsets_for_agg[sorted_file_offsets[i]].length,
memory_displacements[sorted_file_offsets[i]],
disp_index[i]);
}
#endif
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
(entries_per_aggregator * sizeof (mca_io_ompio_io_array_t));
if (NULL == fh->f_io_array) {
opal_output(1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
fh->f_num_of_io_entries = 0;
fh->f_io_array[0].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[0]].offset;
fh->f_io_array[0].length = file_offsets_for_agg[sorted_file_offsets[0]].length;
fh->f_io_array[0].memory_address = global_buf+memory_displacements[sorted_file_offsets[0]];
fh->f_num_of_io_entries++;
for (i=1;i<entries_per_aggregator;i++){
if (file_offsets_for_agg[sorted_file_offsets[i-1]].offset +
file_offsets_for_agg[sorted_file_offsets[i-1]].length ==
file_offsets_for_agg[sorted_file_offsets[i]].offset){
fh->f_io_array[fh->f_num_of_io_entries - 1].length +=
file_offsets_for_agg[sorted_file_offsets[i]].length;
}
else{
fh->f_io_array[fh->f_num_of_io_entries].offset =
(IOVBASE_TYPE *)(intptr_t)file_offsets_for_agg[sorted_file_offsets[i]].offset;
fh->f_io_array[fh->f_num_of_io_entries].length =
file_offsets_for_agg[sorted_file_offsets[i]].length;
fh->f_io_array[fh->f_num_of_io_entries].memory_address =
global_buf+memory_displacements[sorted_file_offsets[i]];
fh->f_num_of_io_entries++;
}
}
#if DEBUG_ON
printf("*************************** %d\n", fh->f_num_of_io_entries);
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
printf(" ADDRESS: %p OFFSET: %ld LENGTH: %ld\n",
fh->f_io_array[i].memory_address,
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].offset,
fh->f_io_array[i].length);
}
#endif
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_read_time = MPI_Wtime();
#endif
if (fh->f_num_of_io_entries) {
if ( 0 > fh->f_fbtl->fbtl_preadv (fh)) {
opal_output (1, "READ FAILED\n");
ret = OMPI_ERROR;
goto exit;
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_read_time = MPI_Wtime();
read_time += end_read_time - start_read_time;
#endif
#if DEBUG_ON
printf("************Cycle: %d, Aggregator: %d ***************\n",
index+1,fh->f_rank);
if (my_aggregator == fh->f_rank){
for (i=0 ; i<global_count/4 ; i++)
printf (" READ %d \n",((int *)global_buf)[i]);
}
#endif
temp_disp_index = (int *)calloc (1, fh->f_procs_per_group * sizeof (int));
if (NULL == temp_disp_index) {
opal_output (1, "OUT OF MEMORY\n");
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
for (i=0; i<entries_per_aggregator; i++){
temp_index =
file_offsets_for_agg[sorted_file_offsets[i]].process_id;
displs_per_process[temp_index][temp_disp_index[temp_index]] =
memory_displacements[sorted_file_offsets[i]];
if (temp_disp_index[temp_index] < disp_index[temp_index]){
temp_disp_index[temp_index] += 1;
}
else{
printf("temp_disp_index[%d]: %d is greater than disp_index[%d]: %d\n",
temp_index, temp_disp_index[temp_index],
temp_index, disp_index[temp_index]);
}
}
if (NULL != temp_disp_index){
free(temp_disp_index);
temp_disp_index = NULL;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
start_rcomm_time = MPI_Wtime();
#endif
for (i=0;i<fh->f_procs_per_group; i++){
send_req[i] = MPI_REQUEST_NULL;
ompi_datatype_create_hindexed(disp_index[i],
blocklen_per_process[i],
displs_per_process[i],
MPI_BYTE,
&sendtype[i]);
ompi_datatype_commit(&sendtype[i]);
ret = MCA_PML_CALL (isend(global_buf,
1,
sendtype[i],
fh->f_procs_in_group[i],
123,
MCA_PML_BASE_SEND_STANDARD,
fh->f_comm,
&send_req[i]));
if(OMPI_SUCCESS != ret){
goto exit;
}
}
ret = ompi_request_wait_all (fh->f_procs_per_group,
send_req,
MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
} /* if ( my_aggregator == fh->f_rank ) */
ret = ompi_request_wait (&recv_req, MPI_STATUS_IGNORE);
if (OMPI_SUCCESS != ret){
goto exit;
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rcomm_time = MPI_Wtime();
rcomm_time += end_rcomm_time - start_rcomm_time;
#endif
position += bytes_to_read_in_cycle;
if (!recvbuf_is_contiguous) {
OPAL_PTRDIFF_TYPE mem_address;
size_t remaining = 0;
size_t temp_position = 0;
remaining = bytes_to_read_in_cycle;
while (remaining && (iov_count > iov_index)){
mem_address = (OPAL_PTRDIFF_TYPE)
(decoded_iov[iov_index].iov_base) + current_position;
if (remaining >=
(decoded_iov[iov_index].iov_len - current_position)) {
memcpy ((IOVBASE_TYPE *) mem_address,
receive_buf+temp_position,
decoded_iov[iov_index].iov_len - current_position);
remaining = remaining -
(decoded_iov[iov_index].iov_len - current_position);
temp_position = temp_position +
(decoded_iov[iov_index].iov_len - current_position);
iov_index = iov_index + 1;
current_position = 0;
}
else{
memcpy ((IOVBASE_TYPE *) mem_address,
receive_buf+temp_position,
remaining);
current_position = current_position + remaining;
remaining = 0;
}
}
if (NULL != receive_buf) {
free (receive_buf);
receive_buf = NULL;
}
}
}
#if OMPIO_FCOLL_WANT_TIME_BREAKDOWN
end_rexch = MPI_Wtime();
read_exch += end_rexch - start_rexch;
nentry.time[0] = read_time;
nentry.time[1] = rcomm_time;
nentry.time[2] = read_exch;
if (my_aggregator == fh->f_rank)
nentry.aggregator = 1;
else
nentry.aggregator = 0;
nentry.nprocs_for_coll = static_num_io_procs;
if (!mca_common_ompio_full_print_queue(fh->f_coll_read_time)){
mca_common_ompio_register_print_entry(fh->f_coll_read_time,
nentry);
}
#endif
exit:
if (NULL != decoded_iov){
free(decoded_iov);
decoded_iov = NULL;
}
if (NULL != displs){
free(displs);
displs = NULL;
}
if (NULL != iovec_count_per_process){
free(iovec_count_per_process);
iovec_count_per_process=NULL;
}
if (NULL != local_iov_array){
free(local_iov_array);
local_iov_array=NULL;
}
if (NULL != global_iov_array){
free(global_iov_array);
global_iov_array=NULL;
}
if (my_aggregator == fh->f_rank) {
for(l=0;l<fh->f_procs_per_group;l++){
if (blocklen_per_process) {
free(blocklen_per_process[l]);
}
if (NULL != displs_per_process[l]){
free(displs_per_process[l]);
displs_per_process[l] = NULL;
}
}
}
if (NULL != bytes_per_process){
free(bytes_per_process);
bytes_per_process =NULL;
}
if (NULL != disp_index){
free(disp_index);
disp_index =NULL;
}
if (NULL != displs_per_process){
free(displs_per_process);
displs_per_process = NULL;
}
if(NULL != bytes_remaining){
free(bytes_remaining);
bytes_remaining = NULL;
}
if(NULL != current_index){
free(current_index);
current_index = NULL;
}
if (NULL != blocklen_per_process){
free(blocklen_per_process);
blocklen_per_process =NULL;
}
if (NULL != bytes_remaining){
free(bytes_remaining);
bytes_remaining =NULL;
}
if (NULL != memory_displacements){
free(memory_displacements);
memory_displacements= NULL;
}
if (NULL != file_offsets_for_agg){
free(file_offsets_for_agg);
file_offsets_for_agg = NULL;
}
if (NULL != sorted_file_offsets){
free(sorted_file_offsets);
sorted_file_offsets = NULL;
}
if (NULL != sendtype){
free(sendtype);
sendtype=NULL;
}
if ( !recvbuf_is_contiguous ) {
if (NULL != receive_buf){
free(receive_buf);
receive_buf=NULL;
}
}
if (NULL != global_buf) {
free(global_buf);
global_buf = NULL;
}
if (NULL != sorted) {
free(sorted);
sorted = NULL;
}
if (NULL != send_req){
free(send_req);
send_req = NULL;
}
return ret;
}
