static void _XMP_reflect_acc_sched_dim(_XMP_array_t *adesc, int dim, int is_periodic) { _XMP_array_info_t *ai = &(adesc->info[dim]); _XMP_reflect_sched_t *reflect = ai->reflect_acc_sched; _XMP_array_info_t *ainfo = adesc->info; int target_tdim = ai->align_template_index; _XMP_nodes_info_t *ni = adesc->align_template->chunk[target_tdim].onto_nodes_info; int lwidth = ai->shadow_size_lo; int uwidth = ai->shadow_size_hi; int my_pos = ni->rank; int lb_pos = _XMP_get_owner_pos(adesc, dim, ai->ser_lower); int ub_pos = _XMP_get_owner_pos(adesc, dim, ai->ser_upper); int lo_pos = (my_pos == lb_pos) ? ub_pos : my_pos - 1; int hi_pos = (my_pos == ub_pos) ? lb_pos : my_pos + 1; int my_rank = adesc->align_template->onto_nodes->comm_rank; int lo_rank = my_rank + (lo_pos - my_pos) * ni->multiplier; int hi_rank = my_rank + (hi_pos - my_pos) * ni->multiplier; int count, blocklength; int type_size = adesc->type_size; int ndims = adesc->dim; long long stride; if (_XMPF_running & !_XMPC_running){ /* for XMP/F */ count = 1; blocklength = type_size; stride = ainfo[0].alloc_size * type_size; for(int i=ndims-2; i>=dim; i--) count *= ainfo[i+1].alloc_size; for(int i=1; i<=dim; i++){ blocklength *= ainfo[i-1].alloc_size; stride *= ainfo[i].alloc_size; } } else if(!_XMPF_running & _XMPC_running){ /* for XMP/C */ count = 1; blocklength = type_size; stride = ainfo[ndims-1].alloc_size * type_size; for(int i=1; i <= dim; i++) count *= ainfo[i-1].alloc_size; for(int i=ndims-2; i >= dim; i--){ blocklength *= ainfo[i+1].alloc_size; stride *= ainfo[i].alloc_size; } } else{ _XMP_fatal("cannot determin the base language."); } if (!is_periodic && my_pos == lb_pos) // no periodic lo_rank = -1; if (!is_periodic && my_pos == ub_pos) // no periodic hi_rank = -1; // Calulate offset off_t hi_src_offset = 0, hi_dst_offset = 0; if(lwidth){ for(int i=0; i<ndims; i++) { int lb_send, lb_recv, dim_acc; if (i == dim){ lb_send = ainfo[i].local_upper - lwidth + 1; lb_recv = ainfo[i].shadow_size_lo - lwidth; } else { // Note: including shadow area lb_send = 0; lb_recv = 0; } dim_acc = ainfo[i].dim_acc; hi_src_offset += lb_send * dim_acc * type_size; hi_dst_offset += lb_recv * dim_acc * type_size; } } off_t lo_src_offset = 0, lo_dst_offset = 0; if(uwidth){ for(int i=0; i<ndims; i++){ int lb_send, lb_recv, dim_acc; if (i == dim) { lb_send = ainfo[i].local_lower; lb_recv = ainfo[i].local_upper + 1; } else { // Note: including shadow area lb_send = 0; lb_recv = 0; } dim_acc = ainfo[i].dim_acc; lo_src_offset += lb_send * dim_acc * type_size; lo_dst_offset += lb_recv * dim_acc * type_size; } } reflect->count = count; reflect->blocklength = blocklength; reflect->stride = stride; reflect->lo_rank = lo_rank; reflect->hi_rank = hi_rank; reflect->lo_src_offset = lo_src_offset; reflect->lo_dst_offset = lo_dst_offset; reflect->hi_src_offset = hi_src_offset; reflect->hi_dst_offset = hi_dst_offset; }
void _XMP_reflect_pcopy_sched_dim(_XMP_array_t *adesc, int target_dim, int lwidth, int uwidth, int is_periodic, int shadow_comm_type){ if (lwidth == 0 && uwidth == 0) return; _XMP_array_info_t *ai = &(adesc->info[target_dim]); _XMP_array_info_t *ainfo = adesc->info; _XMP_ASSERT(ai->align_manner == _XMP_N_ALIGN_BLOCK); _XMP_ASSERT(ai->is_shadow_comm_member); if (lwidth > ai->shadow_size_lo || uwidth > ai->shadow_size_hi){ _XMP_fatal("reflect width is larger than shadow width."); } _XMP_reflect_sched_t *reflect = ai->reflect_sched; int target_tdim = ai->align_template_index; _XMP_nodes_info_t *ni = adesc->align_template->chunk[target_tdim].onto_nodes_info; if (ni->size == 1 && !is_periodic) return; int ndims = adesc->dim; // 0-origin int my_pos = ni->rank; int lb_pos = _XMP_get_owner_pos(adesc, target_dim, ai->ser_lower); int ub_pos = _XMP_get_owner_pos(adesc, target_dim, ai->ser_upper); int lo_pos = (my_pos == lb_pos) ? ub_pos : my_pos - 1; int hi_pos = (my_pos == ub_pos) ? lb_pos : my_pos + 1; MPI_Comm *comm = adesc->align_template->onto_nodes->comm; int my_rank = adesc->align_template->onto_nodes->comm_rank; int lo_rank = my_rank + (lo_pos - my_pos) * ni->multiplier; int hi_rank = my_rank + (hi_pos - my_pos) * ni->multiplier; int count = 0, blocklength = 0; long long stride = 0; int type_size = adesc->type_size; void *array_addr = adesc->array_addr_p; void *lo_send_array = NULL, *lo_recv_array = NULL; void *hi_send_array = NULL, *hi_recv_array = NULL; void *lo_send_buf = NULL; void *lo_recv_buf = NULL; void *hi_send_buf = NULL; void *hi_recv_buf = NULL; int lo_buf_size = 0; int hi_buf_size = 0; if (reflect->prev_pcopy_sched_type && lwidth == reflect->lo_width && uwidth == reflect->hi_width && is_periodic == reflect->is_periodic){ if ((adesc->order == MPI_ORDER_FORTRAN && target_dim != ndims - 1) || (adesc->order == MPI_ORDER_C && target_dim != 0)){ goto init_comm; } else if (reflect->prev_pcopy_sched_type != shadow_comm_type){ count = reflect->count; blocklength = reflect->blocklength; stride = reflect->stride; goto alloc_buf; } } // // setup data_type // if (adesc->order == MPI_ORDER_FORTRAN){ /* for XMP/F */ count = 1; blocklength = type_size; stride = ainfo[0].alloc_size * type_size; for (int i = ndims - 2; i >= target_dim; i--){ count *= ainfo[i+1].alloc_size; } for (int i = 1; i <= target_dim; i++){ blocklength *= ainfo[i-1].alloc_size; stride *= ainfo[i].alloc_size; } } else if (adesc->order == MPI_ORDER_C){ /* for XMP/C */ count = 1; blocklength = type_size; stride = ainfo[ndims-1].alloc_size * type_size; for (int i = 1; i <= target_dim; i++){ count *= ainfo[i-1].alloc_size; } for (int i = ndims - 2; i >= target_dim; i--){ blocklength *= ainfo[i+1].alloc_size; stride *= ainfo[i].alloc_size; } } else { _XMP_fatal("cannot determin the base language."); } // // calculate base address // alloc_buf: // for lower reflect if (lwidth){ lo_send_array = array_addr; lo_recv_array = array_addr; for (int i = 0; i < ndims; i++) { int lb_send, lb_recv; unsigned long long dim_acc; if (i == target_dim) { lb_send = ainfo[i].local_upper - lwidth + 1; lb_recv = ainfo[i].shadow_size_lo - lwidth;; } else { // Note: including shadow area lb_send = 0; lb_recv = 0; } dim_acc = ainfo[i].dim_acc; lo_send_array = (void *)((char *)lo_send_array + lb_send * dim_acc * type_size); lo_recv_array = (void *)((char *)lo_recv_array + lb_recv * dim_acc * type_size); } } // for upper reflect if (uwidth){ hi_send_array = array_addr; hi_recv_array = array_addr; for (int i = 0; i < ndims; i++) { int lb_send, lb_recv; unsigned long long dim_acc; if (i == target_dim) { lb_send = ainfo[i].local_lower; lb_recv = ainfo[i].local_upper + 1; } else { // Note: including shadow area lb_send = 0; lb_recv = 0; } dim_acc = ainfo[i].dim_acc; hi_send_array = (void *)((char *)hi_send_array + lb_send * dim_acc * type_size); hi_recv_array = (void *)((char *)hi_recv_array + lb_recv * dim_acc * type_size); } } // // Allocate buffers // if (reflect->prev_pcopy_sched_type == _XMP_COMM_REFLECT && ((adesc->order == MPI_ORDER_FORTRAN && target_dim == ndims - 1) || (adesc->order == MPI_ORDER_C && target_dim == 0))){ ; } else { _XMP_free(reflect->lo_send_buf); _XMP_free(reflect->lo_recv_buf); _XMP_free(reflect->hi_send_buf); _XMP_free(reflect->hi_recv_buf); } // for lower reflect if (lwidth){ lo_buf_size = lwidth * blocklength * count; if (shadow_comm_type == _XMP_COMM_REFLECT && ((adesc->order == MPI_ORDER_FORTRAN && target_dim == ndims - 1) || (adesc->order == MPI_ORDER_C && target_dim == 0))){ lo_send_buf = lo_send_array; lo_recv_buf = lo_recv_array; } else { _XMP_TSTART(t0); lo_send_buf = _XMP_alloc(lo_buf_size); lo_recv_buf = _XMP_alloc(lo_buf_size); _XMP_TEND2(xmptiming_.t_mem, xmptiming_.tdim_mem[target_dim], t0); } } // for upper reflect if (uwidth){ hi_buf_size = uwidth * blocklength * count; if (shadow_comm_type == _XMP_COMM_REFLECT && ((adesc->order == MPI_ORDER_FORTRAN && target_dim == ndims - 1) || (adesc->order == MPI_ORDER_C && target_dim == 0))){ hi_send_buf = hi_send_array; hi_recv_buf = hi_recv_array; } else { _XMP_TSTART(t0); hi_send_buf = _XMP_alloc(hi_buf_size); hi_recv_buf = _XMP_alloc(hi_buf_size); _XMP_TEND2(xmptiming_.t_mem, xmptiming_.tdim_mem[target_dim], t0); } } // // cache schedule // reflect->count = count; reflect->blocklength = blocklength; reflect->stride = stride; reflect->lo_send_array = lo_send_array; reflect->lo_recv_array = lo_recv_array; reflect->hi_send_array = hi_send_array; reflect->hi_recv_array = hi_recv_array; reflect->lo_send_buf = lo_send_buf; reflect->lo_recv_buf = lo_recv_buf; reflect->hi_send_buf = hi_send_buf; reflect->hi_recv_buf = hi_recv_buf; // // initialize communication // int src, dst; init_comm: if (!is_periodic && my_pos == lb_pos){ // no periodic lo_rank = MPI_PROC_NULL; } if (!is_periodic && my_pos == ub_pos){ // no periodic hi_rank = MPI_PROC_NULL; } lo_buf_size = lwidth * reflect->blocklength * reflect->count; hi_buf_size = uwidth * reflect->blocklength * reflect->count; // for lower shadow if (lwidth){ src = lo_rank; dst = hi_rank; } else { src = MPI_PROC_NULL; dst = MPI_PROC_NULL; } if (shadow_comm_type == _XMP_COMM_REDUCE_SHADOW){ if (reflect->req_reduce[0] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req_reduce[0]); } if (reflect->req_reduce[1] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req_reduce[1]); } MPI_Send_init(reflect->lo_recv_buf, lo_buf_size, MPI_BYTE, src, _XMP_N_MPI_TAG_REFLECT_LO, *comm, &reflect->req_reduce[0]); MPI_Recv_init(reflect->lo_send_buf, lo_buf_size, MPI_BYTE, dst, _XMP_N_MPI_TAG_REFLECT_LO, *comm, &reflect->req_reduce[1]); } else { if (reflect->req[0] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[0]); } if (reflect->req[1] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[1]); } MPI_Recv_init(reflect->lo_recv_buf, lo_buf_size, MPI_BYTE, src, _XMP_N_MPI_TAG_REFLECT_LO, *comm, &reflect->req[0]); MPI_Send_init(reflect->lo_send_buf, lo_buf_size, MPI_BYTE, dst, _XMP_N_MPI_TAG_REFLECT_LO, *comm, &reflect->req[1]); } // for upper shadow if (uwidth){ src = hi_rank; dst = lo_rank; } else { src = MPI_PROC_NULL; dst = MPI_PROC_NULL; } if (shadow_comm_type == _XMP_COMM_REDUCE_SHADOW){ if (reflect->req_reduce[2] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req_reduce[2]); } if (reflect->req_reduce[3] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req_reduce[3]); } MPI_Send_init(reflect->hi_recv_buf, hi_buf_size, MPI_BYTE, src, _XMP_N_MPI_TAG_REFLECT_HI, *comm, &reflect->req_reduce[2]); MPI_Recv_init(reflect->hi_send_buf, hi_buf_size, MPI_BYTE, dst, _XMP_N_MPI_TAG_REFLECT_HI, *comm, &reflect->req_reduce[3]); } else { if (reflect->req[2] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[2]); } if (reflect->req[3] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[3]); } MPI_Recv_init(reflect->hi_recv_buf, hi_buf_size, MPI_BYTE, src, _XMP_N_MPI_TAG_REFLECT_HI, *comm, &reflect->req[2]); MPI_Send_init(reflect->hi_send_buf, hi_buf_size, MPI_BYTE, dst, _XMP_N_MPI_TAG_REFLECT_HI, *comm, &reflect->req[3]); } reflect->prev_pcopy_sched_type = shadow_comm_type; reflect->lo_rank = lo_rank; reflect->hi_rank = hi_rank; }
static void _XMP_reflect_normal_sched_dim(_XMP_array_t *adesc, int target_dim, int lwidth, int uwidth, int is_periodic){ if (lwidth == 0 && uwidth == 0) return; _XMP_array_info_t *ai = &(adesc->info[target_dim]); _XMP_array_info_t *ainfo = adesc->info; _XMP_ASSERT(ai->align_manner == _XMP_N_ALIGN_BLOCK); _XMP_ASSERT(ai->is_shadow_comm_member); if (lwidth > ai->shadow_size_lo || uwidth > ai->shadow_size_hi){ _XMP_fatal("reflect width is larger than shadow width."); } _XMP_reflect_sched_t *reflect = ai->reflect_sched; int target_tdim = ai->align_template_index; _XMP_nodes_info_t *ni = adesc->align_template->chunk[target_tdim].onto_nodes_info; if (ni->size == 1 && !is_periodic) return; int ndims = adesc->dim; // 0-origin int my_pos = ni->rank; int lb_pos = _XMP_get_owner_pos(adesc, target_dim, ai->ser_lower); int ub_pos = _XMP_get_owner_pos(adesc, target_dim, ai->ser_upper); int lo_pos = (my_pos == lb_pos) ? ub_pos : my_pos - 1; int hi_pos = (my_pos == ub_pos) ? lb_pos : my_pos + 1; MPI_Comm *comm = adesc->align_template->onto_nodes->comm; int my_rank = adesc->align_template->onto_nodes->comm_rank; int lo_rank = my_rank + (lo_pos - my_pos) * ni->multiplier; int hi_rank = my_rank + (hi_pos - my_pos) * ni->multiplier; int type_size = adesc->type_size; void *lo_recv_buf = adesc->array_addr_p; void *lo_send_buf = adesc->array_addr_p; void *hi_recv_buf = adesc->array_addr_p; void *hi_send_buf = adesc->array_addr_p; // // setup MPI_data_type // int count = 0, blocklength = 0; long long stride = 0; if (adesc->order == MPI_ORDER_FORTRAN){ /* for XMP/F */ count = 1; blocklength = type_size; stride = ainfo[0].alloc_size * type_size; for (int i = ndims - 2; i >= target_dim; i--){ count *= ainfo[i+1].alloc_size; } for (int i = 1; i <= target_dim; i++){ blocklength *= ainfo[i-1].alloc_size; stride *= ainfo[i].alloc_size; } } else if (adesc->order == MPI_ORDER_C){ /* for XMP/C */ count = 1; blocklength = type_size; stride = ainfo[ndims-1].alloc_size * type_size; for (int i = 1; i <= target_dim; i++){ count *= ainfo[i-1].alloc_size; } for (int i = ndims - 2; i >= target_dim; i--){ blocklength *= ainfo[i+1].alloc_size; stride *= ainfo[i].alloc_size; } } else { _XMP_fatal("cannot determin the base language."); } // for lower reflect if (reflect->datatype_lo != MPI_DATATYPE_NULL){ MPI_Type_free(&reflect->datatype_lo); } MPI_Type_vector(count, blocklength * lwidth, stride, MPI_BYTE, &reflect->datatype_lo); MPI_Type_commit(&reflect->datatype_lo); // for upper reflect if (reflect->datatype_hi != MPI_DATATYPE_NULL){ MPI_Type_free(&reflect->datatype_hi); } MPI_Type_vector(count, blocklength * uwidth, stride, MPI_BYTE, &reflect->datatype_hi); MPI_Type_commit(&reflect->datatype_hi); // // calculate base address // // for lower reflect if (lwidth){ for (int i = 0; i < ndims; i++) { int lb_send, lb_recv, dim_acc; if (i == target_dim) { lb_send = ainfo[i].local_upper - lwidth + 1; lb_recv = ainfo[i].shadow_size_lo - lwidth; } else { // Note: including shadow area lb_send = 0; lb_recv = 0; } dim_acc = ainfo[i].dim_acc; lo_send_buf = (void *)((char *)lo_send_buf + lb_send * dim_acc * type_size); lo_recv_buf = (void *)((char *)lo_recv_buf + lb_recv * dim_acc * type_size); } } // for upper reflect if (uwidth){ for (int i = 0; i < ndims; i++) { int lb_send, lb_recv, dim_acc; if (i == target_dim) { lb_send = ainfo[i].local_lower; lb_recv = ainfo[i].local_upper + 1; } else { // Note: including shadow area lb_send = 0; lb_recv = 0; } dim_acc = ainfo[i].dim_acc; hi_send_buf = (void *)((char *)hi_send_buf + lb_send * dim_acc * type_size); hi_recv_buf = (void *)((char *)hi_recv_buf + lb_recv * dim_acc * type_size); } } // // initialize communication // int src, dst; if (!is_periodic && my_pos == lb_pos){ // no periodic lo_rank = MPI_PROC_NULL; } if (!is_periodic && my_pos == ub_pos){ // no periodic hi_rank = MPI_PROC_NULL; } // for lower reflect if (lwidth){ src = lo_rank; dst = hi_rank; } else { src = MPI_PROC_NULL; dst = MPI_PROC_NULL; } if (reflect->req[0] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[0]); } if (reflect->req[1] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[1]); } MPI_Recv_init(lo_recv_buf, 1, reflect->datatype_lo, src, _XMP_N_MPI_TAG_REFLECT_LO, *comm, &reflect->req[0]); MPI_Send_init(lo_send_buf, 1, reflect->datatype_lo, dst, _XMP_N_MPI_TAG_REFLECT_LO, *comm, &reflect->req[1]); // for upper reflect if (uwidth){ src = hi_rank; dst = lo_rank; } else { src = MPI_PROC_NULL; dst = MPI_PROC_NULL; } if (reflect->req[2] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[2]); } if (reflect->req[3] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[3]); } MPI_Recv_init(hi_recv_buf, 1, reflect->datatype_hi, src, _XMP_N_MPI_TAG_REFLECT_HI, *comm, &reflect->req[2]); MPI_Send_init(hi_send_buf, 1, reflect->datatype_hi, dst, _XMP_N_MPI_TAG_REFLECT_HI, *comm, &reflect->req[3]); }
static void _XMP_reflect_rdma_sched_dim(_XMP_array_t *adesc, int target_dim, int lwidth, int uwidth, int is_periodic){ if (lwidth == 0 && uwidth == 0) return; _XMP_array_info_t *ai = &(adesc->info[target_dim]); _XMP_array_info_t *ainfo = adesc->info; _XMP_ASSERT(ai->align_manner == _XMP_N_ALIGN_BLOCK); _XMP_ASSERT(ai->is_shadow_comm_member); if (lwidth > ai->shadow_size_lo || uwidth > ai->shadow_size_hi){ _XMP_fatal("reflect width is larger than shadow width."); } _XMP_reflect_sched_t *reflect = ai->reflect_sched; int target_tdim = ai->align_template_index; _XMP_nodes_info_t *ni = adesc->align_template->chunk[target_tdim].onto_nodes_info; int ndims = adesc->dim; // 0-origin int my_pos = ni->rank; int lb_pos = _XMP_get_owner_pos(adesc, target_dim, ai->ser_lower); int ub_pos = _XMP_get_owner_pos(adesc, target_dim, ai->ser_upper); int my_rank = adesc->align_template->onto_nodes->comm_rank; int lo_pos = (my_pos == lb_pos) ? ub_pos : my_pos - 1; int hi_pos = (my_pos == ub_pos) ? lb_pos : my_pos + 1; int lo_rank = my_rank + (lo_pos - my_pos) * ni->multiplier; int hi_rank = my_rank + (hi_pos - my_pos) * ni->multiplier; int type_size = adesc->type_size; void *lo_send_array, *lo_recv_array; void *hi_send_array, *hi_recv_array; uint64_t rdma_raddr; // // calculate offset // int count = 0, blocklength = 0; long long stride = 0; if (adesc->order == MPI_ORDER_FORTRAN){ /* for XMP/F */ count = 1; blocklength = type_size; stride = ainfo[0].alloc_size * type_size; for (int i = ndims - 2; i >= target_dim; i--){ count *= ainfo[i+1].alloc_size; } for (int i = 1; i <= target_dim; i++){ blocklength *= ainfo[i-1].alloc_size; stride *= ainfo[i].alloc_size; } } else if (adesc->order == MPI_ORDER_C){ /* for XMP/C */ count = 1; blocklength = type_size; stride = ainfo[ndims-1].alloc_size * type_size; for (int i = 1; i <= target_dim; i++){ count *= ainfo[i-1].alloc_size; } for (int i = ndims - 2; i >= target_dim; i--){ blocklength *= ainfo[i+1].alloc_size; stride *= ainfo[i].alloc_size; } } else { _XMP_fatal("cannot determin the base language."); } // // calculate base address // // for lower reflect while ((rdma_raddr = FJMPI_Rdma_get_remote_addr(hi_rank, adesc->rdma_memid)) == FJMPI_RDMA_ERROR); if (lwidth){ lo_send_array = (void *)adesc->rdma_addr; lo_recv_array = (void *)rdma_raddr; for (int i = 0; i < ndims; i++) { int lb_send, lb_recv, dim_acc; if (i == target_dim) { lb_send = ainfo[i].local_upper - lwidth + 1; lb_recv = ainfo[i].shadow_size_lo - lwidth; } else { // Note: including shadow area lb_send = 0; lb_recv = 0; } dim_acc = ainfo[i].dim_acc; lo_send_array = (void *)((uint64_t)lo_send_array + lb_send * dim_acc * type_size); lo_recv_array = (void *)((uint64_t)lo_recv_array + lb_recv * dim_acc * type_size); } } // for upper reflect while ((rdma_raddr = FJMPI_Rdma_get_remote_addr(lo_rank, adesc->rdma_memid)) == FJMPI_RDMA_ERROR); if (uwidth){ hi_send_array = (void *)adesc->rdma_addr; hi_recv_array = (void *)rdma_raddr; for (int i = 0; i < ndims; i++) { int lb_send, lb_recv, dim_acc; if (i == target_dim) { lb_send = ainfo[i].local_lower; lb_recv = ainfo[i].local_upper + 1; } else { // Note: including shadow area lb_send = 0; lb_recv = 0; } dim_acc = ainfo[i].dim_acc; hi_send_array = (void *)((uint64_t)hi_send_array + lb_send * dim_acc * type_size); hi_recv_array = (void *)((uint64_t)hi_recv_array + lb_recv * dim_acc * type_size); } } // // cache schedule // if (!is_periodic && my_pos == lb_pos){ // no periodic lo_rank = -1; } if (!is_periodic && my_pos == ub_pos){ // no periodic hi_rank = -1; } reflect->count = count; reflect->blocklength = blocklength; reflect->stride = stride; reflect->lo_send_array = lo_send_array; reflect->lo_recv_array = lo_recv_array; reflect->hi_send_array = hi_send_array; reflect->hi_recv_array = hi_recv_array; reflect->lo_rank = lo_rank; reflect->hi_rank = hi_rank; }
static void _XMP_reflect_sched_dir(_XMP_array_t *adesc, int ishadow[], int lwidth[], int uwidth[], int is_periodic_dim[]){ int ndims = adesc->dim; _XMP_array_info_t *ainfo = adesc->info; MPI_Comm *comm = adesc->align_template->onto_nodes->comm; int my_rank = adesc->align_template->onto_nodes->comm_rank; int src = my_rank; int dst = my_rank; _XMP_async_reflect_t *async_reflect = adesc->async_reflect; MPI_Datatype *send_dtype = &async_reflect->datatype[async_reflect->nreqs]; MPI_Datatype *recv_dtype = send_dtype + 1; MPI_Request *send_req = &async_reflect->reqs[async_reflect->nreqs]; MPI_Request *recv_req = send_req + 1; int width[_XMP_N_MAX_DIM] = { 0 }; int is_periodic = 1; int at_tail = 0, at_head = 0; void *recv_buf = adesc->array_addr_p; void *send_buf = adesc->array_addr_p; // // setup neighbor nodes // for (int i = 0; i < ndims; i++){ if (ishadow[i] == 0) continue; width[i] = ishadow[i] > 0 ? uwidth[i] : lwidth[i]; is_periodic = is_periodic * is_periodic_dim[i]; _XMP_array_info_t *ai = &(adesc->info[i]); _XMP_ASSERT(ai->align_manner == _XMP_N_ALIGN_BLOCK); _XMP_ASSERT(ai->is_shadow_comm_member); if (lwidth[i] > ai->shadow_size_lo || uwidth[i] > ai->shadow_size_hi){ _XMP_fatal("reflect width is larger than shadow width."); } int tdim = ai->align_template_index; _XMP_nodes_info_t *ni = adesc->align_template->chunk[tdim].onto_nodes_info; // don't skip if no comm. is needed. //if (ni->size == 1 && !is_periodic_dim[i]) return; // 0-origin int my_pos = ni->rank; int lb_pos = _XMP_get_owner_pos(adesc, i, ai->ser_lower); int ub_pos = _XMP_get_owner_pos(adesc, i, ai->ser_upper); int src_pos; int dst_pos; if (ishadow[i] > 0){ src_pos = my_pos + 1; dst_pos = my_pos - 1; if (my_pos == lb_pos){ at_head = 1; dst_pos = ub_pos; } if (my_pos == ub_pos){ at_tail = 1; src_pos = lb_pos; } } else { //ishadow[i] < 0 src_pos = my_pos - 1; dst_pos = my_pos + 1; if (my_pos == lb_pos){ at_tail = 1; src_pos = ub_pos; } if (my_pos == ub_pos){ at_head = 1; dst_pos = lb_pos; } } src = src + (src_pos - my_pos) * ni->multiplier; dst = dst + (dst_pos - my_pos) * ni->multiplier; } src = (is_periodic || !at_tail) ? src : MPI_PROC_NULL; dst = (is_periodic || !at_head) ? dst : MPI_PROC_NULL; // // setup MPI_data_type // int sizes[_XMP_N_MAX_DIM]; int subsizes[_XMP_N_MAX_DIM]; int send_starts[_XMP_N_MAX_DIM]; int recv_starts[_XMP_N_MAX_DIM]; for (int i = 0; i < ndims; i++){ sizes[i] = ainfo[i].alloc_size; subsizes[i] = (ishadow[i] == 0) ? ainfo[i].par_size : width[i]; if (ishadow[i] == 0){ // excludes shadow area send_starts[i] = ainfo[i].shadow_size_lo; recv_starts[i] = ainfo[i].shadow_size_lo; } else if (ishadow[i] > 0){ send_starts[i] = ainfo[i].shadow_size_lo; recv_starts[i] = ainfo[i].local_upper + 1; } else { send_starts[i] = ainfo[i].local_upper - width[i] + 1; recv_starts[i] = ainfo[i].shadow_size_lo - width[i]; } } MPI_Type_create_subarray(ndims, sizes, subsizes, send_starts, adesc->order, adesc->mpi_type, send_dtype); MPI_Type_create_subarray(ndims, sizes, subsizes, recv_starts, adesc->order, adesc->mpi_type, recv_dtype); MPI_Type_commit(send_dtype); MPI_Type_commit(recv_dtype); // // initialize communication // MPI_Send_init(send_buf, 1, *send_dtype, dst, _XMP_N_MPI_TAG_REFLECT_LO, *comm, send_req); MPI_Recv_init(recv_buf, 1, *recv_dtype, src, _XMP_N_MPI_TAG_REFLECT_LO, *comm, recv_req); async_reflect->nreqs += 2; }
static void _XMP_reflect_pcopy_sched_dim(_XMP_array_t *adesc, int target_dim, int lwidth, int uwidth, int is_periodic, void *dev_array_addr, int *lwidths, int *uwidths){ //printf("desc=%p, tardim=%d, lw=%d, uw=%d, devp=%p\n", adesc, target_dim, lwidth, uwidth, dev_array_addr); if (lwidth == 0 && uwidth == 0) return; _XMP_array_info_t *ai = &(adesc->info[target_dim]); _XMP_array_info_t *ainfo = adesc->info; _XMP_ASSERT(ai->align_manner == _XMP_N_ALIGN_BLOCK); _XMP_ASSERT(ai->is_shadow_comm_member); if (lwidth > ai->shadow_size_lo || uwidth > ai->shadow_size_hi){ _XMP_fatal("reflect width is larger than shadow width."); } _XMP_reflect_sched_t *reflect = ai->reflect_acc_sched; int target_tdim = ai->align_template_index; _XMP_nodes_info_t *ni = adesc->align_template->chunk[target_tdim].onto_nodes_info; int ndims = adesc->dim; // 0-origin int my_pos = ni->rank; int lb_pos = _XMP_get_owner_pos(adesc, target_dim, ai->ser_lower); int ub_pos = _XMP_get_owner_pos(adesc, target_dim, ai->ser_upper); int lo_pos = (my_pos == lb_pos) ? ub_pos : my_pos - 1; int hi_pos = (my_pos == ub_pos) ? lb_pos : my_pos + 1; MPI_Comm *comm = adesc->align_template->onto_nodes->comm; int my_rank = adesc->align_template->onto_nodes->comm_rank; int lo_rank = my_rank + (lo_pos - my_pos) * ni->multiplier; int hi_rank = my_rank + (hi_pos - my_pos) * ni->multiplier; int type_size = adesc->type_size; //void *array_addr = adesc->array_addr_p; void *lo_send_array = NULL; void *lo_recv_array = NULL; void *hi_send_array = NULL; void *hi_recv_array = NULL; void *lo_send_dev_buf = NULL; void *lo_recv_dev_buf = NULL; void *hi_send_dev_buf = NULL; void *hi_recv_dev_buf = NULL; void *lo_send_host_buf = NULL; void *lo_recv_host_buf = NULL; void *hi_send_host_buf = NULL; void *hi_recv_host_buf = NULL; void *mpi_lo_send_buf = NULL; void *mpi_lo_recv_buf = NULL; void *mpi_hi_send_buf = NULL; void *mpi_hi_recv_buf = NULL; int lo_buf_size = 0; int hi_buf_size = 0; // // setup data_type // int count = 0, blocklength = 0; long long stride = 0; // int count_offset = 0; if (_XMPF_running && !_XMPC_running){ /* for XMP/F */ count = 1; blocklength = type_size; stride = ainfo[0].alloc_size * type_size; for (int i = ndims - 2; i >= target_dim; i--){ count *= ainfo[i+1].alloc_size; } for (int i = 1; i <= target_dim; i++){ blocklength *= ainfo[i-1].alloc_size; stride *= ainfo[i].alloc_size; } } else if (!_XMPF_running && _XMPC_running){ /* for XMP/C */ count = 1; blocklength = type_size; stride = ainfo[ndims-1].alloc_size * type_size; /* if(target_dim > 0){ */ /* count *= ainfo[0].par_size; */ /* count_offset = ainfo[0].shadow_size_lo; */ /* } */ /* for (int i = 1; i < target_dim; i++){ */ /* count *= ainfo[i].alloc_size; */ /* } */ /* for (int i = ndims - 2; i >= target_dim; i--){ */ /* blocklength *= ainfo[i+1].alloc_size; */ /* stride *= ainfo[i].alloc_size; */ /* } */ if(target_dim == 0){ count *= 1; if(ndims >= 2){ blocklength *= (ainfo[1].par_size + lwidths[1] + uwidths[1]); } }else{ count *= (ainfo[0].par_size + lwidths[0] + uwidths[0]); for(int i = 1; i < target_dim; i++){ count *= ainfo[i].alloc_size; } blocklength *= ainfo[target_dim+1].alloc_size; stride *= ainfo[target_dim].alloc_size; } for(int i = target_dim+2; i < ndims; i++){ blocklength *= ainfo[i].alloc_size; } for(int i = target_dim+1 ; i < ndims - 1; i++){ stride *= ainfo[i].alloc_size; } /* mod_4 */ count = 1; blocklength = 1; stride = 1; for(int i = 0; i < ndims; i++){ int fact = (i == target_dim)? 1 : (ainfo[i].par_size + lwidths[i] + uwidths[i]); int alloc_size = ainfo[i].alloc_size; if(blocklength == 1 || fact == alloc_size){ blocklength *= fact; stride *= alloc_size; }else if(count == 1 && target_dim != 0){ //to be contiguous if target_dim==0 count = blocklength; blocklength = fact; stride = alloc_size; }else{ blocklength *= alloc_size; stride *= alloc_size; } //printf("tar=%d, i=%d, fact=%d, allocsize=%d, (%d,%d,%lld)\n", target_dim, i, fact, alloc_size, count , blocklength, stride); } blocklength *= type_size; stride *= type_size; /* mod_4 end */ /* it used at 150717 for (int i = 1; i <= target_dim; i++){ count *= ainfo[i-1].alloc_size; } for (int i = ndims - 2; i >= target_dim; i--){ blocklength *= ainfo[i+1].alloc_size; stride *= ainfo[i].alloc_size; } */ /* for (int i = target_dim + 1; i < ndims; i++){ */ /* blocklength *= ainfo[i].alloc_size; */ /* } */ /* for (int i = target_dim; i < ndims - 1; i++){ */ /* stride *= ainfo[i].alloc_size; */ /* } */ // printf("count =%d, blength=%d, stride=%lld\n", count ,blocklength, stride); // printf("ainfo[0].par_size=%d\n", ainfo[0].par_size); // printf("count_ofset=%d,\n", count_offset); } else { _XMP_fatal("cannot determin the base language."); } // // calculate base address // // for lower reflect if (lwidth){ lo_send_array = lo_recv_array = (void *)((char*)dev_array_addr + /*count_offset*/0 * stride); for (int i = 0; i < ndims; i++) { int lb_send, lb_recv; unsigned long long dim_acc; if (i == target_dim) { //printf("ainfo[%d].local_upper=%d\n",i,ainfo[i].local_upper); lb_send = ainfo[i].local_upper - lwidth + 1; lb_recv = ainfo[i].shadow_size_lo - lwidth; ////ainfo[i].local_lower - lwidth; } else { // Note: including shadow area lb_send = 0; //// ainfo[i].local_lower - ainfo[i].shadow_size_lo; lb_recv = 0; //// ainfo[i].local_lower - ainfo[i].shadow_size_lo; } dim_acc = ainfo[i].dim_acc; lo_send_array = (void *)((char *)lo_send_array + lb_send * dim_acc * type_size); lo_recv_array = (void *)((char *)lo_recv_array + lb_recv * dim_acc * type_size); } } // for upper reflect if (uwidth){ hi_send_array = hi_recv_array = (void *)((char*)dev_array_addr + /*count_offset*/0 * stride); for (int i = 0; i < ndims; i++) { int lb_send, lb_recv; unsigned long long dim_acc; if (i == target_dim) { lb_send = ainfo[i].local_lower; lb_recv = ainfo[i].local_upper + 1; } else { // Note: including shadow area lb_send = 0; //ainfo[i].local_lower - ainfo[i].shadow_size_lo; lb_recv = 0; //ainfo[i].local_lower - ainfo[i].shadow_size_lo; } dim_acc = ainfo[i].dim_acc; hi_send_array = (void *)((char *)hi_send_array + lb_send * dim_acc * type_size); hi_recv_array = (void *)((char *)hi_recv_array + lb_recv * dim_acc * type_size); } } // for lower reflect if (reflect->datatype_lo != MPI_DATATYPE_NULL){ MPI_Type_free(&reflect->datatype_lo); } if(packVector || count == 1){ MPI_Type_contiguous(blocklength * lwidth * count, MPI_BYTE, &reflect->datatype_lo); // MPI_Type_contiguous(blocklength * lwidth * count / type_size, MPI_FLOAT, &reflect->datatype_lo); fprintf(stderr, "dim=%d, send elements lo = %d\n", target_dim, blocklength * lwidth * count / type_size); //fprintf(stderr, "useHostBuf=%c , packVector=%c\n", useHostBuffer, packVector); // if(useHostBuffer){ fprintf(stderr,"using host buffer\n"); } // if(packVector){ fprintf(stderr, "using pack vector\n"); } }else{ MPI_Type_vector(count, blocklength * lwidth, stride, MPI_BYTE, &reflect->datatype_lo); } MPI_Type_commit(&reflect->datatype_lo); // for upper reflect if (reflect->datatype_hi != MPI_DATATYPE_NULL){ MPI_Type_free(&reflect->datatype_hi); } if(packVector || count == 1){ MPI_Type_contiguous(blocklength * uwidth * count, MPI_BYTE, &reflect->datatype_hi); // MPI_Type_contiguous(blocklength * uwidth * count / type_size, MPI_FLOAT, &reflect->datatype_hi); fprintf(stderr, "dim=%d, send elements hi = %d\n", target_dim, blocklength * uwidth * count / type_size); }else{ MPI_Type_vector(count, blocklength * uwidth, stride, MPI_BYTE, &reflect->datatype_hi); } MPI_Type_commit(&reflect->datatype_hi); // // Allocate buffers // if(useHostBuffer){ CUDA_SAFE_CALL(cudaFreeHost(reflect->lo_send_host_buf)); CUDA_SAFE_CALL(cudaFreeHost(reflect->lo_recv_host_buf)); } if ((_XMPF_running && target_dim != ndims - 1) || (_XMPC_running && target_dim != 0)){ if(packVector){ CUDA_SAFE_CALL(cudaFree(reflect->lo_send_buf)); CUDA_SAFE_CALL(cudaFree(reflect->lo_recv_buf)); } } if ((_XMPF_running && target_dim == ndims - 1) || (_XMPC_running && target_dim == 0)){ // } // for lower reflect if (lwidth){ lo_buf_size = lwidth * blocklength * count; hi_buf_size = uwidth * blocklength * count; if ((_XMPF_running && target_dim == ndims - 1) || (_XMPC_running && target_dim == 0)){ lo_send_dev_buf = lo_send_array; lo_recv_dev_buf = lo_recv_array; hi_send_dev_buf = hi_send_array; hi_recv_dev_buf = hi_recv_array; } else { _XMP_TSTART(t0); if(packVector){ CUDA_SAFE_CALL(cudaMalloc((void **)&lo_send_dev_buf, lo_buf_size + hi_buf_size)); hi_send_dev_buf = (char*)lo_send_dev_buf + lo_buf_size; CUDA_SAFE_CALL(cudaMalloc((void **)&lo_recv_dev_buf, lo_buf_size + hi_buf_size)); hi_recv_dev_buf = (char*)lo_recv_dev_buf + lo_buf_size; }else{ lo_send_dev_buf = lo_send_array; lo_recv_dev_buf = lo_recv_array; hi_send_dev_buf = hi_send_array; hi_recv_dev_buf = hi_recv_array; } _XMP_TEND2(xmptiming_.t_mem, xmptiming_.tdim_mem[target_dim], t0); } if(useHostBuffer){ CUDA_SAFE_CALL(cudaMallocHost((void**)&lo_send_host_buf, lo_buf_size + hi_buf_size)); hi_send_host_buf = (char*)lo_send_host_buf + lo_buf_size; CUDA_SAFE_CALL(cudaMallocHost((void**)&lo_recv_host_buf, lo_buf_size + hi_buf_size)); hi_recv_host_buf = (char*)lo_recv_host_buf + lo_buf_size; mpi_lo_send_buf = lo_send_host_buf; mpi_lo_recv_buf = lo_recv_host_buf; mpi_hi_send_buf = hi_send_host_buf; mpi_hi_recv_buf = hi_recv_host_buf; }else{ mpi_lo_send_buf = lo_send_dev_buf; mpi_lo_recv_buf = lo_recv_dev_buf; mpi_hi_send_buf = hi_send_dev_buf; mpi_hi_recv_buf = hi_recv_dev_buf; } } // for upper reflect // // initialize communication // int src, dst; if (!is_periodic && my_pos == lb_pos){ // no periodic lo_rank = MPI_PROC_NULL; } if (!is_periodic && my_pos == ub_pos){ // no periodic hi_rank = MPI_PROC_NULL; } // for lower shadow if (lwidth){ src = lo_rank; dst = hi_rank; } else { src = MPI_PROC_NULL; dst = MPI_PROC_NULL; } // fprintf(stderr, "dim=%d, lo_src=%d, lo_dst=%d\n", target_dim, src, dst); if (reflect->req[0] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[0]); } if (reflect->req[1] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[1]); } MPI_Recv_init(mpi_lo_recv_buf, 1, reflect->datatype_lo, src, _XMP_N_MPI_TAG_REFLECT_LO, *comm, &reflect->req[0]); MPI_Send_init(mpi_lo_send_buf, 1, reflect->datatype_lo, dst, _XMP_N_MPI_TAG_REFLECT_LO, *comm, &reflect->req[1]); // for upper shadow if (uwidth){ src = hi_rank; dst = lo_rank; } else { src = MPI_PROC_NULL; dst = MPI_PROC_NULL; } // fprintf(stderr, "dim=%d, hi_src=%d, hi_dst=%d\n", target_dim, src, dst); if (reflect->req[2] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[2]); } if (reflect->req[3] != MPI_REQUEST_NULL){ MPI_Request_free(&reflect->req[3]); } MPI_Recv_init(mpi_hi_recv_buf, 1, reflect->datatype_hi, src, _XMP_N_MPI_TAG_REFLECT_HI, *comm, &reflect->req[2]); MPI_Send_init(mpi_hi_send_buf, 1, reflect->datatype_hi, dst, _XMP_N_MPI_TAG_REFLECT_HI, *comm, &reflect->req[3]); // // cache schedule // reflect->count = count; reflect->blocklength = blocklength; reflect->stride = stride; reflect->lo_send_array = lo_send_array; reflect->lo_recv_array = lo_recv_array; reflect->hi_send_array = hi_send_array; reflect->hi_recv_array = hi_recv_array; if(packVector){ reflect->lo_send_buf = lo_send_dev_buf; reflect->lo_recv_buf = lo_recv_dev_buf; reflect->hi_send_buf = hi_send_dev_buf; reflect->hi_recv_buf = hi_recv_dev_buf; } if(useHostBuffer){ reflect->lo_send_host_buf = lo_send_host_buf; reflect->lo_recv_host_buf = lo_recv_host_buf; reflect->hi_send_host_buf = hi_send_host_buf; reflect->hi_recv_host_buf = hi_recv_host_buf; } reflect->lo_rank = lo_rank; reflect->hi_rank = hi_rank; // gpu async reflect->lo_async_id = _XMP_alloc(sizeof(cudaStream_t)); CUDA_SAFE_CALL(cudaStreamCreate(reflect->lo_async_id)); if(target_dim != 0 && (!useHostBuffer || (lo_rank != MPI_PROC_NULL && hi_rank != MPI_PROC_NULL && (lo_buf_size / type_size) <= useSingleStreamLimit)) ){ reflect->hi_async_id = NULL; }else{ cudaStream_t *hi_stream = (cudaStream_t*)_XMP_alloc(sizeof(cudaStream_t)); CUDA_SAFE_CALL(cudaStreamCreate(hi_stream)); reflect->hi_async_id = (void*)hi_stream; } reflect->event = _XMP_alloc(sizeof(cudaEvent_t)); CUDA_SAFE_CALL(cudaEventCreateWithFlags(reflect->event, cudaEventDisableTiming)); }