/* query to see if the component is available for use, and can
* satisfy the thread and progress requirements
*/
int mca_bcol_iboffload_init_query(bool enable_progress_threads,
bool enable_mpi_threads)
{
int rc;
mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
IBOFFLOAD_VERBOSE(10, ("Init Iboffload component.\n"));
/* Get list of HCAs and ports */
rc = iboffload_load_devices();
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_ERROR(("Load devices error.\n"));
goto unload_devices;
}
/* Setup the BSRQ QP's based on the final value of
mca_bcol_iboffload_component.receive_queues. */
rc = setup_qps();
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_ERROR(("QPs setup error.\n"));
goto unload_devices;
}
cm->super.collm_init_query = mca_bcol_iboffload_dummy_init_query;
return OMPI_SUCCESS;
/* done */
unload_devices:
IBOFFLOAD_ERROR(("Release devices: an error occured.\n"));
iboffload_release_devices();
return rc;
}
static int mca_bcol_iboffload_alloc_reg_qp_resource(int qp_index, mca_bcol_iboffload_device_t *device)
{
int length;
mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
ompi_free_list_t *frags_free = &device->frags_free[qp_index];
OBJ_CONSTRUCT(frags_free, ompi_free_list_t);
length = cm->qp_infos[qp_index].size;
IBOFFLOAD_VERBOSE(10, ("free list len %d\n", length));
if (OMPI_SUCCESS != ompi_free_list_init_ex_new(frags_free,
sizeof(mca_bcol_iboffload_frag_t), MCA_IBOFFLOAD_CACHE_LINE_SIZE,
OBJ_CLASS(mca_bcol_iboffload_frag_t),
length, cm->buffer_alignment,
cm->free_list_num,
cm->free_list_max,
cm->free_list_inc,
device->mpool,
mca_bcol_iboffload_frag_init,
(void *) &cm->qp_infos[qp_index].qp_index)) {
IBOFFLOAD_ERROR(("Failed to allocate frags_free"));
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/* Create list of IB HCA that have active port */
static int iboffload_load_devices(void)
{
int num_devs = 0, i;
mca_bcol_iboffload_device_t *device = NULL;
mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
IBOFFLOAD_VERBOSE(10, ("Entering to iboffload_load_devices"));
/* Get list of devices */
/*cm->ib_devs = ibv_get_device_list_compat(&num_devs);*/
cm->ib_devs = ompi_ibv_get_device_list(&num_devs);
if (0 == num_devs || NULL == cm->ib_devs) {
IBOFFLOAD_ERROR(("No IB devices found"));
/* No hca error*/
orte_show_help("help-mpi-btl-openib.txt", "no-nics", true);
return OMPI_ERROR;
}
cm->num_devs = num_devs;
for (i = 0; i < num_devs; i++) {
device = OBJ_NEW(mca_bcol_iboffload_device_t);
if (NULL != device) {
opal_pointer_array_set_item(&cm->devices, i, (void *) device);
device->dev.ib_dev = cm->ib_devs[i];
IBOFFLOAD_VERBOSE(10, ("Device %s with index %d was appended.\n",
ibv_get_device_name(device->dev.ib_dev), i));
}
}
if (0 == opal_pointer_array_get_size(&cm->devices)) {
/* No relevand devices were found, return error */
IBOFFLOAD_ERROR(("No active devices found.\n"));
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
/*
* Open the component
*/
static int iboffload_open(void)
{
int rc;
/* local variables */
mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
IBOFFLOAD_VERBOSE(10, ("Open Iboffload component.\n"));
cm->super.priority = 100;
cm->super.n_net_contexts = 0;
cm->super.network_contexts = NULL;
OBJ_CONSTRUCT(&cm->recv_wrs.lock, opal_mutex_t);
/* construct lists */
OBJ_CONSTRUCT(&cm->devices, opal_pointer_array_t);
rc = opal_pointer_array_init(&cm->devices, 10, INT_MAX, 10);
if (OMPI_SUCCESS != rc) {
goto close_device;
}
/* load mca parametres */
rc = mca_bcol_iboffload_register_params();
if (OMPI_SUCCESS != rc) {
goto close_device;
}
/* Register the progress function */
rc = opal_progress_register(mca_bcol_iboffload_component_progress);
if (OMPI_SUCCESS != rc) {
IBOFFLOAD_ERROR(("Failed to register the progress function"
" for iboffload component.\n"));
goto close_device;
}
map_ompi_to_ib_dtype();
map_ompi_to_ib_op_type();
/* The init_done set to true on first component usage */
cm->init_done = false;
return OMPI_SUCCESS;
close_device:
OBJ_DESTRUCT(&cm->devices);
OBJ_DESTRUCT(&cm->recv_wrs.lock);
return rc;
}
static inline __opal_attribute_always_inline__ int
handle_collfrag_done(mca_bcol_iboffload_collfrag_t *coll_frag,
mca_bcol_iboffload_collreq_t *coll_request,
mca_bcol_iboffload_device_t *device)
{
int rc;
if (COLLFRAG_IS_DONE(coll_frag)) {
IBOFFLOAD_VERBOSE(10, ("Coll frag - %p already done.\n", coll_frag));
coll_request->n_frag_net_complete++;
IBOFFLOAD_VERBOSE(10, ("Free tasks resourse.\n"));
/* Check if we are done with this coll_frag and release resources if so. */
rc = mca_bcol_iboffload_free_tasks_frags_resources(coll_frag, device->frags_free);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_ERROR(("mca_bcol_iboffload_free_tasks_frags_resources FAILED"));
fatal_error("Failed to mca_bcol_iboffload_free_tasks_frags_resources\n");
return -1;
}
BCOL_IBOFFLOAD_MQ_RETURN_CREDITS(coll_request->module, coll_frag->mq_index, coll_frag->mq_credits);
RELEASE_COLLFRAG(coll_frag);
PROGRESS_PENDING_COLLFRAG(coll_frag);
IBOFFLOAD_VERBOSE(10, ("Alg %d: user_handle_freed - %d, n_frag_mpi_complete - %d, "
"n_fragments- %d, n_frag_net_complete - %d, n_fragments - %d.\n",
coll_frag->alg,
coll_request->user_handle_freed,
coll_request->n_frag_mpi_complete,
coll_request->n_fragments,
coll_request->n_frag_net_complete,
coll_request->n_fragments));
/* check for full message completion */
if (COLLREQ_IS_DONE(coll_request)) {
IBOFFLOAD_VERBOSE(10, ("Coll request already done.\n"));
RELEASE_COLLREQ(coll_request);
}
}
IBOFFLOAD_VERBOSE(10, ("Exit with success.\n"));
return 0;
}
/*
* Close the component
*/
static int iboffload_close(void)
{
int rc;
mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
IBOFFLOAD_VERBOSE(10, ("Destroy component free lists.\n"));
if (true == cm->init_done) {
OBJ_DESTRUCT(&cm->tasks_free);
OBJ_DESTRUCT(&cm->collreqs_free);
OBJ_DESTRUCT(&cm->collfrags_free);
OBJ_DESTRUCT(&cm->calc_tasks_free);
}
/* Unregister the progress function */
rc = opal_progress_unregister(mca_bcol_iboffload_component_progress);
if (OMPI_SUCCESS != rc) {
IBOFFLOAD_ERROR(("Failed to unregister the progress function"
" for iboffload component.\n"));
}
rc = iboffload_release_devices();
if (OMPI_SUCCESS != rc) {
return rc;
}
if (NULL != cm->receive_queues) {
free(cm->receive_queues);
}
OBJ_DESTRUCT(&cm->recv_wrs.lock);
IBOFFLOAD_VERBOSE(10, ("The component closed.\n"));
return OMPI_SUCCESS;
}
static int mca_bcol_iboffload_dummy_frag_qp_prepost(
mca_bcol_iboffload_endpoint_t *endpoint,
int qp_index, int num_to_prepost)
{
struct ibv_recv_wr *recv_wr, *recv_bad;
int ret, num_preposted = 0, start_wr_index;
mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
mca_bcol_iboffload_recv_wr_manager *recv_wrs = &cm->recv_wrs;
IBOFFLOAD_VERBOSE(10, ("Recv prepost call: endpoint %p, to prepost %d",
(void *) endpoint, num_to_prepost));
if (OPAL_UNLIKELY(0 == num_to_prepost)) {
IBOFFLOAD_VERBOSE(10, ("num_to_prepost = 0, return immediate"));
return OMPI_SUCCESS;
}
/* make sure that we do not overrun number of rd_wqe */
if (num_to_prepost > endpoint->qps[qp_index].rd_wqe) {
IBOFFLOAD_VERBOSE(10, ("Reset num_to_prepost = %d, to rd_wqe = %d",
num_to_prepost, endpoint->qps[qp_index].rd_wqe));
num_to_prepost = endpoint->qps[qp_index].rd_wqe;
}
OPAL_THREAD_LOCK(&recv_wrs->lock);
/* calculate start index in array
* of pre-allocated work requests */
start_wr_index = cm->qp_infos[qp_index].rd_num - num_to_prepost;
recv_wr = &recv_wrs->recv_work_requests[qp_index][start_wr_index];
IBOFFLOAD_VERBOSE(10, ("Endpoint %p, qp_index - %d, to_porepost %d, "
"start index of WRs - %d, rd_wqe - %d",
(void *) endpoint, qp_index, num_to_prepost,
start_wr_index, endpoint->qps[qp_index].rd_wqe));
while (num_preposted < num_to_prepost) {
/* prepost the special barrier frag to recv queue */
struct ibv_sge *dummy_sg_entry =
&endpoint->iboffload_module->device->dummy_frags[qp_index].sg_entry;
recv_wr[num_preposted].sg_list = dummy_sg_entry;
++num_preposted;
}
if (OPAL_LIKELY(num_preposted > 0)) {
/* Set the tail */
recv_wr[num_preposted - 1].next = NULL;
/* post the list of recvs */
ret = ibv_post_recv(endpoint->qps[qp_index].qp->lcl_qp, recv_wr, &recv_bad);
if (OPAL_UNLIKELY(0 != ret)) {
IBOFFLOAD_ERROR(("ibv_post_recv failed, error: %s [%d], "
"qp_index - %d.\n", strerror(errno), ret, qp_index));
return OMPI_ERROR;
}
/* recover last recv_wr if needed */
if (OPAL_UNLIKELY(num_to_prepost != num_preposted)) {
recv_wr[num_preposted - 1].next = &recv_wr[num_preposted];
}
/* decresing numbers of free recv wqe */
endpoint->qps[qp_index].rd_wqe -= num_preposted;
}
OPAL_THREAD_UNLOCK(&recv_wrs->lock);
IBOFFLOAD_VERBOSE(10, ("Endpoint %p, to_porepost %d, num preposted - %d, qp_index - %d",
(void *) endpoint, num_to_prepost, num_preposted, qp_index));
return OMPI_SUCCESS;
}
/*
* Receive prepost:
* return values:
* 0 - no prepost was done
* -1 - fatal error during prepost
* other value - number preposted elements
*/
static int mca_bcol_iboffload_frag_reg_qp_prepost(
mca_bcol_iboffload_endpoint_t *endpoint,
int qp_index, int num_to_prepost)
{
ompi_free_list_item_t *item;
mca_bcol_iboffload_frag_t *frag;
struct ibv_recv_wr *recv_wr, *recv_bad;
int i, ret, num_preposted = 0, start_wr_index;
mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
mca_bcol_iboffload_device_t *device = endpoint->iboffload_module->device;
opal_list_t *preposted = &(endpoint->qps[qp_index].preposted_frags);
mca_bcol_iboffload_recv_wr_manager *recv_wrs = &cm->recv_wrs;
IBOFFLOAD_VERBOSE(10, ("Recv prepost call: endpoint %p, to prepost %d",
(void *) endpoint, num_to_prepost));
if (OPAL_UNLIKELY(0 == num_to_prepost)) {
IBOFFLOAD_VERBOSE(10, ("num_to_prepost = 0, return immediate"));
return OMPI_SUCCESS;
}
/* make sure that we do not overrun number of rd_wqe */
if (num_to_prepost > endpoint->qps[qp_index].rd_wqe) {
IBOFFLOAD_VERBOSE(10, ("Reset num_to_prepost = %d, to rd_wqe = %d",
num_to_prepost, endpoint->qps[qp_index].rd_wqe));
num_to_prepost = endpoint->qps[qp_index].rd_wqe;
}
OPAL_THREAD_LOCK(&recv_wrs->lock);
/* calculate start index in array
* of pre-allocated work requests */
start_wr_index = cm->qp_infos[qp_index].rd_num - num_to_prepost;
recv_wr = &recv_wrs->recv_work_requests[qp_index][start_wr_index];
IBOFFLOAD_VERBOSE(10, ("Endpoint %p, qp_index - %d, to_porepost %d, "
"start index of WRs - %d, rd_wqe - %d",
(void *) endpoint, qp_index, num_to_prepost,
start_wr_index, endpoint->qps[qp_index].rd_wqe));
while (num_preposted < num_to_prepost) {
/* put the item on list of preposted */
OMPI_FREE_LIST_GET_MT(&device->frags_free[qp_index], item);
if (OPAL_UNLIKELY(NULL == item)) {
break;
}
frag = (mca_bcol_iboffload_frag_t *) item;
opal_list_append(preposted, (opal_list_item_t *) item);
recv_wr[num_preposted].sg_list = &frag->sg_entry;
/* TODO Pasha - fix it later */ /* Vasily: Is it right place to take a size value ???? */
frag->sg_entry.length = cm->qp_infos[qp_index].size;
++num_preposted;
}
if (OPAL_LIKELY(num_preposted > 0)) {
/* Set the tail */
recv_wr[num_preposted - 1].next = NULL;
/* post the list of recvs */
ret = ibv_post_recv(endpoint->qps[qp_index].qp->lcl_qp, recv_wr, &recv_bad);
if (OPAL_UNLIKELY(0 != ret)) {
IBOFFLOAD_ERROR(("ibv_post_recv failed (%s), error: %s [%d], "
"qp_index - %d.\n",
ibv_get_device_name(device->dev.ib_dev),
strerror(errno), ret, qp_index));
/* Return allocated frags */
for (i = 0; i < num_preposted; i++) {
OMPI_FREE_LIST_RETURN_MT(&device->frags_free[qp_index],
(ompi_free_list_item_t *)
opal_list_remove_last(preposted));
}
return OMPI_ERROR;
}
/* recover last recv_wr if needed */
if (OPAL_UNLIKELY(num_to_prepost != num_preposted)) {
recv_wr[num_preposted - 1].next = &recv_wr[num_preposted];
}
/* decresing numbers of free recv wqe */
endpoint->qps[qp_index].rd_wqe -= num_preposted;
}
OPAL_THREAD_UNLOCK(&recv_wrs->lock);
IBOFFLOAD_VERBOSE(10, ("Endpoint %p, to_porepost %d, num preposted - %d",
(void *) endpoint, num_to_prepost, num_preposted));
return OMPI_SUCCESS;
}
static int mca_bcol_iboffload_bcast_scatter_allgather_extra_exec(mca_bcol_iboffload_module_t *iboffload_module,
mca_bcol_iboffload_collreq_t *coll_request)
{
netpatterns_pair_exchange_node_t *recursive_doubling_tree =
&iboffload_module->recursive_doubling_tree;
int rc, dst;
int count = coll_request->count * coll_request->dtype->super.size;
int my_group_index = iboffload_module->ibnet->super.my_index;
struct mqe_task *last_send = NULL,
*last_wait = NULL;
mca_bcol_iboffload_collfrag_t *coll_fragment = &coll_request->first_collfrag;
if (OPAL_UNLIKELY(!iboffload_module->connection_status[RECURSIVE_DOUBLING_TREE_BCAST])) {
bcol_iboffload_setup_binomial_connection(iboffload_module);
}
/* register memory in mpool/rcache */
rc = mca_bcol_iboffload_prepare_buffer(coll_request->buffer_info[SBUF].buf, count,
&coll_request->buffer_info[SBUF].iboffload_reg, iboffload_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_ERROR(("Cannot register memory: "
"addr - %p, %d bytes.\n",
coll_request->buffer_info[SBUF].buf, count));
return OMPI_ERROR;
}
coll_request->buffer_info[SBUF].lkey = coll_request->buffer_info[SBUF].iboffload_reg->mr->lkey;
/* it is estimated mq consumption... */
if (OPAL_UNLIKELY(false == BCOL_IBOFFLOAD_MQ_HAVE_CREDITS(
iboffload_module, coll_fragment->mq_index, coll_fragment->mq_credits) ||
false == opal_list_is_empty(&iboffload_module->collfrag_pending))) {
IBOFFLOAD_VERBOSE(10, ("There are not enough credits on MQ.\n"));
goto out_of_resources;
}
coll_fragment->tail_next = &coll_fragment->to_post;
/* send or recv the data */
if (coll_request->root == my_group_index) {
IBOFFLOAD_VERBOSE(10, ("I'm root of the data %d", iboffload_module->power_of_2));
/* send the all data to your extra peer */
dst = recursive_doubling_tree->rank_extra_source;
rc = mca_bcol_iboffload_recv_rtr_setup(
&last_wait, dst, iboffload_module, coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to"
" mca_bcol_iboffload_recv_rtr_setup"));
return OMPI_ERROR;
}
rc = mca_bcol_iboffload_send_large_buff_setup(
&last_send, SBUF, 0, count, dst,
iboffload_module, coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to"
" mca_bcol_iboffload_send_large_buff_setup"));
return OMPI_ERROR;
}
} else {
/* Not root case */
dst = recursive_doubling_tree->rank_extra_source;
rc = mca_bcol_iboffload_send_rtr_setup(&last_send,
dst, iboffload_module,
coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to setup send rtr"));
return OMPI_ERROR;
}
rc = mca_bcol_iboffload_recv_large_buff_setup(&last_wait,
SBUF, 0, count, dst,
iboffload_module, coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to setup data receive"));
return OMPI_ERROR;
}
}
IBOFFLOAD_VERBOSE(10, ("Fill in the the rest of the coll_fragment.\n"));
/* end of list */
*coll_fragment->tail_next = NULL;
/* finish initializing full message descriptor */
coll_request->n_fragments += 1;
coll_request->n_frags_sent += 1;
if (NULL != last_wait) {
last_wait->flags |= MQE_WR_FLAG_SIGNAL;
coll_fragment->signal_task_wr_id = last_wait->wr_id;
last_wait->wr_id = (uint64_t) (uintptr_t) coll_fragment;
} else {
last_send->flags |= MQE_WR_FLAG_SIGNAL;
coll_fragment->signal_task_wr_id = last_send->wr_id;
last_send->wr_id = (uint64_t) (uintptr_t) coll_fragment;
}
/* post the mwr */
rc = mca_bcol_iboffload_post_mqe_tasks(iboffload_module, coll_fragment->to_post);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("MQE task posting failing.\n"));
/* Note: need to clean up */
return rc;
}
MCA_BCOL_UPDATE_ORDER_COUNTER(&iboffload_module->super, coll_request->order_info);
IBOFFLOAD_VERBOSE(10, ("Return success.\n"));
return BCOL_FN_STARTED;
out_of_resources:
/* Release all resources */
IBOFFLOAD_VERBOSE(10, ("Barrier, adding collfrag to collfrag_pending.\n"));
rc =
mca_bcol_iboffload_free_resources_and_move_to_pending(coll_fragment, iboffload_module);
return (OMPI_SUCCESS != rc) ? BCOL_FN_NOT_STARTED : BCOL_FN_STARTED;
}
static int mca_bcol_iboffload_bcast_scatter_allgather_exec(mca_bcol_iboffload_module_t *iboffload_module,
mca_bcol_iboffload_collreq_t *coll_request)
{
netpatterns_pair_exchange_node_t *recursive_doubling_tree =
&iboffload_module->recursive_doubling_tree;
int rc,
dst,
group_src, power_of_2_distance,
recv_count;
size_t offset;
int count = coll_request->count * coll_request->dtype->super.size;
int my_group_index = iboffload_module->ibnet->super.my_index;
size_t base_block_size =
(count + iboffload_module->power_of_2_ranks - 1) /
iboffload_module->power_of_2_ranks;
struct mqe_task *last_send = NULL,
*last_wait = NULL;
mca_bcol_iboffload_collfrag_t *coll_fragment = &coll_request->first_collfrag;
if (OPAL_UNLIKELY(!iboffload_module->connection_status[RECURSIVE_DOUBLING_TREE_BCAST])) {
bcol_iboffload_setup_binomial_connection(iboffload_module);
}
/* register memory in mpool/rcache */
rc = mca_bcol_iboffload_prepare_buffer(coll_request->buffer_info[SBUF].buf, count,
&coll_request->buffer_info[SBUF].iboffload_reg, iboffload_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_ERROR(("Cannot register memory: "
"addr - %p, %d bytes.\n",
coll_request->buffer_info[SBUF].buf, count));
return OMPI_ERROR;
}
coll_request->buffer_info[SBUF].lkey = coll_request->buffer_info[SBUF].iboffload_reg->mr->lkey;
/* it is estimated mq consumption... */
if (OPAL_UNLIKELY(false == BCOL_IBOFFLOAD_MQ_HAVE_CREDITS(
iboffload_module, coll_fragment->mq_index, coll_fragment->mq_credits) ||
false == opal_list_is_empty(&iboffload_module->collfrag_pending))) {
IBOFFLOAD_VERBOSE(10, ("There are not enough credits on MQ.\n"));
goto out_of_resources;
}
coll_fragment->tail_next = &coll_fragment->to_post;
if (coll_request->root == my_group_index) {
IBOFFLOAD_VERBOSE(10, ("I'm root of the data %d %d",
iboffload_module->power_of_2, recursive_doubling_tree->n_extra_sources ));
/* for proxy we have little bit more work to do */
if (recursive_doubling_tree->n_extra_sources > 0) {
/* send the all data to your extra peer */
dst = recursive_doubling_tree->rank_extra_source;
rc = mca_bcol_iboffload_recv_rtr_setup(
&last_wait, dst, iboffload_module, coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to"
" mca_bcol_iboffload_recv_rtr_setup"));
return OMPI_ERROR;
}
rc = mca_bcol_iboffload_send_large_buff_setup(
&last_send, SBUF, 0, count, dst,
iboffload_module, coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to"
" mca_bcol_iboffload_send_large_buff_setup"));
return OMPI_ERROR;
}
}
power_of_2_distance = iboffload_module->power_of_2;
BINOMIAL_SCATTER(iboffload_module, coll_fragment,
last_wait, last_send, power_of_2_distance - 1,
my_group_index, base_block_size, count
);
/* EXIT OR GO TO Gather */
goto GATHER;
}
/* prepare and post recv operation */
group_src = bcol_iboffload_binomial_root_to_src(coll_request->root,
my_group_index, iboffload_module->power_of_2_ranks,
iboffload_module->group_size, &power_of_2_distance);
IBOFFLOAD_VERBOSE(10, ("SRC %d DIST %d ranks %d gsize %d root %d my rank %d",
group_src, power_of_2_distance, iboffload_module->power_of_2_ranks,
iboffload_module->group_size,
coll_request->root, my_group_index));
assert(group_src >= 0);
if (0 > power_of_2_distance) {
/* the rank is virtual root for this group, receive the data
and scatter gather as root */
power_of_2_distance =
iboffload_module->power_of_2;
offset = 0;
recv_count = count;
IBOFFLOAD_VERBOSE(10, ("Virtual root %d , set mask to %d",
my_group_index, power_of_2_distance));
} else {
int my_left_boundary_rank;
int delta;
recv_count = base_block_size * (1 << power_of_2_distance); /* we may receive larger data */
my_left_boundary_rank = my_group_index & ((~(int)0) << power_of_2_distance );
offset = (size_t) (base_block_size * my_left_boundary_rank);
delta = count - offset;
if (OPAL_UNLIKELY(delta <= 0)) {
/* no data to recv */
goto GATHER;
} else {
recv_count = (delta < recv_count) ? delta : recv_count;
}
IBOFFLOAD_VERBOSE(10, ("Recv data set mask to %d",
power_of_2_distance));
}
IBOFFLOAD_VERBOSE(10, ("Bcast, receive data from %d[%d], count %d, offset %d",
group_src, recv_count, offset));
/* Receive data to user buffer */
rc = mca_bcol_iboffload_send_rtr_setup(&last_send,
group_src, iboffload_module,
coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to setup send rtr"));
return OMPI_ERROR;
}
rc = mca_bcol_iboffload_recv_large_buff_setup(&last_wait,
SBUF, offset, recv_count, group_src,
iboffload_module, coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to setup data receive"));
return OMPI_ERROR;
}
BINOMIAL_SCATTER(iboffload_module, coll_fragment,
last_wait, last_send, power_of_2_distance - 1,
my_group_index, base_block_size, count);
GATHER:
rc = bcol_iboffload_bcast_binomial_gather(iboffload_module,
&last_send, &last_wait, coll_fragment,
count, base_block_size, power_of_2_distance);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to setup gather. Return %d", rc));
return rc;
}
if (recursive_doubling_tree->n_extra_sources > 0 &&
iboffload_module->power_of_2 != power_of_2_distance) {
dst = recursive_doubling_tree->rank_extra_source;
rc = mca_bcol_iboffload_recv_rtr_setup(
&last_wait, dst, iboffload_module, coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to"
" mca_bcol_iboffload_recv_rtr_setup"));
return OMPI_ERROR;
}
rc = mca_bcol_iboffload_send_large_buff_setup(
&last_send, SBUF, 0, count, dst,
iboffload_module, coll_fragment);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("Failed to"
" mca_bcol_iboffload_send_large_buff_setup"));
return OMPI_ERROR;
}
}
IBOFFLOAD_VERBOSE(10, ("Fill in the the rest of the coll_fragment.\n"));
/* end of list */
*coll_fragment->tail_next = NULL;
/* finish initializing full message descriptor */
coll_request->n_fragments += 1;
coll_request->n_frags_sent += 1;
if (NULL != last_wait) {
last_wait->flags |= MQE_WR_FLAG_SIGNAL;
coll_fragment->signal_task_wr_id = last_wait->wr_id;
last_wait->wr_id = (uint64_t) (uintptr_t) coll_fragment;
} else {
last_send->flags |= MQE_WR_FLAG_SIGNAL;
coll_fragment->signal_task_wr_id = last_send->wr_id;
last_send->wr_id = (uint64_t) (uintptr_t) coll_fragment;
}
/* post the mwr */
rc = mca_bcol_iboffload_post_mqe_tasks(iboffload_module, coll_fragment->to_post);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
IBOFFLOAD_VERBOSE(10, ("MQE task posting failing.\n"));
/* Note: need to clean up */
return rc;
}
MCA_BCOL_UPDATE_ORDER_COUNTER(&iboffload_module->super, coll_request->order_info);
IBOFFLOAD_VERBOSE(10, ("Return success.\n"));
return BCOL_FN_STARTED;
out_of_resources:
/* Release all resources */
IBOFFLOAD_VERBOSE(10, ("Barrier, adding collfrag to collfrag_pending.\n"));
rc =
mca_bcol_iboffload_free_resources_and_move_to_pending(coll_fragment, iboffload_module);
return (OMPI_SUCCESS != rc) ? BCOL_FN_NOT_STARTED : BCOL_FN_STARTED;
}
static int mca_bcol_iboffload_bcast_init(
bcol_function_args_t *fn_arguments,
mca_bcol_iboffload_module_t *iboffload_module,
mca_bcol_iboffload_collreq_t **coll_request,
bool if_bcol_last, int mq_credits,
collective_message_progress_function progress_fn)
{
ompi_free_list_item_t *item;
mca_bcol_iboffload_collfrag_t *coll_fragment;
mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
int my_group_index = iboffload_module->super.sbgp_partner_module->my_index;
OMPI_FREE_LIST_WAIT_MT(&cm->collreqs_free, item);
if (OPAL_UNLIKELY(NULL == item)) {
IBOFFLOAD_ERROR(("Wait for free list failed.\n"));
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* setup call request */
(*coll_request) = (mca_bcol_iboffload_collreq_t *) item;
(*coll_request)->n_fragments = 0;
(*coll_request)->n_frags_sent = 0;
(*coll_request)->n_frag_mpi_complete = 0;
(*coll_request)->n_frag_net_complete = 0;
(*coll_request)->if_bcol_last = if_bcol_last;
(*coll_request)->ml_buffer_index = fn_arguments->buffer_index;
(*coll_request)->completion_cb_fn = NULL;
(*coll_request)->buffer_info[SBUF].buf = (void *) (
(unsigned char *)fn_arguments->sbuf +
fn_arguments->sbuf_offset);
(*coll_request)->buffer_info[SBUF].offset = fn_arguments->sbuf_offset;
(*coll_request)->buffer_info[RBUF].offset = fn_arguments->rbuf_offset;
(*coll_request)->dtype = fn_arguments->dtype;
(*coll_request)->count = fn_arguments->count;
(*coll_request)->module = iboffload_module;
/* TODO Pasha: we need it for pending quque. Set it later. */
(*coll_request)->progress_fn = progress_fn;
/* TODO Pasha: fix it later */
(*coll_request)->qp_index = MCA_BCOL_IBOFFLOAD_QP_REGULAR;
(*coll_request)->order_info = &fn_arguments->order_info;
coll_fragment = &((*coll_request)->first_collfrag);
mca_bcol_iboffload_collfrag_init(coll_fragment);
/** Vasily ????? */
/* mq_credits = (*coll_request)->total_tasks_num; */
coll_fragment->mq_credits = mq_credits;
coll_fragment->mq_index = COLL_MQ;
/* Pasha: just set it to zero */
coll_fragment->last_wait_num = 0;
coll_fragment->alg = -2; /* used only for debug */
/*
if (my_rank == algthm_ptr->root) {
coll_fragment->last_wait_num = 0;
} else {
coll_fragment->last_wait_num = algth_lst->last_wait_num;
}
*/
/* Pasha: we have nothing to unpack */
coll_fragment->unpack_size = 0;
/* coll_fragment->unpack_size = pack_len; */
/* coll_fragment->alg = RECURSIVE_DOUBLING_TREE_BCAST; */
/* set pointers for (coll frag) <-> (coll full request) */
(*coll_request)->user_handle_freed = false;
fn_arguments->bcol_opaque_data = (void *) (*coll_request);
if (true == fn_arguments->root_flag) {
(*coll_request)->root = my_group_index;
} else {
(*coll_request)->root = fn_arguments->root_route->rank;
}
MCA_BCOL_IBOFFLOAD_SET_COLL_REQ_LINKS((*coll_request), coll_fragment);
return OMPI_SUCCESS;
}
static int progress_one_device(mca_bcol_iboffload_device_t *device)
{
int ne, rc, count = 0;
mca_bcol_iboffload_collfrag_t *coll_frag;
mca_bcol_iboffload_collreq_t *coll_request;
struct ibv_wc wc;
memset(&wc, 0, sizeof(struct ibv_wc));
/*
* poll for collective completion - does not mean resources can
* be freed, as incomplete network level sends may still be pending
*/
/* Poll for completion on completion on wait MQEs */
if(0 != (ne = ibv_poll_cq(device->ib_mq_cq, 1, &wc))) {
do {
if (OPAL_UNLIKELY(0 > ne)) {
IBOFFLOAD_ERROR(("Device %s: "
"failed to poll MQ completion queue\n",
ibv_get_device_name(device->dev.ib_dev)));
fatal_error("failed to poll MQ completion queue\n");
return count;
}
if (OPAL_UNLIKELY(IBV_WC_SUCCESS != wc.status)) {
IBOFFLOAD_ERROR(("Device %s: "
"the completion with error on wait was gotten, status %d, opcode %d, "
"vendor_err 0x%x, qp %x, id 0x%x\n", ibv_get_device_name(device->dev.ib_dev),
wc.status, wc.opcode, wc.vendor_err, wc.qp_num, wc.wr_id));
fatal_error("wc.status \n");
return count;
}
IBOFFLOAD_VERBOSE(10, ("The MQ completion was polled.\n"));
++count;
/* get pointer to mca_bcol_iboffload_collfrag_t */
coll_frag = (mca_bcol_iboffload_collfrag_t*)
(uint64_t) (uintptr_t) wc.wr_id;
/* Only last MQ task of collective frag
sends completion signal, so if we got it =>
all MQEs were done. */
coll_frag->complete = true;
IBOFFLOAD_VERBOSE(10, ("MQ completion for algorithm %d coll_frag_addr %p ml buffer index %d",
coll_frag->alg, (void *)coll_frag, coll_frag->coll_full_req->ml_buffer_index));
/* full request descriptor */
coll_request = coll_frag->coll_full_req;
coll_request->n_frag_mpi_complete++;
/*
* at this stage all receives have been completed, so
* unpack the data to user buffer, the resources will be released when we will done with all
* element in the task list
*/
if (NULL != coll_request->completion_cb_fn) {
if (OMPI_SUCCESS !=
coll_request->completion_cb_fn(coll_frag)) {
fatal_error("coll_request->completion_cb_fn\n");
return count;
}
}
if (coll_request->n_frag_mpi_complete ==
coll_request->n_fragments) {
coll_request->super.req_complete = true;
opal_condition_broadcast(&ompi_request_cond);
IBOFFLOAD_VERBOSE(10, ("After opal_condition_broadcast.\n"));
}
rc = handle_collfrag_done(coll_frag, coll_request, device);
if (0 != rc) {
return count;
}
} while(0 != (ne = ibv_poll_cq(device->ib_mq_cq, 1, &wc)));
return count;
}
/* poll the send completion queue */
do {
ne = ibv_poll_cq(device->ib_cq, 1, &wc);
if (0 < ne) {
if (OPAL_UNLIKELY(IBV_WC_SUCCESS != wc.status)) {
IBOFFLOAD_ERROR(("Device %s, "
"the completion with error on send was gotten, status %d, opcode %d, "
"vendor_err 0x%x, qp %x, id 0x%x\n", ibv_get_device_name(device->dev.ib_dev),
wc.status, wc.opcode, wc.vendor_err, wc.qp_num, wc.wr_id));
#if OPAL_ENABLE_DEBUG
{
mca_bcol_iboffload_module_t *iboffload;
int i, qp_index, num_qps = mca_bcol_iboffload_component.num_qps;
coll_frag = (mca_bcol_iboffload_collfrag_t*)
(uint64_t) (uintptr_t) wc.wr_id;
iboffload = coll_frag->coll_full_req->module;
for (i = 0; i < iboffload->num_endpoints; ++i) {
mca_bcol_iboffload_endpoint_t *ep = iboffload->endpoints[i];
for (qp_index = 0; qp_index < num_qps; ++qp_index) {
if (NULL != ep->qps[qp_index].qp->lcl_qp &&
wc.qp_num == ep->qps[qp_index].qp->lcl_qp->qp_num) {
IBOFFLOAD_ERROR(("Module - %p, coll_frag - %p, "
"destination %d, qp index - %d.",
iboffload, coll_frag, i, qp_index));
}
}
}
}
#endif
fatal_error("Failed to ibv_poll_cq\n");
return count;
}
++count;
/* get pointer to mca_bcol_iboffload_collfrag_t */
coll_frag = (mca_bcol_iboffload_collfrag_t*)
(uint64_t) (uintptr_t) wc.wr_id;
/* update the number of completed sends */
coll_frag->n_sends_completed++;
IBOFFLOAD_VERBOSE(10, ("Send CQ completion for algorithm %d coll_frag_addr %p ml buffer index %d",
coll_frag->alg, (void *)coll_frag, coll_frag->coll_full_req->ml_buffer_index));
IBOFFLOAD_VERBOSE(10, ("Alg %d coll_frag_addr %p: n_sends_completed - %d, n_sends - %d.\n",
coll_frag->alg, (void *)coll_frag,
coll_frag->n_sends_completed,
coll_frag->n_sends));
assert(coll_frag->n_sends_completed <= coll_frag->n_sends);
/* full message descriptor */
coll_request = coll_frag->coll_full_req;
/* check to see if all sends are complete from the network
* perspective */
rc = handle_collfrag_done(coll_frag, coll_request, device);
if (0 != rc) {
return count;
}
} else if (OPAL_UNLIKELY(0 > ne)) {
IBOFFLOAD_ERROR(("Device %s: "
"failed to poll send completion queue\n",
ibv_get_device_name(device->dev.ib_dev)));
fatal_error("failed to poll send completion queue\n");
return count;
}
} while (0 != ne);
return count;
}
static void fatal_error(char *mesg)
{
IBOFFLOAD_ERROR(("FATAL ERROR: %s", mesg));
ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_INTERN, true);
}
static void mca_bcol_iboffload_device_destructor
(mca_bcol_iboffload_device_t *device)
{
int qp_index, num_qps = mca_bcol_iboffload_component.num_qps;
IBOFFLOAD_VERBOSE(10, ("Device %s will be destroyed.\n",
ibv_get_device_name(device->dev.ib_dev)));
if (NULL != device->frags_free) {
for (qp_index = 0; qp_index < num_qps; ++qp_index) {
mca_bcol_iboffload_dealloc_qps_resource_fn_t dealloc_resource =
mca_bcol_iboffload_component.qp_infos[qp_index].dealloc_resource;
if (NULL != dealloc_resource) {
dealloc_resource(qp_index, device);
}
}
free(device->frags_free);
}
if (NULL != device->mpool) {
IBOFFLOAD_VERBOSE(10, ("Mpool destroy - %p.\n", device->mpool));
if (OMPI_SUCCESS != mca_mpool_base_module_destroy(device->mpool)) {
IBOFFLOAD_ERROR(("Device %s, failed to destroy mpool",
ibv_get_device_name(device->dev.ib_dev)));
}
}
if (NULL != device->dummy_reg.mr) {
IBOFFLOAD_VERBOSE(10, ("Dummy memory MR unregister - %p.\n", device->dummy_reg.mr));
if (OMPI_SUCCESS !=
mca_bcol_iboffload_deregister_mr((void *) device, &device->dummy_reg.base)) {
IBOFFLOAD_ERROR(("Device %s: failed to unregister dummy memory MR.",
ibv_get_device_name(device->dev.ib_dev)));
}
}
if (NULL != device->ib_cq) {
if (ibv_destroy_cq(device->ib_cq)) {
IBOFFLOAD_ERROR(("Device %s, failed to destroy CQ, errno says %s",
ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
}
}
if (NULL != device->ib_mq_cq) {
if (ibv_destroy_cq(device->ib_mq_cq)) {
IBOFFLOAD_ERROR(("Device %s, failed to destroy mq CQ, errno says %s",
ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
}
}
/* Release IB PD if we have one */
if (NULL != device->ib_pd) {
if(ibv_dealloc_pd(device->ib_pd)) {
IBOFFLOAD_ERROR(("Device %s, failed to release PD, errno says %s",
ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
}
}
/* close the device */
if (NULL != device->dev.ib_dev_context) {
if (ibv_close_device(device->dev.ib_dev_context)) {
IBOFFLOAD_ERROR(("Device %s "
", failed to close the device, errno says %s",
ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
}
}
/* release memory */
if (NULL != device->ports) {
free(device->ports);
}
}
/*
* Open the component
*/
static int iboffload_open(void)
{
int rc;
/* local variables */
mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
IBOFFLOAD_VERBOSE(10, ("Open Iboffload component.\n"));
(void) mca_bcol_iboffload_verify_params();
cm->super.priority = 100;
cm->super.n_net_contexts = 0;
cm->super.network_contexts = NULL;
OBJ_CONSTRUCT(&cm->recv_wrs.lock, opal_mutex_t);
/* construct lists */
OBJ_CONSTRUCT(&cm->devices, opal_pointer_array_t);
rc = opal_pointer_array_init(&cm->devices, 10, INT_MAX, 10);
if (OMPI_SUCCESS != rc) {
goto close_device;
}
/* Check MCA parameters */
if (0 != (mca_bcol_iboffload_component.exchange_tree_order & (mca_bcol_iboffload_component.exchange_tree_order - 1))) {
IBOFFLOAD_ERROR(("Warning: ibcol_iboffload_exchange_tree_order is %d which is not a power of 2, setting it to 2",
mca_bcol_iboffload_component.exchange_tree_order));
mca_bcol_iboffload_component.exchange_tree_order = 2;
}
/* Pasha: Since we do not have max inline check like in openib,
I will put some dummy check here. All mlnx devices support at least 512b */
if (mca_bcol_iboffload_component.max_inline_data > 512) {
IBOFFLOAD_ERROR(("Warning the inline %d, is to big and unsupported",
mca_bcol_iboffload_component.max_inline_data));
rc = OMPI_ERROR;
goto close_device;
}
/* Register the progress function */
rc = opal_progress_register(mca_bcol_iboffload_component_progress);
if (OMPI_SUCCESS != rc) {
IBOFFLOAD_ERROR(("Failed to register the progress function"
" for iboffload component.\n"));
goto close_device;
}
map_ompi_to_ib_dtype();
map_ompi_to_ib_op_type();
/* The init_done set to true on first component usage */
cm->init_done = false;
return OMPI_SUCCESS;
close_device:
OBJ_DESTRUCT(&cm->devices);
OBJ_DESTRUCT(&cm->recv_wrs.lock);
return rc;
}