/* query to see if the component is available for use, and can
 * satisfy the thread and progress requirements
 */
int mca_bcol_iboffload_init_query(bool enable_progress_threads,
                                  bool enable_mpi_threads)
{
    int rc;
    mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;

    IBOFFLOAD_VERBOSE(10, ("Init Iboffload component.\n"));

    /* Get list of HCAs and ports */
    rc = iboffload_load_devices();
    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
        IBOFFLOAD_ERROR(("Load devices error.\n"));
        goto unload_devices;
    }

    /* Setup the BSRQ QP's based on the final value of
       mca_bcol_iboffload_component.receive_queues. */
    rc = setup_qps();
    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
        IBOFFLOAD_ERROR(("QPs setup error.\n"));
        goto unload_devices;
    }

    cm->super.collm_init_query = mca_bcol_iboffload_dummy_init_query;

    return OMPI_SUCCESS;

    /* done */
unload_devices:
    IBOFFLOAD_ERROR(("Release devices: an error occured.\n"));

    iboffload_release_devices();

    return rc;
}
static int mca_bcol_iboffload_alloc_reg_qp_resource(int qp_index, mca_bcol_iboffload_device_t *device)
{
    int length;
    mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;

    ompi_free_list_t *frags_free = &device->frags_free[qp_index];

    OBJ_CONSTRUCT(frags_free, ompi_free_list_t);
    length = cm->qp_infos[qp_index].size;

    IBOFFLOAD_VERBOSE(10, ("free list len %d\n", length));
    if (OMPI_SUCCESS != ompi_free_list_init_ex_new(frags_free,
                sizeof(mca_bcol_iboffload_frag_t), MCA_IBOFFLOAD_CACHE_LINE_SIZE,
                OBJ_CLASS(mca_bcol_iboffload_frag_t),
                length, cm->buffer_alignment,
                cm->free_list_num,
                cm->free_list_max,
                cm->free_list_inc,
                device->mpool,
                mca_bcol_iboffload_frag_init,
                (void *) &cm->qp_infos[qp_index].qp_index)) {
        IBOFFLOAD_ERROR(("Failed to allocate frags_free"));
        return OMPI_ERROR;
    }

    return OMPI_SUCCESS;
}
/* Create list of IB HCA that have active port */
static int iboffload_load_devices(void)
{
    int num_devs = 0, i;
    mca_bcol_iboffload_device_t *device = NULL;
    mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;

    IBOFFLOAD_VERBOSE(10, ("Entering to iboffload_load_devices"));

    /* Get list of devices */
    /*cm->ib_devs = ibv_get_device_list_compat(&num_devs);*/
    cm->ib_devs = ompi_ibv_get_device_list(&num_devs);
    if (0 == num_devs || NULL == cm->ib_devs) {
        IBOFFLOAD_ERROR(("No IB devices found"));
        /* No hca error*/
        orte_show_help("help-mpi-btl-openib.txt", "no-nics", true);
        return OMPI_ERROR;
    }

    cm->num_devs = num_devs;

    for (i = 0; i < num_devs; i++) {
        device = OBJ_NEW(mca_bcol_iboffload_device_t);
        if (NULL != device) {
            opal_pointer_array_set_item(&cm->devices, i, (void *) device);
            device->dev.ib_dev = cm->ib_devs[i];

            IBOFFLOAD_VERBOSE(10, ("Device %s with index %d was appended.\n",
                                   ibv_get_device_name(device->dev.ib_dev), i));
        }
    }

    if (0 == opal_pointer_array_get_size(&cm->devices)) {
        /* No relevand devices were found, return error */
        IBOFFLOAD_ERROR(("No active devices found.\n"));

        return OMPI_ERROR;
    }

    return OMPI_SUCCESS;
}
/*
 * Open the component
 */
static int iboffload_open(void)
{
    int rc;

    /* local variables */
    mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;

    IBOFFLOAD_VERBOSE(10, ("Open Iboffload component.\n"));

    cm->super.priority = 100;
    cm->super.n_net_contexts = 0;
    cm->super.network_contexts = NULL;

    OBJ_CONSTRUCT(&cm->recv_wrs.lock, opal_mutex_t);

    /* construct lists */
    OBJ_CONSTRUCT(&cm->devices, opal_pointer_array_t);
    rc = opal_pointer_array_init(&cm->devices, 10, INT_MAX, 10);
    if (OMPI_SUCCESS != rc) {
        goto close_device;
    }

    /* load mca parametres */
    rc = mca_bcol_iboffload_register_params();
    if (OMPI_SUCCESS != rc) {
        goto close_device;
    }

    /* Register the progress function */
    rc = opal_progress_register(mca_bcol_iboffload_component_progress);
    if (OMPI_SUCCESS != rc) {
        IBOFFLOAD_ERROR(("Failed to register the progress function"
                         " for iboffload component.\n"));
        goto close_device;
    }

    map_ompi_to_ib_dtype();
    map_ompi_to_ib_op_type();

    /* The init_done set to true on first component usage */
    cm->init_done = false;

    return OMPI_SUCCESS;

close_device:
    OBJ_DESTRUCT(&cm->devices);
    OBJ_DESTRUCT(&cm->recv_wrs.lock);
    return rc;
}
static inline __opal_attribute_always_inline__ int
handle_collfrag_done(mca_bcol_iboffload_collfrag_t *coll_frag,
                     mca_bcol_iboffload_collreq_t *coll_request,
                     mca_bcol_iboffload_device_t *device)
{
    int rc;

    if (COLLFRAG_IS_DONE(coll_frag)) {
        IBOFFLOAD_VERBOSE(10, ("Coll frag - %p already done.\n", coll_frag));

        coll_request->n_frag_net_complete++;
        IBOFFLOAD_VERBOSE(10, ("Free tasks resourse.\n"));
        /* Check if we are done with this coll_frag and release resources if so.  */
        rc = mca_bcol_iboffload_free_tasks_frags_resources(coll_frag, device->frags_free);
        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
            IBOFFLOAD_ERROR(("mca_bcol_iboffload_free_tasks_frags_resources FAILED"));
            fatal_error("Failed to mca_bcol_iboffload_free_tasks_frags_resources\n");
            return -1;
        }

        BCOL_IBOFFLOAD_MQ_RETURN_CREDITS(coll_request->module, coll_frag->mq_index, coll_frag->mq_credits);

        RELEASE_COLLFRAG(coll_frag);

        PROGRESS_PENDING_COLLFRAG(coll_frag);

        IBOFFLOAD_VERBOSE(10, ("Alg %d: user_handle_freed - %d, n_frag_mpi_complete - %d, "
                               "n_fragments- %d, n_frag_net_complete - %d, n_fragments - %d.\n",
                               coll_frag->alg,
                               coll_request->user_handle_freed,
                               coll_request->n_frag_mpi_complete,
                               coll_request->n_fragments,
                               coll_request->n_frag_net_complete,
                               coll_request->n_fragments));

        /* check for full message completion */
        if (COLLREQ_IS_DONE(coll_request)) {
            IBOFFLOAD_VERBOSE(10, ("Coll request already done.\n"));
            RELEASE_COLLREQ(coll_request);
        }
    }

    IBOFFLOAD_VERBOSE(10, ("Exit with success.\n"));

    return 0;
}
/*
 * Close the component
 */
static int iboffload_close(void)
{
    int rc;

    mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;

    IBOFFLOAD_VERBOSE(10, ("Destroy component free lists.\n"));

    if (true == cm->init_done) {
        OBJ_DESTRUCT(&cm->tasks_free);
        OBJ_DESTRUCT(&cm->collreqs_free);
        OBJ_DESTRUCT(&cm->collfrags_free);
        OBJ_DESTRUCT(&cm->calc_tasks_free);
    }

    /* Unregister the progress function */
    rc = opal_progress_unregister(mca_bcol_iboffload_component_progress);
    if (OMPI_SUCCESS != rc) {
        IBOFFLOAD_ERROR(("Failed to unregister the progress function"
                         " for iboffload component.\n"));
    }

    rc = iboffload_release_devices();
    if (OMPI_SUCCESS != rc) {
        return rc;
    }

    if (NULL != cm->receive_queues) {
        free(cm->receive_queues);
    }

    OBJ_DESTRUCT(&cm->recv_wrs.lock);

    IBOFFLOAD_VERBOSE(10, ("The component closed.\n"));

    return OMPI_SUCCESS;
}
static int mca_bcol_iboffload_dummy_frag_qp_prepost(
                mca_bcol_iboffload_endpoint_t *endpoint,
                int qp_index, int num_to_prepost)
{
    struct ibv_recv_wr *recv_wr, *recv_bad;
    int ret, num_preposted = 0, start_wr_index;

    mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
    mca_bcol_iboffload_recv_wr_manager *recv_wrs = &cm->recv_wrs;

    IBOFFLOAD_VERBOSE(10, ("Recv prepost call: endpoint %p, to prepost %d",
                          (void *) endpoint, num_to_prepost));

    if (OPAL_UNLIKELY(0 == num_to_prepost)) {
        IBOFFLOAD_VERBOSE(10, ("num_to_prepost = 0, return immediate"));
        return OMPI_SUCCESS;
    }

    /* make sure that we do not overrun number of rd_wqe */
    if (num_to_prepost > endpoint->qps[qp_index].rd_wqe) {
        IBOFFLOAD_VERBOSE(10, ("Reset num_to_prepost = %d, to rd_wqe = %d",
                                num_to_prepost, endpoint->qps[qp_index].rd_wqe));

        num_to_prepost = endpoint->qps[qp_index].rd_wqe;
    }

    OPAL_THREAD_LOCK(&recv_wrs->lock);

    /* calculate start index in array
     * of pre-allocated work requests */
    start_wr_index = cm->qp_infos[qp_index].rd_num - num_to_prepost;
    recv_wr = &recv_wrs->recv_work_requests[qp_index][start_wr_index];

    IBOFFLOAD_VERBOSE(10, ("Endpoint %p, qp_index - %d, to_porepost %d, "
                           "start index of WRs - %d, rd_wqe - %d",
                           (void *) endpoint, qp_index, num_to_prepost,
                            start_wr_index, endpoint->qps[qp_index].rd_wqe));

    while (num_preposted < num_to_prepost) {
        /* prepost the special barrier frag to recv queue */
        struct ibv_sge *dummy_sg_entry =
                    &endpoint->iboffload_module->device->dummy_frags[qp_index].sg_entry;

        recv_wr[num_preposted].sg_list = dummy_sg_entry;
        ++num_preposted;
    }

    if (OPAL_LIKELY(num_preposted > 0)) {
        /* Set the tail */
        recv_wr[num_preposted - 1].next = NULL;

        /* post the list of recvs */
        ret = ibv_post_recv(endpoint->qps[qp_index].qp->lcl_qp, recv_wr, &recv_bad);
        if (OPAL_UNLIKELY(0 != ret)) {
            IBOFFLOAD_ERROR(("ibv_post_recv failed, error: %s [%d], "
                             "qp_index - %d.\n", strerror(errno), ret, qp_index));

            return OMPI_ERROR;
        }

        /* recover last recv_wr if needed */
        if (OPAL_UNLIKELY(num_to_prepost != num_preposted)) {
            recv_wr[num_preposted - 1].next = &recv_wr[num_preposted];
        }

        /* decresing numbers of free recv wqe */
        endpoint->qps[qp_index].rd_wqe -= num_preposted;
    }

    OPAL_THREAD_UNLOCK(&recv_wrs->lock);

    IBOFFLOAD_VERBOSE(10, ("Endpoint %p, to_porepost %d, num preposted - %d, qp_index - %d",
                          (void *) endpoint, num_to_prepost, num_preposted, qp_index));

    return OMPI_SUCCESS;
}
/*
 * Receive prepost:
 * return values:
 * 0 - no prepost was done
 * -1 - fatal error during prepost
 * other value - number preposted elements
 */
static int mca_bcol_iboffload_frag_reg_qp_prepost(
                mca_bcol_iboffload_endpoint_t *endpoint,
                int qp_index, int num_to_prepost)
{
    ompi_free_list_item_t *item;
    mca_bcol_iboffload_frag_t *frag;

    struct ibv_recv_wr *recv_wr, *recv_bad;
    int i, ret, num_preposted = 0, start_wr_index;

    mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
    mca_bcol_iboffload_device_t *device = endpoint->iboffload_module->device;

    opal_list_t *preposted = &(endpoint->qps[qp_index].preposted_frags);
    mca_bcol_iboffload_recv_wr_manager *recv_wrs = &cm->recv_wrs;

    IBOFFLOAD_VERBOSE(10, ("Recv prepost call: endpoint %p, to prepost %d",
                          (void *) endpoint, num_to_prepost));

    if (OPAL_UNLIKELY(0 == num_to_prepost)) {
        IBOFFLOAD_VERBOSE(10, ("num_to_prepost = 0, return immediate"));
        return OMPI_SUCCESS;
    }

    /* make sure that we do not overrun number of rd_wqe */
    if (num_to_prepost > endpoint->qps[qp_index].rd_wqe) {
        IBOFFLOAD_VERBOSE(10, ("Reset num_to_prepost = %d, to rd_wqe = %d",
                                num_to_prepost, endpoint->qps[qp_index].rd_wqe));

        num_to_prepost = endpoint->qps[qp_index].rd_wqe;
    }

    OPAL_THREAD_LOCK(&recv_wrs->lock);

    /* calculate start index in array
     * of pre-allocated work requests */
    start_wr_index = cm->qp_infos[qp_index].rd_num - num_to_prepost;
    recv_wr = &recv_wrs->recv_work_requests[qp_index][start_wr_index];

    IBOFFLOAD_VERBOSE(10, ("Endpoint %p, qp_index - %d, to_porepost %d, "
                           "start index of WRs - %d, rd_wqe - %d",
                           (void *) endpoint, qp_index, num_to_prepost,
                            start_wr_index, endpoint->qps[qp_index].rd_wqe));

    while (num_preposted < num_to_prepost) {
        /* put the item on list of preposted */
        OMPI_FREE_LIST_GET_MT(&device->frags_free[qp_index], item);
        if (OPAL_UNLIKELY(NULL == item)) {
            break;
        }

        frag = (mca_bcol_iboffload_frag_t *) item;
        opal_list_append(preposted, (opal_list_item_t *) item);

        recv_wr[num_preposted].sg_list = &frag->sg_entry;
        /* TODO Pasha - fix it later */ /* Vasily: Is it right place to take a size value ???? */
        frag->sg_entry.length = cm->qp_infos[qp_index].size;
        ++num_preposted;
    }

    if (OPAL_LIKELY(num_preposted > 0)) {
        /* Set the tail */
        recv_wr[num_preposted - 1].next = NULL;

        /* post the list of recvs */
        ret = ibv_post_recv(endpoint->qps[qp_index].qp->lcl_qp, recv_wr, &recv_bad);
        if (OPAL_UNLIKELY(0 != ret)) {
            IBOFFLOAD_ERROR(("ibv_post_recv failed (%s), error: %s [%d], "
                             "qp_index - %d.\n",
                              ibv_get_device_name(device->dev.ib_dev),
                              strerror(errno), ret, qp_index));

            /* Return allocated frags */
            for (i = 0; i < num_preposted; i++) {
                OMPI_FREE_LIST_RETURN_MT(&device->frags_free[qp_index],
                        (ompi_free_list_item_t *)
                            opal_list_remove_last(preposted));
            }

            return OMPI_ERROR;
        }

        /* recover last recv_wr if needed */
        if (OPAL_UNLIKELY(num_to_prepost != num_preposted)) {
            recv_wr[num_preposted - 1].next = &recv_wr[num_preposted];
        }

        /* decresing numbers of free recv wqe */
        endpoint->qps[qp_index].rd_wqe -= num_preposted;
    }

    OPAL_THREAD_UNLOCK(&recv_wrs->lock);

    IBOFFLOAD_VERBOSE(10, ("Endpoint %p, to_porepost %d, num preposted - %d",
                          (void *) endpoint, num_to_prepost, num_preposted));

    return OMPI_SUCCESS;
}
Exemplo n.º 9
0
static int mca_bcol_iboffload_bcast_scatter_allgather_extra_exec(mca_bcol_iboffload_module_t *iboffload_module,
        mca_bcol_iboffload_collreq_t *coll_request)
{
    netpatterns_pair_exchange_node_t *recursive_doubling_tree =
        &iboffload_module->recursive_doubling_tree;

    int rc, dst;
    int count = coll_request->count * coll_request->dtype->super.size;
    int my_group_index = iboffload_module->ibnet->super.my_index;
    struct mqe_task *last_send = NULL,
                    *last_wait = NULL;
    mca_bcol_iboffload_collfrag_t *coll_fragment = &coll_request->first_collfrag;

    if (OPAL_UNLIKELY(!iboffload_module->connection_status[RECURSIVE_DOUBLING_TREE_BCAST])) {
        bcol_iboffload_setup_binomial_connection(iboffload_module);
    }

    /* register memory in mpool/rcache */
    rc = mca_bcol_iboffload_prepare_buffer(coll_request->buffer_info[SBUF].buf, count,
            &coll_request->buffer_info[SBUF].iboffload_reg, iboffload_module);
    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
        IBOFFLOAD_ERROR(("Cannot register memory: "
                         "addr - %p, %d bytes.\n",
                          coll_request->buffer_info[SBUF].buf, count));
        return OMPI_ERROR;
    }

    coll_request->buffer_info[SBUF].lkey = coll_request->buffer_info[SBUF].iboffload_reg->mr->lkey;

    /* it is estimated mq consumption... */
    if (OPAL_UNLIKELY(false == BCOL_IBOFFLOAD_MQ_HAVE_CREDITS(
                 iboffload_module, coll_fragment->mq_index, coll_fragment->mq_credits) ||
                 false == opal_list_is_empty(&iboffload_module->collfrag_pending))) {
        IBOFFLOAD_VERBOSE(10, ("There are not enough credits on MQ.\n"));
        goto out_of_resources;
    }

    coll_fragment->tail_next = &coll_fragment->to_post;

    /* send or recv the data */

    if (coll_request->root == my_group_index) {
        IBOFFLOAD_VERBOSE(10, ("I'm root of the data %d", iboffload_module->power_of_2));
        /* send the all data to your extra peer */
        dst = recursive_doubling_tree->rank_extra_source;
        rc = mca_bcol_iboffload_recv_rtr_setup(
                &last_wait, dst, iboffload_module, coll_fragment);
        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
            IBOFFLOAD_VERBOSE(10, ("Failed to"
                        " mca_bcol_iboffload_recv_rtr_setup"));
            return OMPI_ERROR;
        }
        rc = mca_bcol_iboffload_send_large_buff_setup(
                &last_send, SBUF, 0, count, dst,
                iboffload_module, coll_fragment);
        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
            IBOFFLOAD_VERBOSE(10, ("Failed to"
                        " mca_bcol_iboffload_send_large_buff_setup"));
            return OMPI_ERROR;
        }
    } else {
        /* Not root case */
        dst = recursive_doubling_tree->rank_extra_source;
        rc = mca_bcol_iboffload_send_rtr_setup(&last_send,
                dst, iboffload_module,
                coll_fragment);
        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
            IBOFFLOAD_VERBOSE(10, ("Failed to setup send rtr"));
            return OMPI_ERROR;
        }

        rc = mca_bcol_iboffload_recv_large_buff_setup(&last_wait,
                SBUF, 0, count, dst,
                iboffload_module, coll_fragment);
        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
            IBOFFLOAD_VERBOSE(10, ("Failed to setup data receive"));
            return OMPI_ERROR;
        }
    }

    IBOFFLOAD_VERBOSE(10, ("Fill in the the rest of the coll_fragment.\n"));

    /* end of list */
    *coll_fragment->tail_next = NULL;

    /* finish initializing full message descriptor */
    coll_request->n_fragments  += 1;
    coll_request->n_frags_sent += 1;

    if (NULL != last_wait) {
        last_wait->flags |= MQE_WR_FLAG_SIGNAL;
        coll_fragment->signal_task_wr_id = last_wait->wr_id;
        last_wait->wr_id = (uint64_t) (uintptr_t) coll_fragment;
    } else {
        last_send->flags |= MQE_WR_FLAG_SIGNAL;
        coll_fragment->signal_task_wr_id = last_send->wr_id;
        last_send->wr_id = (uint64_t) (uintptr_t) coll_fragment;
    }

    /* post the mwr */
    rc = mca_bcol_iboffload_post_mqe_tasks(iboffload_module, coll_fragment->to_post);
    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
        IBOFFLOAD_VERBOSE(10, ("MQE task posting failing.\n"));
        /* Note: need to clean up */
        return rc;
    }

    MCA_BCOL_UPDATE_ORDER_COUNTER(&iboffload_module->super, coll_request->order_info);

    IBOFFLOAD_VERBOSE(10, ("Return success.\n"));

    return BCOL_FN_STARTED;

out_of_resources:
    /* Release all resources */
    IBOFFLOAD_VERBOSE(10, ("Barrier, adding collfrag to collfrag_pending.\n"));
    rc =
        mca_bcol_iboffload_free_resources_and_move_to_pending(coll_fragment, iboffload_module);
    return (OMPI_SUCCESS != rc) ? BCOL_FN_NOT_STARTED : BCOL_FN_STARTED;
}
Exemplo n.º 10
0
static int mca_bcol_iboffload_bcast_scatter_allgather_exec(mca_bcol_iboffload_module_t *iboffload_module,
        mca_bcol_iboffload_collreq_t *coll_request)
{
    netpatterns_pair_exchange_node_t *recursive_doubling_tree =
        &iboffload_module->recursive_doubling_tree;

    int rc,
        dst,
        group_src, power_of_2_distance,
        recv_count;
    size_t offset;
    int count = coll_request->count * coll_request->dtype->super.size;
    int my_group_index = iboffload_module->ibnet->super.my_index;
    size_t base_block_size =
        (count +  iboffload_module->power_of_2_ranks - 1) /
        iboffload_module->power_of_2_ranks;

    struct mqe_task *last_send = NULL,
                    *last_wait = NULL;
    mca_bcol_iboffload_collfrag_t *coll_fragment = &coll_request->first_collfrag;

    if (OPAL_UNLIKELY(!iboffload_module->connection_status[RECURSIVE_DOUBLING_TREE_BCAST])) {
        bcol_iboffload_setup_binomial_connection(iboffload_module);
    }

    /* register memory in mpool/rcache */
    rc = mca_bcol_iboffload_prepare_buffer(coll_request->buffer_info[SBUF].buf, count,
            &coll_request->buffer_info[SBUF].iboffload_reg, iboffload_module);
    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
        IBOFFLOAD_ERROR(("Cannot register memory: "
                         "addr - %p, %d bytes.\n",
                          coll_request->buffer_info[SBUF].buf, count));
        return OMPI_ERROR;
    }

    coll_request->buffer_info[SBUF].lkey = coll_request->buffer_info[SBUF].iboffload_reg->mr->lkey;

    /* it is estimated mq consumption... */
    if (OPAL_UNLIKELY(false == BCOL_IBOFFLOAD_MQ_HAVE_CREDITS(
                 iboffload_module, coll_fragment->mq_index, coll_fragment->mq_credits) ||
                 false == opal_list_is_empty(&iboffload_module->collfrag_pending))) {
        IBOFFLOAD_VERBOSE(10, ("There are not enough credits on MQ.\n"));
        goto out_of_resources;
    }

    coll_fragment->tail_next = &coll_fragment->to_post;

    if (coll_request->root == my_group_index) {
        IBOFFLOAD_VERBOSE(10, ("I'm root of the data %d %d",
                    iboffload_module->power_of_2, recursive_doubling_tree->n_extra_sources ));
        /* for proxy we have little bit more work to do */
        if (recursive_doubling_tree->n_extra_sources > 0) {
            /* send the all data to your extra peer */
            dst = recursive_doubling_tree->rank_extra_source;
            rc = mca_bcol_iboffload_recv_rtr_setup(
                    &last_wait, dst, iboffload_module, coll_fragment);
            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                IBOFFLOAD_VERBOSE(10, ("Failed to"
                            " mca_bcol_iboffload_recv_rtr_setup"));
                return OMPI_ERROR;
            }
            rc = mca_bcol_iboffload_send_large_buff_setup(
                    &last_send, SBUF, 0, count, dst,
                    iboffload_module, coll_fragment);
            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
                IBOFFLOAD_VERBOSE(10, ("Failed to"
                            " mca_bcol_iboffload_send_large_buff_setup"));
                return OMPI_ERROR;
            }
        }
        power_of_2_distance = iboffload_module->power_of_2;

        BINOMIAL_SCATTER(iboffload_module, coll_fragment,
                last_wait, last_send,  power_of_2_distance - 1,
                my_group_index, base_block_size, count
                );
        /* EXIT OR GO TO Gather */
        goto GATHER;
    }

    /* prepare and post recv operation */
    group_src = bcol_iboffload_binomial_root_to_src(coll_request->root,
            my_group_index, iboffload_module->power_of_2_ranks,
            iboffload_module->group_size, &power_of_2_distance);

    IBOFFLOAD_VERBOSE(10, ("SRC %d DIST %d ranks %d gsize %d root %d my rank %d",
                group_src, power_of_2_distance, iboffload_module->power_of_2_ranks,
                iboffload_module->group_size,
                coll_request->root, my_group_index));
    assert(group_src >= 0);

    if (0 > power_of_2_distance) {
        /* the rank is virtual root for this group, receive the data
           and scatter gather as root */
        power_of_2_distance =
            iboffload_module->power_of_2;
        offset = 0;
        recv_count = count;
        IBOFFLOAD_VERBOSE(10, ("Virtual root %d , set mask to %d",
                    my_group_index, power_of_2_distance));
    } else {
        int my_left_boundary_rank;
        int delta;
        recv_count = base_block_size * (1 << power_of_2_distance); /* we may receive larger data */
        my_left_boundary_rank = my_group_index & ((~(int)0) << power_of_2_distance );
        offset = (size_t) (base_block_size * my_left_boundary_rank);
        delta = count - offset;
        if (OPAL_UNLIKELY(delta <= 0)) {
            /* no data to recv */
            goto GATHER;
        } else {
            recv_count = (delta < recv_count) ? delta : recv_count;
        }

        IBOFFLOAD_VERBOSE(10, ("Recv data set mask to %d",
                    power_of_2_distance));
    }

    IBOFFLOAD_VERBOSE(10, ("Bcast, receive data from %d[%d], count %d, offset %d",
                group_src, recv_count, offset));

    /* Receive data to user buffer */
    rc = mca_bcol_iboffload_send_rtr_setup(&last_send,
                                group_src, iboffload_module,
                                coll_fragment);
    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to setup send rtr"));
        return OMPI_ERROR;
    }

    rc = mca_bcol_iboffload_recv_large_buff_setup(&last_wait,
                                SBUF, offset, recv_count, group_src,
                                iboffload_module, coll_fragment);
    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to setup data receive"));
        return OMPI_ERROR;
    }

    BINOMIAL_SCATTER(iboffload_module, coll_fragment,
            last_wait, last_send, power_of_2_distance - 1,
            my_group_index, base_block_size, count);

GATHER:
    rc = bcol_iboffload_bcast_binomial_gather(iboffload_module,
            &last_send, &last_wait, coll_fragment,
            count, base_block_size, power_of_2_distance);
    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
        IBOFFLOAD_VERBOSE(10, ("Failed to setup gather. Return %d", rc));
        return rc;
    }

    if (recursive_doubling_tree->n_extra_sources > 0 &&
            iboffload_module->power_of_2 != power_of_2_distance) {
        dst = recursive_doubling_tree->rank_extra_source;

        rc = mca_bcol_iboffload_recv_rtr_setup(
                &last_wait, dst, iboffload_module, coll_fragment);
        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
            IBOFFLOAD_VERBOSE(10, ("Failed to"
                        " mca_bcol_iboffload_recv_rtr_setup"));
            return OMPI_ERROR;
        }

        rc = mca_bcol_iboffload_send_large_buff_setup(
                &last_send, SBUF, 0, count, dst,
                iboffload_module, coll_fragment);
        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
            IBOFFLOAD_VERBOSE(10, ("Failed to"
                        " mca_bcol_iboffload_send_large_buff_setup"));
            return OMPI_ERROR;
        }
    }

    IBOFFLOAD_VERBOSE(10, ("Fill in the the rest of the coll_fragment.\n"));

    /* end of list */
    *coll_fragment->tail_next = NULL;

    /* finish initializing full message descriptor */
    coll_request->n_fragments  += 1;
    coll_request->n_frags_sent += 1;

    if (NULL != last_wait) {
        last_wait->flags |= MQE_WR_FLAG_SIGNAL;
        coll_fragment->signal_task_wr_id = last_wait->wr_id;
        last_wait->wr_id = (uint64_t) (uintptr_t) coll_fragment;
    } else {
        last_send->flags |= MQE_WR_FLAG_SIGNAL;
        coll_fragment->signal_task_wr_id = last_send->wr_id;
        last_send->wr_id = (uint64_t) (uintptr_t) coll_fragment;
    }

    /* post the mwr */
    rc = mca_bcol_iboffload_post_mqe_tasks(iboffload_module, coll_fragment->to_post);
    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
        IBOFFLOAD_VERBOSE(10, ("MQE task posting failing.\n"));
        /* Note: need to clean up */
        return rc;
    }

    MCA_BCOL_UPDATE_ORDER_COUNTER(&iboffload_module->super, coll_request->order_info);

    IBOFFLOAD_VERBOSE(10, ("Return success.\n"));

    return BCOL_FN_STARTED;

out_of_resources:
    /* Release all resources */
    IBOFFLOAD_VERBOSE(10, ("Barrier, adding collfrag to collfrag_pending.\n"));
    rc =
        mca_bcol_iboffload_free_resources_and_move_to_pending(coll_fragment, iboffload_module);
    return (OMPI_SUCCESS != rc) ? BCOL_FN_NOT_STARTED : BCOL_FN_STARTED;
}
Exemplo n.º 11
0
static int mca_bcol_iboffload_bcast_init(
                               bcol_function_args_t *fn_arguments,
                               mca_bcol_iboffload_module_t *iboffload_module,
                               mca_bcol_iboffload_collreq_t **coll_request,
                               bool if_bcol_last, int mq_credits,
                               collective_message_progress_function progress_fn)
{
    ompi_free_list_item_t *item;
    mca_bcol_iboffload_collfrag_t *coll_fragment;
    mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;
    int my_group_index = iboffload_module->super.sbgp_partner_module->my_index;

    OMPI_FREE_LIST_WAIT_MT(&cm->collreqs_free, item);
    if (OPAL_UNLIKELY(NULL == item)) {
        IBOFFLOAD_ERROR(("Wait for free list failed.\n"));
        return OMPI_ERR_OUT_OF_RESOURCE;
    }
    /* setup call request */
    (*coll_request) = (mca_bcol_iboffload_collreq_t *) item;

    (*coll_request)->n_fragments  = 0;
    (*coll_request)->n_frags_sent = 0;
    (*coll_request)->n_frag_mpi_complete = 0;
    (*coll_request)->n_frag_net_complete = 0;
    (*coll_request)->if_bcol_last = if_bcol_last;
    (*coll_request)->ml_buffer_index = fn_arguments->buffer_index;
    (*coll_request)->completion_cb_fn = NULL;
    (*coll_request)->buffer_info[SBUF].buf = (void *) (
            (unsigned char *)fn_arguments->sbuf +
            fn_arguments->sbuf_offset);
    (*coll_request)->buffer_info[SBUF].offset = fn_arguments->sbuf_offset;
    (*coll_request)->buffer_info[RBUF].offset = fn_arguments->rbuf_offset;

    (*coll_request)->dtype = fn_arguments->dtype;
    (*coll_request)->count = fn_arguments->count;
    (*coll_request)->module = iboffload_module;
    /* TODO Pasha: we need it for pending quque. Set it later. */
    (*coll_request)->progress_fn = progress_fn;
    /* TODO Pasha: fix it  later */
    (*coll_request)->qp_index = MCA_BCOL_IBOFFLOAD_QP_REGULAR;

    (*coll_request)->order_info = &fn_arguments->order_info;

    coll_fragment = &((*coll_request)->first_collfrag);
    mca_bcol_iboffload_collfrag_init(coll_fragment);

    /** Vasily ????? */
    /* mq_credits = (*coll_request)->total_tasks_num; */
    coll_fragment->mq_credits = mq_credits;
    coll_fragment->mq_index = COLL_MQ;
    /* Pasha: just set it to zero */
    coll_fragment->last_wait_num = 0;
    coll_fragment->alg = -2; /* used only for debug */
    /*
    if (my_rank == algthm_ptr->root) {
        coll_fragment->last_wait_num = 0;
    } else {
        coll_fragment->last_wait_num = algth_lst->last_wait_num;
    }
    */
    /* Pasha: we have nothing to unpack */
    coll_fragment->unpack_size = 0;
    /* coll_fragment->unpack_size = pack_len; */
    /* coll_fragment->alg = RECURSIVE_DOUBLING_TREE_BCAST; */

    /* set pointers for (coll frag) <-> (coll full request) */
    (*coll_request)->user_handle_freed = false;

    fn_arguments->bcol_opaque_data = (void *) (*coll_request);

    if (true == fn_arguments->root_flag) {
        (*coll_request)->root = my_group_index;
    } else {
        (*coll_request)->root = fn_arguments->root_route->rank;
    }

    MCA_BCOL_IBOFFLOAD_SET_COLL_REQ_LINKS((*coll_request), coll_fragment);
    return OMPI_SUCCESS;
}
static int progress_one_device(mca_bcol_iboffload_device_t *device)
{
    int ne, rc, count = 0;

    mca_bcol_iboffload_collfrag_t *coll_frag;
    mca_bcol_iboffload_collreq_t *coll_request;

    struct ibv_wc wc;
    memset(&wc, 0, sizeof(struct ibv_wc));

    /*
     * poll for collective completion - does not mean resources can
     * be freed, as incomplete network level sends may still be pending
     */

    /* Poll for completion on completion on wait MQEs */
    if(0 != (ne = ibv_poll_cq(device->ib_mq_cq, 1, &wc))) {
        do {
            if (OPAL_UNLIKELY(0 > ne)) {
                IBOFFLOAD_ERROR(("Device %s: "
                                 "failed to poll MQ completion queue\n",
                                 ibv_get_device_name(device->dev.ib_dev)));
                fatal_error("failed to poll MQ completion queue\n");
                return count;
            }

            if (OPAL_UNLIKELY(IBV_WC_SUCCESS != wc.status)) {
                IBOFFLOAD_ERROR(("Device %s: "
                                 "the completion with error on wait was gotten, status %d, opcode %d, "
                                 "vendor_err 0x%x, qp %x, id 0x%x\n", ibv_get_device_name(device->dev.ib_dev),
                                 wc.status, wc.opcode, wc.vendor_err, wc.qp_num, wc.wr_id));
                fatal_error("wc.status \n");
                return count;
            }

            IBOFFLOAD_VERBOSE(10, ("The MQ completion was polled.\n"));

            ++count;

            /* get pointer to mca_bcol_iboffload_collfrag_t */
            coll_frag = (mca_bcol_iboffload_collfrag_t*)
                        (uint64_t) (uintptr_t) wc.wr_id;

            /* Only last MQ task of collective frag
               sends completion signal, so if we got it =>
               all MQEs were done. */
            coll_frag->complete = true;

            IBOFFLOAD_VERBOSE(10, ("MQ completion for algorithm %d coll_frag_addr %p ml buffer index %d",
                                   coll_frag->alg, (void *)coll_frag, coll_frag->coll_full_req->ml_buffer_index));

            /* full request descriptor */
            coll_request = coll_frag->coll_full_req;

            coll_request->n_frag_mpi_complete++;

            /*
             * at this stage all receives have been completed, so
             * unpack the data to user buffer, the resources will be released when we will done with all
             * element in the task list
             */

            if (NULL != coll_request->completion_cb_fn) {
                if (OMPI_SUCCESS !=
                        coll_request->completion_cb_fn(coll_frag)) {
                    fatal_error("coll_request->completion_cb_fn\n");
                    return count;
                }
            }

            if (coll_request->n_frag_mpi_complete ==
                    coll_request->n_fragments) {
                coll_request->super.req_complete = true;
                opal_condition_broadcast(&ompi_request_cond);
                IBOFFLOAD_VERBOSE(10, ("After opal_condition_broadcast.\n"));
            }

            rc = handle_collfrag_done(coll_frag, coll_request, device);
            if (0 != rc) {
                return count;
            }
        } while(0 != (ne = ibv_poll_cq(device->ib_mq_cq, 1, &wc)));

        return count;
    }

    /* poll the send completion queue */
    do {
        ne = ibv_poll_cq(device->ib_cq, 1, &wc);
        if (0 < ne) {
            if (OPAL_UNLIKELY(IBV_WC_SUCCESS != wc.status)) {
                IBOFFLOAD_ERROR(("Device %s, "
                                 "the completion with error on send was gotten, status %d, opcode %d, "
                                 "vendor_err 0x%x, qp %x, id 0x%x\n", ibv_get_device_name(device->dev.ib_dev),
                                 wc.status, wc.opcode, wc.vendor_err, wc.qp_num, wc.wr_id));

#if OPAL_ENABLE_DEBUG
                {
                    mca_bcol_iboffload_module_t *iboffload;
                    int i, qp_index, num_qps = mca_bcol_iboffload_component.num_qps;

                    coll_frag = (mca_bcol_iboffload_collfrag_t*)
                                (uint64_t) (uintptr_t) wc.wr_id;

                    iboffload = coll_frag->coll_full_req->module;

                    for (i = 0; i < iboffload->num_endpoints; ++i) {
                        mca_bcol_iboffload_endpoint_t *ep = iboffload->endpoints[i];

                        for (qp_index = 0; qp_index < num_qps; ++qp_index) {
                            if (NULL != ep->qps[qp_index].qp->lcl_qp &&
                                    wc.qp_num == ep->qps[qp_index].qp->lcl_qp->qp_num) {
                                IBOFFLOAD_ERROR(("Module - %p, coll_frag - %p, "
                                                 "destination %d, qp index - %d.",
                                                 iboffload, coll_frag, i, qp_index));
                            }
                        }
                    }
                }
#endif
                fatal_error("Failed to ibv_poll_cq\n");
                return count;
            }

            ++count;

            /* get pointer to mca_bcol_iboffload_collfrag_t */
            coll_frag = (mca_bcol_iboffload_collfrag_t*)
                        (uint64_t) (uintptr_t) wc.wr_id;

            /* update the number of completed sends */
            coll_frag->n_sends_completed++;

            IBOFFLOAD_VERBOSE(10, ("Send CQ completion for algorithm %d coll_frag_addr %p ml buffer index %d",
                                   coll_frag->alg, (void *)coll_frag, coll_frag->coll_full_req->ml_buffer_index));

            IBOFFLOAD_VERBOSE(10, ("Alg %d coll_frag_addr %p: n_sends_completed - %d, n_sends - %d.\n",
                                   coll_frag->alg, (void *)coll_frag,
                                   coll_frag->n_sends_completed,
                                   coll_frag->n_sends));

            assert(coll_frag->n_sends_completed <= coll_frag->n_sends);

            /* full message descriptor */
            coll_request = coll_frag->coll_full_req;

            /* check to see if all sends are complete from the network
             * perspective */
            rc = handle_collfrag_done(coll_frag, coll_request, device);
            if (0 != rc) {
                return count;
            }
        } else if (OPAL_UNLIKELY(0 > ne)) {
            IBOFFLOAD_ERROR(("Device %s: "
                             "failed to poll send completion queue\n",
                             ibv_get_device_name(device->dev.ib_dev)));
            fatal_error("failed to poll send completion queue\n");
            return count;
        }
    } while (0 != ne);

    return count;
}
static void fatal_error(char *mesg)
{
    IBOFFLOAD_ERROR(("FATAL ERROR: %s", mesg));
    ompi_mpi_abort(&ompi_mpi_comm_world.comm, MPI_ERR_INTERN, true);
}
static void mca_bcol_iboffload_device_destructor
(mca_bcol_iboffload_device_t *device)
{
    int qp_index, num_qps = mca_bcol_iboffload_component.num_qps;

    IBOFFLOAD_VERBOSE(10, ("Device %s will be destroyed.\n",
                           ibv_get_device_name(device->dev.ib_dev)));

    if (NULL != device->frags_free) {
        for (qp_index = 0; qp_index < num_qps; ++qp_index) {
            mca_bcol_iboffload_dealloc_qps_resource_fn_t dealloc_resource =
                mca_bcol_iboffload_component.qp_infos[qp_index].dealloc_resource;
            if (NULL != dealloc_resource) {
                dealloc_resource(qp_index, device);
            }
        }

        free(device->frags_free);
    }

    if (NULL != device->mpool) {
        IBOFFLOAD_VERBOSE(10, ("Mpool destroy - %p.\n", device->mpool));
        if (OMPI_SUCCESS != mca_mpool_base_module_destroy(device->mpool)) {
            IBOFFLOAD_ERROR(("Device %s, failed to destroy mpool",
                             ibv_get_device_name(device->dev.ib_dev)));
        }
    }

    if (NULL != device->dummy_reg.mr) {
        IBOFFLOAD_VERBOSE(10, ("Dummy memory MR unregister - %p.\n", device->dummy_reg.mr));
        if (OMPI_SUCCESS !=
                mca_bcol_iboffload_deregister_mr((void *) device, &device->dummy_reg.base)) {
            IBOFFLOAD_ERROR(("Device %s: failed to unregister dummy memory MR.",
                             ibv_get_device_name(device->dev.ib_dev)));
        }
    }

    if (NULL != device->ib_cq) {
        if (ibv_destroy_cq(device->ib_cq)) {
            IBOFFLOAD_ERROR(("Device %s, failed to destroy CQ, errno says %s",
                             ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
        }
    }

    if (NULL != device->ib_mq_cq) {
        if (ibv_destroy_cq(device->ib_mq_cq)) {
            IBOFFLOAD_ERROR(("Device %s, failed to destroy mq CQ, errno says %s",
                             ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
        }
    }

    /* Release IB PD if we have one */
    if (NULL != device->ib_pd) {
        if(ibv_dealloc_pd(device->ib_pd)) {
            IBOFFLOAD_ERROR(("Device %s, failed to release PD, errno says %s",
                             ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
        }
    }

    /* close the device */
    if (NULL != device->dev.ib_dev_context) {
        if (ibv_close_device(device->dev.ib_dev_context)) {
            IBOFFLOAD_ERROR(("Device %s "
                             ", failed to close the device, errno says %s",
                             ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
        }
    }

    /* release memory */
    if (NULL != device->ports) {
        free(device->ports);
    }
}
/*
 * Open the component
 */
static int iboffload_open(void)
{
    int rc;

    /* local variables */
    mca_bcol_iboffload_component_t *cm = &mca_bcol_iboffload_component;

    IBOFFLOAD_VERBOSE(10, ("Open Iboffload component.\n"));

    (void) mca_bcol_iboffload_verify_params();

    cm->super.priority = 100;
    cm->super.n_net_contexts = 0;
    cm->super.network_contexts = NULL;

    OBJ_CONSTRUCT(&cm->recv_wrs.lock, opal_mutex_t);

    /* construct lists */
    OBJ_CONSTRUCT(&cm->devices, opal_pointer_array_t);
    rc = opal_pointer_array_init(&cm->devices, 10, INT_MAX, 10);
    if (OMPI_SUCCESS != rc) {
        goto close_device;
    }

    /* Check MCA parameters */
    if (0 != (mca_bcol_iboffload_component.exchange_tree_order & (mca_bcol_iboffload_component.exchange_tree_order - 1))) {
        IBOFFLOAD_ERROR(("Warning: ibcol_iboffload_exchange_tree_order is %d which is not a power of 2, setting it to 2", 
                         mca_bcol_iboffload_component.exchange_tree_order));
        mca_bcol_iboffload_component.exchange_tree_order = 2;
    }

    /* Pasha: Since we do not have max inline check like in openib,
       I will put some dummy check here. All mlnx devices support at least 512b */
    if (mca_bcol_iboffload_component.max_inline_data > 512) {
        IBOFFLOAD_ERROR(("Warning the inline %d, is to big and unsupported",
                    mca_bcol_iboffload_component.max_inline_data));
        rc = OMPI_ERROR;
        goto close_device;
    }

    /* Register the progress function */
    rc = opal_progress_register(mca_bcol_iboffload_component_progress);
    if (OMPI_SUCCESS != rc) {
        IBOFFLOAD_ERROR(("Failed to register the progress function"
                         " for iboffload component.\n"));
        goto close_device;
    }

    map_ompi_to_ib_dtype();
    map_ompi_to_ib_op_type();

    /* The init_done set to true on first component usage */
    cm->init_done = false;

    return OMPI_SUCCESS;

close_device:
    OBJ_DESTRUCT(&cm->devices);
    OBJ_DESTRUCT(&cm->recv_wrs.lock);
    return rc;
}