void
mca_btl_udapl_error(DAT_RETURN ret, char* str)
{
    char* major;
    char* minor;

    if(DAT_SUCCESS != dat_strerror(ret,
            (const char**)&major, (const char**)&minor))
    {
        printf("dat_strerror failed! ret is %d\n", ret);
        exit(-1);
    }

    OPAL_OUTPUT((0, "ERROR: %s %s %s\n", str, major, minor));
}
示例#2
0
static int udapl_dereg_mr(void *reg_data, mca_mpool_base_registration_t *reg)
{
    mca_btl_udapl_reg_t *udapl_reg = (mca_btl_udapl_reg_t*)reg;
    int rc;

    if(udapl_reg->lmr != NULL) {
        rc = dat_lmr_free(udapl_reg->lmr);
        if(rc != DAT_SUCCESS) {
            char* major;
            char* minor;

            dat_strerror(rc, (const char**)&major,
                (const char**)&minor);
            BTL_ERROR(("ERROR: %s %s %s\n", "dat_lmr_free",
                major, minor));
            return OMPI_ERROR;
        }
    }

    return OMPI_SUCCESS;
}
示例#3
0
static int mca_btl_udapl_set_peer_parameters(
    struct mca_btl_udapl_module_t* udapl_btl,
    size_t nprocs) 
{
    int rc = OMPI_SUCCESS;
    DAT_RETURN dat_rc = DAT_SUCCESS;
    uint potential_udapl_timeout;
    int first_time_sizing = (udapl_btl->udapl_num_peers == 0 ? 1 : 0);
    DAT_EVD_PARAM evd_param;
    
    /* nprocs includes self so subtract 1 */
    udapl_btl->udapl_num_peers += nprocs - 1; 

    /* resize dto_evd_qlen if not already at its max */
    if (udapl_btl->udapl_dto_evd_qlen !=
        udapl_btl->udapl_ia_attr.max_evd_qlen) {

        int potential_dto_evd_qlen;
        int max_connection_dto_events;
        int eager_connection_dto_events;

        /* eager connection dto events already factored into
         * max_recv/request_dtos but need to calculate max connection dtos;
         * see mca_btl_udapl_get_params() for max_recv/request_dtos 
         */
        eager_connection_dto_events = udapl_btl->udapl_max_recv_dtos +
            udapl_btl->udapl_max_request_dtos;
        max_connection_dto_events = mca_btl_udapl_component.udapl_num_recvs +
            mca_btl_udapl_component.udapl_num_sends +
            (mca_btl_udapl_component.udapl_num_recvs /
                mca_btl_udapl_component.udapl_sr_win) + 1;
        potential_dto_evd_qlen = udapl_btl->udapl_num_peers *
            (eager_connection_dto_events + max_connection_dto_events);
        
        /* here we use what the library calculates as the
         * potential_dto_evd_qlen unless the user has set
         */
        if (first_time_sizing) { 
            if (udapl_btl->udapl_dto_evd_qlen < potential_dto_evd_qlen) {
                if (MCA_BTL_UDAPL_DTO_EVD_QLEN_DEFAULT !=
                    udapl_btl->udapl_dto_evd_qlen) {

                    /* user modified so warn */
                    BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
                        ("help-mpi-btl-udapl.txt",
                        "evd_qlen too low", 
                        true,
                        "btl_udapl_dto_evd_qlen",
                        udapl_btl->udapl_dto_evd_qlen,
                        "btl_udapl_dto_evd_qlen",                        
                        potential_dto_evd_qlen));
                } else {
                    udapl_btl->udapl_dto_evd_qlen = potential_dto_evd_qlen;
                }
            }
        } else {
            /* since this is not the first time attempting to resize the
             * evd queue length just use the potential value; this may not
             * be the best solution
             */
            udapl_btl->udapl_dto_evd_qlen = potential_dto_evd_qlen;
        }

        udapl_btl->udapl_dto_evd_qlen = ((udapl_btl->udapl_dto_evd_qlen >
            udapl_btl->udapl_ia_attr.max_evd_qlen) ?
            udapl_btl->udapl_ia_attr.max_evd_qlen :
            udapl_btl->udapl_dto_evd_qlen);
            
        /* OFED stack does not return DAT_INVALID_STATE when
         * the new qlen is less than current value so here we find
         * current value and if greater than what we intend to set
         * it to skip the resize. 
         */
        dat_rc = dat_evd_query(udapl_btl->udapl_evd_dto,
            DAT_EVD_FIELD_EVD_QLEN, &evd_param);
        if(DAT_SUCCESS != dat_rc) {
            char* major;
            char* minor;

            dat_strerror(dat_rc, (const char**)&major,
                (const char**)&minor);
            BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_query",
                major, minor));
        }

        if (udapl_btl->udapl_dto_evd_qlen > evd_param.evd_qlen) {
            /* resize dto event dispatcher queue length */
            dat_rc = dat_evd_resize(udapl_btl->udapl_evd_dto,
                udapl_btl->udapl_dto_evd_qlen);
            if(DAT_SUCCESS != dat_rc) {
                char* major;
                char* minor;

                dat_strerror(dat_rc, (const char**)&major,
                    (const char**)&minor);
                BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_resize",
                    major, minor));
                rc = OMPI_ERR_OUT_OF_RESOURCE;
            } 
        }
    }

    /* resize connection evd qlen */
    if (udapl_btl->udapl_conn_evd_qlen !=
        udapl_btl->udapl_ia_attr.max_evd_qlen) {

        int potential_conn_evd_qlen = 2 * udapl_btl->udapl_num_peers;

        if (first_time_sizing) { 
            if (udapl_btl->udapl_conn_evd_qlen < potential_conn_evd_qlen) {
                if (MCA_BTL_UDAPL_CONN_EVD_QLEN_DEFAULT !=
                    udapl_btl->udapl_conn_evd_qlen) {

                    /* user modified so warn */
                    BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
                        ("help-mpi-btl-udapl.txt",
                        "evd_qlen too low", 
                        true,
                        "btl_udapl_conn_evd_qlen",
                        udapl_btl->udapl_conn_evd_qlen,
                        "btl_udapl_conn_evd_qlen",
                        potential_conn_evd_qlen));
                } else {
                    udapl_btl->udapl_conn_evd_qlen = potential_conn_evd_qlen;
                }
            }
        } else {
            /* since this is not the first time attempting to resize the
             * evd queue length just use the potential value; this may not
             * be the best solution
             */
            udapl_btl->udapl_conn_evd_qlen = potential_conn_evd_qlen;
        }

        udapl_btl->udapl_conn_evd_qlen = ((udapl_btl->udapl_conn_evd_qlen >
            udapl_btl->udapl_ia_attr.max_evd_qlen) ?
            udapl_btl->udapl_ia_attr.max_evd_qlen :
            udapl_btl->udapl_conn_evd_qlen);
        
        /* OFED stack does not return DAT_INVALID_STATE when
         * the new qlen is less than current value so here we find
         * current value and if greater than what we intend to set
         * it to skip the resize. 
         */
        dat_rc = dat_evd_query(udapl_btl->udapl_evd_conn,
            DAT_EVD_FIELD_EVD_QLEN, &evd_param);
        if(DAT_SUCCESS != dat_rc) {
            char* major;
            char* minor;

            dat_strerror(dat_rc, (const char**)&major,
                (const char**)&minor);
            BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_query",
                major, minor));
        }

        if (udapl_btl->udapl_conn_evd_qlen > evd_param.evd_qlen) {
            /* resize conn evd queue length */
            dat_rc = dat_evd_resize(udapl_btl->udapl_evd_conn,
                udapl_btl->udapl_conn_evd_qlen);
            if(DAT_SUCCESS != dat_rc) {
                char* major;
                char* minor;

                dat_strerror(dat_rc, (const char**)&major,
                    (const char**)&minor);
                BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_resize",
                    major, minor));
                rc = OMPI_ERR_OUT_OF_RESOURCE;
            } 
        }
    }
    
    /* adjust connection timeout value, calculated in microseconds */
    potential_udapl_timeout = MCA_BTL_UDAPL_CONN_TIMEOUT_INC *
        udapl_btl->udapl_num_peers;
    
    if (mca_btl_udapl_component.udapl_timeout <
        potential_udapl_timeout) {

        if (MCA_BTL_UDAPL_CONN_TIMEOUT_DEFAULT !=
            mca_btl_udapl_component.udapl_timeout) {

            /* user modified so warn */
            BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
                ("help-mpi-btl-udapl.txt",
                "connection timeout low", 
                true,
                "btl_udapl_timeout",
                mca_btl_udapl_component.udapl_timeout,
                "btl_udapl_timeout",                
                potential_udapl_timeout));         
        } else {
            mca_btl_udapl_component.udapl_timeout =
                potential_udapl_timeout;
        }
    }
    mca_btl_udapl_component.udapl_timeout =
        ((mca_btl_udapl_component.udapl_timeout >
            MCA_BTL_UDAPL_CONN_TIMEOUT_MAX) ?
            MCA_BTL_UDAPL_CONN_TIMEOUT_MAX :
            mca_btl_udapl_component.udapl_timeout);

    return rc;
}
示例#4
0
int
mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t* btl)
{
    mca_mpool_base_resources_t res;
    DAT_CONN_QUAL port;
    DAT_RETURN rc;

    /* open the uDAPL interface */
    btl->udapl_evd_async = DAT_HANDLE_NULL;
    rc = dat_ia_open(ia_name, btl->udapl_async_evd_qlen,
            &btl->udapl_evd_async, &btl->udapl_ia);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);

#if defined(__SVR4) && defined(__sun)
        if (strcmp(major, "DAT_INVALID_PARAMETER") == 0 &&
            strcmp(minor, "DAT_INVALID_RO_COOKIE") == 0) {
            /* Some platforms that Solaris runs on implement the PCI 
	     * standard for relaxed ordering(RO). Using RDMA with 
	     * polling on a memory location as the uDAPL (and openib
	     * by the way) BTL does for short messages with 
	     * relaxed ordering could potentially produce silent data
	     * corruption. For this reason we need to take extra
	     * steps and this is accomplished by setting
	     * "ro_aware_system = 1" and handling as required.
             *
	     * The uDAPL standard does not provide an interface to
	     * inform users of this scenario so Sun has implemented the
	     * following: If a platform supports relaxed ordering
	     * when the interface name is passed into the
	     * dat_ia_open() call, the call will return 
	     * DAT_INVALID_PARAMETER and DAT_INVALID_RO_COOKIE.  
	     * DAT_INVALID_RO_COOKIE is not part of the uDAPL standard
	     * at this time. The only way to open this interface is
	     * to prefix the following cookie "RO_AWARE_" to the ia
	     * name that was retreived from the dat registry.
             *
             * Example: ia_name = "ib0", new expected name will be
             * "RO_AWARE_ib0".
             * 
             * Here, since our first ia open attempt failed in the
             * standard way, add the cookie and try to open again.
             */
            DAT_NAME_PTR ro_ia_name;

            /* prefix relaxed order cookie to ia_name */
            asprintf(&ro_ia_name, "RO_AWARE_%s", ia_name);
            if (NULL == ro_ia_name) {
                return OMPI_ERR_OUT_OF_RESOURCE;
            }

            /* because this is not standard inform user in some way */
            BTL_UDAPL_VERBOSE_HELP(VERBOSE_INFORM,
                ("help-mpi-btl-udapl.txt", "relaxed order support",
                true, ia_name, ro_ia_name));

            /* try and open again */
            btl->udapl_evd_async = DAT_HANDLE_NULL;
            rc = dat_ia_open(ro_ia_name, btl->udapl_async_evd_qlen,
                &btl->udapl_evd_async, &btl->udapl_ia);

	    dat_strerror(rc, (const char**)&major,
		(const char**)&minor);

            if (DAT_SUCCESS == rc) {
                mca_btl_udapl_component.ro_aware_system = 1;
                free(ro_ia_name);
            } else {
                BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
                    ("help-mpi-btl-udapl.txt",
                        "dat_ia_open fail RO", true, ro_ia_name,
                        major, minor, ia_name));

                free(ro_ia_name);
                return OMPI_ERROR;
            }
        } else {
#endif
        BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
            "dat_ia_open fail", true, ia_name, major, minor));

        return OMPI_ERROR;
#if defined(__SVR4) && defined(__sun)	    
        }
#endif
    }

    /* create a protection zone */
    rc = dat_pz_create(btl->udapl_ia, &btl->udapl_pz);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_pz_create",
            major, minor));
        goto failure;
    }

    /* query to get address information */
    rc = dat_ia_query(btl->udapl_ia, &btl->udapl_evd_async,
            DAT_IA_ALL, &(btl->udapl_ia_attr), 0, NULL);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_ia_query",
            major, minor));
        goto failure;
    }

    memcpy(&btl->udapl_addr.addr, (btl->udapl_ia_attr).ia_address_ptr,
        sizeof(DAT_SOCK_ADDR));

    /* determine netmask */
    mca_btl_udapl_assign_netmask(btl);

    /* check evd qlen against adapter max */
    if (btl->udapl_dto_evd_qlen > (btl->udapl_ia_attr).max_evd_qlen) {
        BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
            "evd_qlen adapter max", 
            true,
            "btl_udapl_dto_evd_qlen",
            btl->udapl_dto_evd_qlen,
            (btl->udapl_ia_attr).max_evd_qlen));        
        btl->udapl_dto_evd_qlen = btl->udapl_ia_attr.max_evd_qlen;
    }
    if (btl->udapl_conn_evd_qlen > (btl->udapl_ia_attr).max_evd_qlen) {
        BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
            "evd_qlen adapter max", 
            true,
            "btl_udapl_conn_evd_qlen",
            btl->udapl_conn_evd_qlen,
            (btl->udapl_ia_attr).max_evd_qlen));        
        btl->udapl_conn_evd_qlen = btl->udapl_ia_attr.max_evd_qlen;
    }

    /* set up evd's */
    rc = dat_evd_create(btl->udapl_ia,
        btl->udapl_dto_evd_qlen, DAT_HANDLE_NULL,
        DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_dto);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_create (dto)",
            major, minor));
        goto failure;
    }

    rc = dat_evd_create(btl->udapl_ia,
            btl->udapl_conn_evd_qlen, DAT_HANDLE_NULL,
            DAT_EVD_CR_FLAG | DAT_EVD_CONNECTION_FLAG, &btl->udapl_evd_conn);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_evd_create (conn)",
            major, minor));
        goto failure;
    }

    /* create our public service point */
    rc = dat_psp_create_any(btl->udapl_ia, &port, btl->udapl_evd_conn,
        DAT_PSP_CONSUMER_FLAG, &btl->udapl_psp);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_psp_create_any",
            major, minor));
        goto failure;
    }

    /* establish endpoint parameters */
    rc = mca_btl_udapl_endpoint_get_params(btl, &(btl->udapl_ep_param));
    if(OMPI_SUCCESS != rc) { 
        /* by not erroring out here we can try to continue with
         * the default endpoint parameter values
         */
        BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
            "use default endpoint params", 
            true));
    }

    /* Save the port with the address information */
    /* TODO - since we're doing the hack below, do we need our own port? */
    btl->udapl_addr.port = port;

    /* Using dat_ep_query to obtain the remote port would be ideal but
     * since the current udapl implementations don't seem to support
     * this we store the port in udapl_addr and explictly exchange the
     * information later.
     */
    ((struct sockaddr_in*)&btl->udapl_addr.addr)->sin_port = htons(port);

    /* initialize the memory pool */
    res.pool_name = "udapl";
    res.reg_data = btl;
    res.sizeof_reg = sizeof(mca_btl_udapl_reg_t);
    res.register_mem = udapl_reg_mr;
    res.deregister_mem = udapl_dereg_mr;
    btl->super.btl_mpool = mca_mpool_base_module_create(
            mca_btl_udapl_component.udapl_mpool_name, &btl->super, &res);
    if (NULL == btl->super.btl_mpool) {
        BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_INFORM,
            ("WARNING: Failed to create mpool."));
        goto failure;
    }
 
    /* initialize objects */
    OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t);
    OBJ_CONSTRUCT(&btl->udapl_frag_eager_recv, ompi_free_list_t);
    OBJ_CONSTRUCT(&btl->udapl_frag_max, ompi_free_list_t);
    OBJ_CONSTRUCT(&btl->udapl_frag_max_recv, ompi_free_list_t);
    OBJ_CONSTRUCT(&btl->udapl_frag_user, ompi_free_list_t);
    OBJ_CONSTRUCT(&btl->udapl_frag_control, ompi_free_list_t);
    OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t);
    
     /* check buffer alignment against dat library */
    if (mca_btl_udapl_component.udapl_buffer_alignment !=
        DAT_OPTIMAL_ALIGNMENT) {

        BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt",
            "optimal buffer alignment mismatch", 
            true,
            DAT_OPTIMAL_ALIGNMENT,
            mca_btl_udapl_component.udapl_buffer_alignment,
            DAT_OPTIMAL_ALIGNMENT));
    }

    /* initialize free lists */
    ompi_free_list_init_ex_new(&btl->udapl_frag_eager,
        sizeof(mca_btl_udapl_frag_eager_t) +
            mca_btl_udapl_component.udapl_eager_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        OBJ_CLASS(mca_btl_udapl_frag_eager_t),
        mca_btl_udapl_component.udapl_eager_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        mca_btl_udapl_component.udapl_free_list_num,
        mca_btl_udapl_component.udapl_free_list_max,
        mca_btl_udapl_component.udapl_free_list_inc,
                           btl->super.btl_mpool,
                           NULL,
                           NULL);

    ompi_free_list_init_ex_new(&btl->udapl_frag_eager_recv,
        sizeof(mca_btl_udapl_frag_eager_t) +
            mca_btl_udapl_component.udapl_eager_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        OBJ_CLASS(mca_btl_udapl_frag_eager_t),
        mca_btl_udapl_component.udapl_eager_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        mca_btl_udapl_component.udapl_free_list_num,
        mca_btl_udapl_component.udapl_free_list_max,
        mca_btl_udapl_component.udapl_free_list_inc,
                           btl->super.btl_mpool,
                           NULL,
                           NULL);

    ompi_free_list_init_ex_new(&btl->udapl_frag_max,
        sizeof(mca_btl_udapl_frag_max_t) +
            mca_btl_udapl_component.udapl_max_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        OBJ_CLASS(mca_btl_udapl_frag_max_t),
        mca_btl_udapl_component.udapl_max_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        mca_btl_udapl_component.udapl_free_list_num,
        mca_btl_udapl_component.udapl_free_list_max,
        mca_btl_udapl_component.udapl_free_list_inc,
                           btl->super.btl_mpool,
                           NULL,
                           NULL);

    ompi_free_list_init_ex_new(&btl->udapl_frag_max_recv,
        sizeof(mca_btl_udapl_frag_max_t) +
            mca_btl_udapl_component.udapl_max_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        OBJ_CLASS(mca_btl_udapl_frag_max_t),
        mca_btl_udapl_component.udapl_max_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        mca_btl_udapl_component.udapl_free_list_num,
        mca_btl_udapl_component.udapl_free_list_max,
        mca_btl_udapl_component.udapl_free_list_inc,
                           btl->super.btl_mpool,
                           NULL,
                           NULL);

    ompi_free_list_init_ex_new(&btl->udapl_frag_user,
        sizeof(mca_btl_udapl_frag_user_t),
        mca_btl_udapl_component.udapl_buffer_alignment,
        OBJ_CLASS(mca_btl_udapl_frag_user_t),
        0,0,
        mca_btl_udapl_component.udapl_free_list_num,
        mca_btl_udapl_component.udapl_free_list_max,
        mca_btl_udapl_component.udapl_free_list_inc,
                           NULL,
                           NULL,
                           NULL);

    ompi_free_list_init_ex_new(&btl->udapl_frag_control,
        sizeof(mca_btl_udapl_frag_eager_t) +
        mca_btl_udapl_component.udapl_eager_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        OBJ_CLASS(mca_btl_udapl_frag_eager_t),
        mca_btl_udapl_component.udapl_eager_frag_size,
        mca_btl_udapl_component.udapl_buffer_alignment,
        mca_btl_udapl_component.udapl_free_list_num,
        -1,
        mca_btl_udapl_component.udapl_free_list_inc,
                           btl->super.btl_mpool,
                           NULL,
                           NULL);

    /* initialize eager rdma buffer info */
    btl->udapl_eager_rdma_endpoints = OBJ_NEW(opal_pointer_array_t);
    opal_pointer_array_init(btl->udapl_eager_rdma_endpoints, 
        mca_btl_udapl_component.udapl_max_eager_rdma_peers,
        mca_btl_udapl_component.udapl_max_eager_rdma_peers, 
        0);
    btl->udapl_eager_rdma_endpoint_count = 0;
    OBJ_CONSTRUCT(&btl->udapl_eager_rdma_lock, opal_mutex_t);

    /* initialize miscellaneous variables */
    btl->udapl_async_events = 0;
    btl->udapl_connect_inprogress = 0;
    btl->udapl_num_peers = 0;

    /* TODO - Set up SRQ when it is supported */
    return OMPI_SUCCESS;

failure:
    dat_ia_close(btl->udapl_ia, DAT_CLOSE_ABRUPT_FLAG);
    return OMPI_ERROR;
}
示例#5
0
int mca_btl_udapl_put( 
    mca_btl_base_module_t* btl,
    mca_btl_base_endpoint_t* endpoint,
    mca_btl_base_descriptor_t* des)
{
    DAT_RMR_TRIPLET remote_buffer;
    DAT_DTO_COOKIE cookie;
    int rc = OMPI_SUCCESS;
    
    mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des;
    mca_btl_udapl_segment_t *dst_segment = des->des_dst;

    frag->btl = (mca_btl_udapl_module_t *)btl;
    frag->endpoint = endpoint;
    frag->type = MCA_BTL_UDAPL_PUT;

    if (OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], -1) < 0) {
        /* no local work queue tokens available */
        OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], 1);
        OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
        opal_list_append(&endpoint->endpoint_max_frags,
            (opal_list_item_t*)frag);
        OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
        opal_progress();
    } else {
        /* work queue tokens available, try to send  */

        if(OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], -1) < 0) {
            OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], 1);
            OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], 1);
            OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
            opal_list_append(&endpoint->endpoint_max_frags,
                (opal_list_item_t*)frag);
            OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
            opal_progress();
        } else {
            frag->triplet.segment_length = frag->segment.base.seg_len;
        
            remote_buffer.rmr_context = dst_segment->context;
            remote_buffer.target_address =
                (DAT_VADDR)(uintptr_t)dst_segment->base.seg_addr.lval;
            remote_buffer.segment_length = dst_segment->base.seg_len;

            cookie.as_ptr = frag;
            OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
            rc = dat_ep_post_rdma_write(endpoint->endpoint_max,
                1,
                &frag->triplet,
                cookie,
                &remote_buffer,        
                DAT_COMPLETION_DEFAULT_FLAG);
            OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
            if(DAT_SUCCESS != rc) {
                char* major;
                char* minor;

                dat_strerror(rc, (const char**)&major,
                    (const char**)&minor);
                BTL_ERROR(("ERROR: %s %s %s\n", "dat_ep_post_rdma_write",
                    major, minor));
                rc = OMPI_ERROR;
            }
        }
    }
    
    return rc;
}
static inline int mca_btl_udapl_sendrecv(mca_btl_udapl_module_t* btl,
        DAT_EP_HANDLE* endpoint)
{
    int rc;
    mca_btl_udapl_frag_t* frag;
    DAT_DTO_COOKIE cookie;
    static int32_t connection_seq = 1;
    uint32_t flags = 0;
    mca_btl_base_endpoint_t* btl_endpoint = NULL; /* endpoint required by
                                                   * mca_btl_udapl_alloc has not
                                                   * been created at this point
                                                   */
    
    /* Post a receive to get the peer's address data */
    frag = (mca_btl_udapl_frag_t*)
        mca_btl_udapl_alloc(
                            &btl->super,
                            btl_endpoint, 
                            MCA_BTL_NO_ORDER,
                            sizeof(mca_btl_udapl_addr_t) +
                            sizeof(int32_t),
                            flags);
    cookie.as_ptr = frag;

    frag->type = MCA_BTL_UDAPL_CONN_RECV;

    rc = dat_ep_post_recv(endpoint, 1,
            &frag->triplet, cookie, DAT_COMPLETION_DEFAULT_FLAG);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_ep_post_recv",
            major, minor));
        return OMPI_ERROR;
    }


    /* Send our local address data over this EP */
    frag = (mca_btl_udapl_frag_t*)
        mca_btl_udapl_alloc(
                            &btl->super, 
                            btl_endpoint, 
                            MCA_BTL_NO_ORDER,
                            sizeof(mca_btl_udapl_addr_t) +
                            sizeof(int32_t),
                            flags);
    cookie.as_ptr = frag;

    memcpy(frag->segment.seg_addr.pval,
            &btl->udapl_addr, sizeof(mca_btl_udapl_addr_t));
    memcpy((char *)frag->segment.seg_addr.pval + sizeof(mca_btl_udapl_addr_t),
            &connection_seq, sizeof(int32_t));
    connection_seq++;

    frag->type = MCA_BTL_UDAPL_CONN_SEND;

    rc = dat_ep_post_send(endpoint, 1,
            &frag->triplet, cookie, DAT_COMPLETION_DEFAULT_FLAG);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_ep_post_send",
            major, minor));
        return OMPI_ERROR;
    }

    return OMPI_SUCCESS;
}
static int mca_btl_udapl_accept_connect(mca_btl_udapl_module_t* btl,
                                        DAT_CR_HANDLE cr_handle)
{
    DAT_EP_HANDLE ep;
    int rc;
    mca_btl_base_endpoint_t* proc_ep;
    mca_btl_udapl_addr_t priv_data_in_addr;
    int32_t priv_data_in_conn_type;     /* incoming endpoint type  */

    if (mca_btl_udapl_component.udapl_conn_priv_data) {
        DAT_CR_PARAM cr_param;

        /* query the connection request for incoming private data */
        rc = dat_cr_query(cr_handle,
                    DAT_CR_FIELD_ALL,
                    &cr_param);
        if (rc != DAT_SUCCESS) {
            char* major;
            char* minor;

            dat_strerror(rc, (const char**)&major,
                (const char**)&minor);
            BTL_ERROR(("ERROR: %s %s %s\n", "dat_cr_query",
                major, minor));
            return OMPI_ERROR;
        }

        /* retrieve data from connection request event;
         * cr_param contains remote_port_qual but we need to
         * match on the psp port and address of remote
         * so we get this from the private data.
         */
        memcpy(&priv_data_in_addr,
            (mca_btl_udapl_addr_t *)cr_param.private_data,
            sizeof(mca_btl_udapl_addr_t));
        priv_data_in_conn_type = *(int32_t *)
            ((char *)cr_param.private_data + sizeof(mca_btl_udapl_addr_t));
    }

    /* create the endpoint for the incoming connection */
    rc = mca_btl_udapl_endpoint_create(btl, &ep);
    if(OMPI_SUCCESS != rc) {
        BTL_ERROR(("ERROR: mca_btl_udapl_endpoint_create"));
        return OMPI_ERROR;
    }
    
    /* cr_param no longer valid once dat_cr_accept called */
    rc = dat_cr_accept(cr_handle, ep, 0, NULL);
    if(DAT_SUCCESS != rc) {
        char* major;
        char* minor;

        dat_strerror(rc, (const char**)&major,
            (const char**)&minor);
        BTL_ERROR(("ERROR: %s %s %s\n", "dat_cr_accept",
            major, minor));
        return OMPI_ERROR;
    }

    if (mca_btl_udapl_component.udapl_conn_priv_data) {
        /* With accept now in process find a home for the DAT ep by
         * matching against the private data that came in on the
         * connection request event
         */

        /* find the endpoint which matches the address in data received */
        proc_ep = 
            mca_btl_udapl_find_endpoint_address_match(btl, priv_data_in_addr);

        if (proc_ep == NULL) {
            return OMPI_ERROR;
        }

        if (BTL_UDAPL_EAGER_CONNECTION == priv_data_in_conn_type) {
            proc_ep->endpoint_eager = ep;
        } else {
            assert(BTL_UDAPL_MAX_CONNECTION == priv_data_in_conn_type);
            proc_ep->endpoint_max = ep;
        }
    }

    return OMPI_SUCCESS;
}