示例#1
0
static void process_mbox(uct_ugni_smsg_iface_t *iface, uct_ugni_smsg_ep_t *ep){
    ucs_status_t status;
    uint8_t tag;
    void *data_ptr;
    gni_return_t ugni_rc;
    uct_ugni_smsg_header_t *header;
    void *user_data;

    pthread_mutex_lock(&uct_ugni_global_lock);

    while(1){
        tag = GNI_SMSG_ANY_TAG;
        ugni_rc = GNI_SmsgGetNextWTag(ep->super.ep, (void **)&data_ptr, &tag);

        /* Yes, GNI_RC_NOT_DONE means that you're done with the smsg mailbox */
        if(GNI_RC_NOT_DONE == ugni_rc){
            pthread_mutex_unlock(&uct_ugni_global_lock);
            return;
        }

        if(GNI_RC_SUCCESS != ugni_rc){
            ucs_error("Unhandled smsg error: %s %d", gni_err_str[ugni_rc], ugni_rc);
            pthread_mutex_unlock(&uct_ugni_global_lock);
            return;
        }

        if(NULL == data_ptr){
            ucs_error("Empty data pointer in smsg.");
            pthread_mutex_unlock(&uct_ugni_global_lock);
            return;
        }

        header = (uct_ugni_smsg_header_t *)data_ptr;
        user_data = (void *)(header + 1);
        void *user_desc = iface->user_desc+1;

        uct_iface_trace_am(&iface->super.super, UCT_AM_TRACE_TYPE_RECV,
                           tag, user_data, header->length, "RX: AM");

        pthread_mutex_unlock(&uct_ugni_global_lock);
        status = uct_iface_invoke_am(&iface->super.super, tag, user_data,
                                     header->length, user_desc);
        pthread_mutex_lock(&uct_ugni_global_lock);

        if(status != UCS_OK){
            uct_recv_desc_iface(user_desc) = &iface->super.super.super;
            UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_desc,
                                     iface->user_desc, iface->user_desc = NULL);
        }

        ugni_rc = GNI_SmsgRelease(ep->super.ep);
        if(GNI_RC_SUCCESS != ugni_rc){
            ucs_error("Unhandled smsg error in GNI_SmsgRelease: %s %d", gni_err_str[ugni_rc], ugni_rc);
            pthread_mutex_unlock(&uct_ugni_global_lock);
            return;
        }
    }
}
示例#2
0
unsigned uct_tcp_ep_progress_rx(uct_tcp_ep_t *ep)
{
    uct_tcp_iface_t *iface = ucs_derived_of(ep->super.super.iface,
                                            uct_tcp_iface_t);
    uct_tcp_am_hdr_t *hdr;
    ucs_status_t status;
    size_t recv_length;
    ssize_t remainder;

    ucs_trace_func("ep=%p", ep);

    /* Receive next chunk of data */
    recv_length = iface->config.buf_size - ep->length;
    status = uct_tcp_recv(ep->fd, ep->buf + ep->length, &recv_length);
    if (status != UCS_OK) {
        if (status == UCS_ERR_CANCELED) {
            ucs_debug("tcp_ep %p: remote disconnected", ep);
            uct_tcp_ep_mod_events(ep, 0, EPOLLIN);
            uct_tcp_ep_destroy(&ep->super.super);
        }
        return 0;
    }

    ep->length += recv_length;
    ucs_trace_data("tcp_ep %p: recvd %zu bytes", ep, recv_length);

    /* Parse received active messages */
    while ((remainder = ep->length - ep->offset) >= sizeof(*hdr)) {
        hdr = ep->buf + ep->offset;
        if (remainder < sizeof(*hdr) + hdr->length) {
            break;
        }

        /* Full message was received */
        ep->offset += sizeof(*hdr) + hdr->length;

        if (hdr->am_id >= UCT_AM_ID_MAX) {
            ucs_error("invalid am id: %d", hdr->am_id);
            continue;
        }

        uct_iface_trace_am(&iface->super, UCT_AM_TRACE_TYPE_RECV, hdr->am_id,
                           hdr + 1, hdr->length, "RECV fd %d", ep->fd);
        uct_iface_invoke_am(&iface->super, hdr->am_id, hdr + 1,
                            hdr->length, 0);
    }

    /* Move the remaining data to the beginning of the buffer
     * TODO avoid extra copy on partial receive
     */
    ucs_assert(remainder >= 0);
    memmove(ep->buf, ep->buf + ep->offset, remainder);
    ep->offset = 0;
    ep->length = remainder;

    return recv_length > 0;
}
示例#3
0
static ucs_status_t processs_datagram(uct_ugni_udt_iface_t *iface, uct_ugni_udt_desc_t *desc)
{
    ucs_status_t status;
    uct_ugni_udt_header_t *header;
    void *payload;

    header = uct_ugni_udt_get_rheader(desc, iface);
    payload = uct_ugni_udt_get_rpayload(desc, iface);
    uct_iface_trace_am(&iface->super.super, UCT_AM_TRACE_TYPE_RECV,
                       header->am_id, payload, header->length, "RX: AM");
    status = uct_iface_invoke_am(&iface->super.super, header->am_id, payload,
                                 header->length, UCT_CB_FLAG_DESC);
    return status;
}
示例#4
0
文件: cm_iface.c 项目: raffenet/ucx
static void uct_cm_iface_handle_sidr_req(uct_cm_iface_t *iface,
                                         struct ib_cm_event *event)
{
    uct_cm_hdr_t *hdr = event->private_data;
    struct ib_cm_sidr_rep_param rep;
    ucs_status_t status;
    void *cm_desc, *desc;
    int ret;

    VALGRIND_MAKE_MEM_DEFINED(hdr, sizeof(hdr));
    VALGRIND_MAKE_MEM_DEFINED(hdr + 1, hdr->length);

    uct_cm_iface_trace_data(iface, UCT_AM_TRACE_TYPE_RECV, hdr, "RX: SIDR_REQ");

    /* Allocate temporary buffer to serve as receive descriptor */
    cm_desc = ucs_malloc(iface->super.config.rx_payload_offset + hdr->length,
                         "cm_recv_desc");
    if (cm_desc == NULL) {
        ucs_error("failed to allocate cm receive descriptor");
        return;
    }

    /* Send reply */
    ucs_trace_data("TX: SIDR_REP [id %p{%u}]", event->cm_id,
                   event->cm_id->handle);
    memset(&rep, 0, sizeof rep);
    rep.status = IB_SIDR_SUCCESS;
    ret = ib_cm_send_sidr_rep(event->cm_id, &rep);
    if (ret) {
        ucs_error("ib_cm_send_sidr_rep() failed: %m");
    }

    /* Call active message handler */
    desc = cm_desc + iface->super.config.rx_headroom_offset;
    uct_recv_desc_iface(desc) = &iface->super.super.super;
    status = uct_iface_invoke_am(&iface->super.super, hdr->am_id, hdr + 1,
                                 hdr->length, desc);
    if (status == UCS_OK) {
        ucs_free(cm_desc);
    }
}
示例#5
0
static void uct_ugni_udt_progress(void *arg)
{
    uint32_t rem_addr,
             rem_id;
    uint64_t id;
    void *payload;
    void *user_desc;

    ucs_status_t status;

    uct_ugni_udt_desc_t *desc;
    uct_ugni_udt_header_t *header;
    uct_ugni_udt_iface_t * iface = (uct_ugni_udt_iface_t *)arg;
    uct_ugni_udt_ep_t *ep;

    gni_ep_handle_t ugni_ep;
    gni_post_state_t post_state;
    gni_return_t ugni_rc;

    pthread_mutex_lock(&uct_ugni_global_lock);
    ugni_rc = GNI_PostDataProbeById(iface->super.nic_handle, &id);
    if (ucs_unlikely(GNI_RC_SUCCESS != ugni_rc)) {
        if (GNI_RC_NO_MATCH != ugni_rc) {
            ucs_error("GNI_PostDataProbeById , Error status: %s %d",
                      gni_err_str[ugni_rc], ugni_rc);
        }
        goto exit;
    }

    if (UCT_UGNI_UDT_ANY == id) {
        /* New incomming message */
        ep = NULL;
        ugni_ep = iface->ep_any;
        desc = iface->desc_any;
    } else {
        /* Ack message */
        ep = ucs_derived_of(uct_ugni_iface_lookup_ep(&iface->super, id),
                            uct_ugni_udt_ep_t);
        if (ucs_unlikely(NULL == ep)) {
            ucs_error("Can not lookup ep with id %"PRIx64,id);
            goto exit;
        }
        ugni_ep = ep->super.ep;
        desc = ep->posted_desc;
    }

    ugni_rc = GNI_EpPostDataWaitById(ugni_ep, id, -1, &post_state, &rem_addr, &rem_id);
    if (ucs_unlikely(GNI_RC_SUCCESS != ugni_rc)) {
        ucs_error("GNI_EpPostDataWaitById, Error status: %s %d",
                  gni_err_str[ugni_rc], ugni_rc);
        goto exit;
    }

    header = uct_ugni_udt_get_rheader(desc, iface);
    payload = uct_ugni_udt_get_rpayload(desc, iface);
    user_desc = uct_ugni_udt_get_user_desc(desc, iface);

    if (UCT_UGNI_UDT_ANY == id) {
        /* New incomming message */
        ucs_assert_always(header->type == UCT_UGNI_UDT_PAYLOAD);
        uct_iface_trace_am(&iface->super.super, UCT_AM_TRACE_TYPE_RECV,
                           header->am_id, payload, header->length, "RX: AM");
        status = uct_iface_invoke_am(&iface->super.super, header->am_id, payload,
                                     header->length, user_desc);
        if (UCS_OK != status) {
            uct_ugni_udt_desc_t *new_desc;
            /* set iface for a later release call */
            uct_recv_desc_iface(user_desc) = &iface->super.super.super;
            /* Allocate a new element */
            UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_desc,
                                     new_desc, goto exit);
            /* set the new desc */
            iface->desc_any = new_desc;
        }