예제 #1
0
int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey,
                                    void *write_to_buf)
{
    int mpi_errno = MPI_SUCCESS;
    int ibcom_errno;
    struct MPIDI_VC *vc = req->ch.vc;
    MPID_nem_ib_vc_area *vc_ib = VC_IB(vc);

    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_START_RECV_CORE);
    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_START_RECV_CORE);

    ibcom_errno =
        MPID_nem_ib_com_lrecv(vc_ib->sc->fd, (uint64_t) req, raddr, req->ch.lmt_data_sz, rkey,
                              write_to_buf);
    MPID_nem_ib_ncqe += 1;
    //dprintf("start_recv,ncqe=%d\n", MPID_nem_ib_ncqe);
    MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lrecv");
    dprintf("lmt_start_recv_core,MPID_nem_ib_ncqe=%d\n", MPID_nem_ib_ncqe);
    dprintf
        ("lmt_start_recv_core,req=%p,sz=%ld,write_to_buf=%p,lmt_pack_buf=%p,user_buf=%p,raddr=%p,rkey=%08x,tail=%p=%02x\n",
         req, req->ch.lmt_data_sz, write_to_buf, REQ_FIELD(req, lmt_pack_buf), req->dev.user_buf,
         raddr, rkey, write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t),
         *((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t))));

#ifdef MPID_NEM_IB_LMT_GET_CQE
    MPID_nem_ib_ncqe_to_drain += 1;     /* use CQE instead of polling */
#else
    /* drain_scq and ib_poll is not ordered, so both can decrement ref_count */
    MPIR_Request_add_ref(req);

    /* register to poll list in ib_poll() */
    /* don't use req->dev.next because it causes unknown problem */
    MPID_nem_ib_lmtq_enqueue(&MPID_nem_ib_lmtq, req);
    dprintf("lmt_start_recv_core,lmtq enqueue\n");
    //volatile uint8_t* tailmagic = (uint8_t*)((void*)req->dev.user_buf + req->ch.lmt_data_sz - sizeof(uint8_t));
    //dprintf("start_recv_core,cur_tail=%02x,lmt_receiver_tail=%02x\n", *tailmagic, REQ_FIELD(req, lmt_receiver_tail));
#endif

  fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_START_RECV_CORE);
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}
예제 #2
0
int MPID_nem_ib_lmt_start_recv_core(struct MPID_Request *req, void *raddr, uint32_t rkey, long len,
                                    void *write_to_buf, uint32_t max_msg_sz, int end)
{
    int mpi_errno = MPI_SUCCESS;
    int ibcom_errno;
    struct MPIDI_VC *vc = req->ch.vc;
    MPID_nem_ib_vc_area *vc_ib = VC_IB(vc);
    int i;
    int divide;
    int posted_num;
    int last;
    uint32_t r_max_msg_sz;      /* responder's max_msg_sz */
    void *write_pos;
    void *addr;
    long data_sz;
    MPIDI_msg_sz_t rest_data_sz;

    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_IB_LMT_START_RECV_CORE);
    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_IB_LMT_START_RECV_CORE);

    MPID_nem_ib_com_get_info_conn(vc_ib->sc->fd, MPID_NEM_IB_COM_INFOKEY_PATTR_MAX_MSG_SZ,
                                  &r_max_msg_sz, sizeof(uint32_t));

    divide = (max_msg_sz + r_max_msg_sz - 1) / r_max_msg_sz;

    write_pos = write_to_buf;
    posted_num = 0;
    last = MPID_NEM_IB_LMT_PART_OF_SEGMENT;
    rest_data_sz = len;
    addr = raddr;

    for (i = 0; i < divide; i++) {
        if (i == divide - 1)
            data_sz = max_msg_sz - i * r_max_msg_sz;
        else
            data_sz = r_max_msg_sz;

        if (i == divide - 1) {
            if (end)
                last = MPID_NEM_IB_LMT_LAST_PKT;        /* last part of last segment packet */
            else
                last = MPID_NEM_IB_LMT_SEGMENT_LAST;    /* last part of this segment */

            /* last data may be smaller than initiator's max_msg_sz */
            if (rest_data_sz < max_msg_sz)
                data_sz = rest_data_sz;
        }

        ibcom_errno =
            MPID_nem_ib_com_lrecv(vc_ib->sc->fd, (uint64_t) req, addr, data_sz, rkey,
                                  write_pos, last);
        MPIU_ERR_CHKANDJUMP(ibcom_errno, mpi_errno, MPI_ERR_OTHER, "**MPID_nem_ib_com_lrecv");

        /* update position */
        write_pos = (void *) ((char *) write_pos + data_sz);
        addr = (void *) ((char *) addr + data_sz);

        /* update rest data size */
        rest_data_sz -= data_sz;

        /* count request number */
        posted_num++;
    }

    MPIU_Assert(rest_data_sz == 0);
    MPID_nem_ib_ncqe += posted_num;
    //dprintf("start_recv,ncqe=%d\n", MPID_nem_ib_ncqe);
    dprintf("lmt_start_recv_core,MPID_nem_ib_ncqe=%d\n", MPID_nem_ib_ncqe);
    dprintf
        ("lmt_start_recv_core,req=%p,sz=%ld,write_to_buf=%p,lmt_pack_buf=%p,user_buf=%p,raddr=%p,rkey=%08x,tail=%p=%02x\n",
         req, req->ch.lmt_data_sz, write_to_buf, REQ_FIELD(req, lmt_pack_buf), req->dev.user_buf,
         raddr, rkey, write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t),
         *((uint8_t *) (write_to_buf + req->ch.lmt_data_sz - sizeof(uint8_t))));
    //fflush(stdout);

#ifdef MPID_NEM_IB_LMT_GET_CQE
    MPID_nem_ib_ncqe_to_drain += posted_num;    /* use CQE instead of polling */
#else
    /* drain_scq and ib_poll is not ordered, so both can decrement ref_count */
    MPIR_Request_add_ref(req);

    /* register to poll list in ib_poll() */
    /* don't use req->dev.next because it causes unknown problem */
    MPID_nem_ib_lmtq_enqueue(&MPID_nem_ib_lmtq, req);
    dprintf("lmt_start_recv_core,lmtq enqueue\n");
    //volatile uint8_t* tailmagic = (uint8_t*)((void*)req->dev.user_buf + req->ch.lmt_data_sz - sizeof(uint8_t));
    //dprintf("start_recv_core,cur_tail=%02x,lmt_receiver_tail=%02x\n", *tailmagic, REQ_FIELD(req, lmt_receiver_tail));
#endif

  fn_exit:
    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_IB_LMT_START_RECV_CORE);
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}