Exemplo n.º 1
0
int
main()
{
	int    ib_port = 1;
	int    gid_idx = 1;
	int    rc;
	int    rank, nprocs;
	struct ibv_sge  sge_list;
	struct ibv_wc  wc;
	struct ibv_send_wr  *sr;
	unsigned long long start, end;
	float time;

	mypmiInit(&rank, &nprocs);
	fprintf(stderr, "[%d] nprocs(%d)\n", rank, nprocs);
	rc = resource_create(&res, ib_port, rank);
	gid_idx = rank;
	rc = connect_qp(&res, ib_port, gid_idx, rank);
	create_sge(&res, buf, SIZE, &sge_list);
	memset(&wc, 0, sizeof(struct ibv_wc));
	sr = malloc(sizeof(*sr));
	memset(sr, 0, sizeof(*sr));
	mypmiBarrier();
	fprintf(stderr, "[%d] START\n", rank);
	memset(buf, 0, SIZE);

	mypmiBarrier();

	if (rank == 0) {
		struct ibv_mr *mr;
		for (int size = RDMA_MIN_SIZE; size < RDMA_MAX_SIZE; size += STEP) {
			char *received = calloc(size, sizeof(char));

			mr = ibv_reg_mr(res.pd, received, size, IBV_ACCESS_REMOTE_WRITE |  IBV_ACCESS_LOCAL_WRITE);

			INT_TO_BE(buf, mr->rkey);
			INT_TO_BE(buf + 4, (((intptr_t)mr->addr) >> 32));
			INT_TO_BE(buf + 8, (((intptr_t)mr->addr) & 0xffffffff));
			if (post_ibsend(&res, IBV_WR_SEND, &sge_list, sr, 1)) {
				fprintf(stderr, "[%d] failed to post SR\n", rank);
				goto end;
			}
			while ((rc = poll_cq(&res, &wc, 1, SCQ_FLG)) == 0) {
			}
			/* printf("[%d] memory region is sent. key(%x) addr(%lx) rc(%d)\n", rank, mr->rkey, (intptr_t)mr->addr, rc); */

			/* wait for done */
			post_ibreceive(&res, &sge_list, 1);
			while (poll_cq(&res, &wc, 1, RCQ_FLG) == 0) {
			}
			/* printf("[%d] %d byte has received (opcode=%d)\n", rank, wc.byte_len, wc.opcode); */
			/* printf("[%d] Received message: %s\n", rank, buf); */
			/* display_received(received, size); */

			ibv_dereg_mr(mr);
			free(received);
		}
	} else {
Exemplo n.º 2
0
void client_write_once(uint32_t len){
    struct timeval start, end, dt;
    gettimeofday(&start, NULL);
    struct connection *conn = (struct connection *)s_ctx->id->context;
    post_receive(conn);
    uint32_t size = len|(1UL<<31);
    write_remote(conn,size);
    poll_cq(NULL);//wait for write completion
    poll_cq(NULL);//wait for recv completion
    gettimeofday(&end, NULL);
    timersub(&end, &start, &dt);
    long usec = dt.tv_usec + 1000000 * dt.tv_sec;
    printf("[Wriet] takes %ld micro_secs.\n", usec);
}
Exemplo n.º 3
0
void openib_waitall()
{
    // Poll the CQ for all outstanding data transfer
    int rc = poll_cq(l_state.num_outstanding, 0);
    assert(!rc);
    
    l_state.num_outstanding = 0;
}
Exemplo n.º 4
0
void client_test(char *ip, char *port) {


    struct addrinfo *addr;
    struct rdma_cm_event *event = NULL;
    struct rdma_cm_id *conn= NULL;
    struct rdma_event_channel *ec = NULL;

    TEST_NZ(getaddrinfo(ip, port, NULL, &addr));

    TEST_Z(ec = rdma_create_event_channel());
    TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
    TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));

    freeaddrinfo(addr);

    while (rdma_get_cm_event(ec, &event) == 0) {
        struct rdma_cm_event event_copy;

        memcpy(&event_copy, event, sizeof(*event));
        rdma_ack_cm_event(event);

        if (on_event(&event_copy)) {
            s_ctx->ec = ec;
            s_ctx->id = conn;
            on_connection(event_copy.id);//send our memory information to server using post_send
            //printf("wait for msg_send_completion\n");
            poll_cq(NULL);//wait for send_completion
            //printf("wait for msg_recv_completion\n");
            poll_cq(NULL);//wait for recv_completion
            break;
        }
    }
    return;

};
Exemplo n.º 5
0
/*
 * Get one cq entry from c4iw and map it to openib.
 *
 * Returns:
 *	0			cqe returned
 *	-ENODATA		EMPTY;
 *	-EAGAIN			caller must try again
 *	any other -errno	fatal error
 */
static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
{
	struct c4iw_qp *qhp = NULL;
	struct t4_cqe cqe = {0, 0}, *rd_cqe;
	struct t4_wq *wq;
	u32 credit = 0;
	u8 cqe_flushed;
	u64 cookie = 0;
	int ret;

	ret = t4_next_cqe(&chp->cq, &rd_cqe);

	if (ret)
		return ret;

	qhp = get_qhp(chp->rhp, CQE_QPID(rd_cqe));
	if (!qhp)
		wq = NULL;
	else {
		spin_lock(&qhp->lock);
		wq = &(qhp->wq);
	}
	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit);
	if (ret)
		goto out;

	wc->wr_id = cookie;
	wc->qp = &qhp->ibqp;
	wc->vendor_err = CQE_STATUS(&cqe);
	wc->wc_flags = 0;

	PDBG("%s qpid 0x%x type %d opcode %d status 0x%x len %u wrid hi 0x%x "
	     "lo 0x%x cookie 0x%llx\n", __func__, CQE_QPID(&cqe),
	     CQE_TYPE(&cqe), CQE_OPCODE(&cqe), CQE_STATUS(&cqe), CQE_LEN(&cqe),
	     CQE_WRID_HI(&cqe), CQE_WRID_LOW(&cqe), (unsigned long long)cookie);

	if (CQE_TYPE(&cqe) == 0) {
		if (!CQE_STATUS(&cqe))
			wc->byte_len = CQE_LEN(&cqe);
		else
			wc->byte_len = 0;
		wc->opcode = IB_WC_RECV;
		if (CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_INV ||
		    CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_SE_INV) {
			wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe);
			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
		}
	} else {
		switch (CQE_OPCODE(&cqe)) {
		case FW_RI_RDMA_WRITE:
			wc->opcode = IB_WC_RDMA_WRITE;
			break;
		case FW_RI_READ_REQ:
			wc->opcode = IB_WC_RDMA_READ;
			wc->byte_len = CQE_LEN(&cqe);
			break;
		case FW_RI_SEND_WITH_INV:
		case FW_RI_SEND_WITH_SE_INV:
			wc->opcode = IB_WC_SEND;
			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
			break;
		case FW_RI_SEND:
		case FW_RI_SEND_WITH_SE:
			wc->opcode = IB_WC_SEND;
			break;
		case FW_RI_BIND_MW:
			wc->opcode = IB_WC_BIND_MW;
			break;

		case FW_RI_LOCAL_INV:
			wc->opcode = IB_WC_LOCAL_INV;
			break;
		case FW_RI_FAST_REGISTER:
			wc->opcode = IB_WC_FAST_REG_MR;
			break;
		default:
			printk(KERN_ERR MOD "Unexpected opcode %d "
			       "in the CQE received for QPID=0x%0x\n",
			       CQE_OPCODE(&cqe), CQE_QPID(&cqe));
			ret = -EINVAL;
			goto out;
		}
	}

	if (cqe_flushed)
		wc->status = IB_WC_WR_FLUSH_ERR;
	else {

		switch (CQE_STATUS(&cqe)) {
		case T4_ERR_SUCCESS:
			wc->status = IB_WC_SUCCESS;
			break;
		case T4_ERR_STAG:
			wc->status = IB_WC_LOC_ACCESS_ERR;
			break;
		case T4_ERR_PDID:
			wc->status = IB_WC_LOC_PROT_ERR;
			break;
		case T4_ERR_QPID:
		case T4_ERR_ACCESS:
			wc->status = IB_WC_LOC_ACCESS_ERR;
			break;
		case T4_ERR_WRAP:
			wc->status = IB_WC_GENERAL_ERR;
			break;
		case T4_ERR_BOUND:
			wc->status = IB_WC_LOC_LEN_ERR;
			break;
		case T4_ERR_INVALIDATE_SHARED_MR:
		case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND:
			wc->status = IB_WC_MW_BIND_ERR;
			break;
		case T4_ERR_CRC:
		case T4_ERR_MARKER:
		case T4_ERR_PDU_LEN_ERR:
		case T4_ERR_OUT_OF_RQE:
		case T4_ERR_DDP_VERSION:
		case T4_ERR_RDMA_VERSION:
		case T4_ERR_DDP_QUEUE_NUM:
		case T4_ERR_MSN:
		case T4_ERR_TBIT:
		case T4_ERR_MO:
		case T4_ERR_MSN_RANGE:
		case T4_ERR_IRD_OVERFLOW:
		case T4_ERR_OPCODE:
		case T4_ERR_INTERNAL_ERR:
			wc->status = IB_WC_FATAL_ERR;
			break;
		case T4_ERR_SWFLUSH:
			wc->status = IB_WC_WR_FLUSH_ERR;
			break;
		default:
			printk(KERN_ERR MOD
			       "Unexpected cqe_status 0x%x for QPID=0x%0x\n",
			       CQE_STATUS(&cqe), CQE_QPID(&cqe));
			ret = -EINVAL;
		}
	}
out:
	if (wq)
		spin_unlock(&qhp->lock);
	return ret;
}
Exemplo n.º 6
0
Arquivo: cq.c Projeto: avagin/linux
static int __c4iw_poll_cq_one(struct c4iw_cq *chp, struct c4iw_qp *qhp,
			      struct ib_wc *wc, struct c4iw_srq *srq)
{
	struct t4_cqe uninitialized_var(cqe);
	struct t4_wq *wq = qhp ? &qhp->wq : NULL;
	u32 credit = 0;
	u8 cqe_flushed;
	u64 cookie = 0;
	int ret;

	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit,
		      srq ? &srq->wq : NULL);
	if (ret)
		goto out;

	wc->wr_id = cookie;
	wc->qp = qhp ? &qhp->ibqp : NULL;
	wc->vendor_err = CQE_STATUS(&cqe);
	wc->wc_flags = 0;

	/*
	 * Simulate a SRQ_LIMIT_REACHED HW notification if required.
	 */
	if (srq && !(srq->flags & T4_SRQ_LIMIT_SUPPORT) && srq->armed &&
	    srq->wq.in_use < srq->srq_limit)
		c4iw_dispatch_srq_limit_reached_event(srq);

	pr_debug("qpid 0x%x type %d opcode %d status 0x%x len %u wrid hi 0x%x lo 0x%x cookie 0x%llx\n",
		 CQE_QPID(&cqe),
		 CQE_TYPE(&cqe), CQE_OPCODE(&cqe),
		 CQE_STATUS(&cqe), CQE_LEN(&cqe),
		 CQE_WRID_HI(&cqe), CQE_WRID_LOW(&cqe),
		 (unsigned long long)cookie);

	if (CQE_TYPE(&cqe) == 0) {
		if (!CQE_STATUS(&cqe))
			wc->byte_len = CQE_LEN(&cqe);
		else
			wc->byte_len = 0;

		switch (CQE_OPCODE(&cqe)) {
		case FW_RI_SEND:
			wc->opcode = IB_WC_RECV;
			break;
		case FW_RI_SEND_WITH_INV:
		case FW_RI_SEND_WITH_SE_INV:
			wc->opcode = IB_WC_RECV;
			wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe);
			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
			c4iw_invalidate_mr(qhp->rhp, wc->ex.invalidate_rkey);
			break;
		case FW_RI_WRITE_IMMEDIATE:
			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
			wc->ex.imm_data = CQE_IMM_DATA(&cqe);
			wc->wc_flags |= IB_WC_WITH_IMM;
			break;
		default:
			pr_err("Unexpected opcode %d in the CQE received for QPID=0x%0x\n",
			       CQE_OPCODE(&cqe), CQE_QPID(&cqe));
			ret = -EINVAL;
			goto out;
		}
	} else {
		switch (CQE_OPCODE(&cqe)) {
		case FW_RI_WRITE_IMMEDIATE:
		case FW_RI_RDMA_WRITE:
			wc->opcode = IB_WC_RDMA_WRITE;
			break;
		case FW_RI_READ_REQ:
			wc->opcode = IB_WC_RDMA_READ;
			wc->byte_len = CQE_LEN(&cqe);
			break;
		case FW_RI_SEND_WITH_INV:
		case FW_RI_SEND_WITH_SE_INV:
			wc->opcode = IB_WC_SEND;
			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
			break;
		case FW_RI_SEND:
		case FW_RI_SEND_WITH_SE:
			wc->opcode = IB_WC_SEND;
			break;

		case FW_RI_LOCAL_INV:
			wc->opcode = IB_WC_LOCAL_INV;
			break;
		case FW_RI_FAST_REGISTER:
			wc->opcode = IB_WC_REG_MR;

			/* Invalidate the MR if the fastreg failed */
			if (CQE_STATUS(&cqe) != T4_ERR_SUCCESS)
				c4iw_invalidate_mr(qhp->rhp,
						   CQE_WRID_FR_STAG(&cqe));
			break;
		default:
			pr_err("Unexpected opcode %d in the CQE received for QPID=0x%0x\n",
			       CQE_OPCODE(&cqe), CQE_QPID(&cqe));
			ret = -EINVAL;
			goto out;
		}
	}

	if (cqe_flushed)
		wc->status = IB_WC_WR_FLUSH_ERR;
	else {

		switch (CQE_STATUS(&cqe)) {
		case T4_ERR_SUCCESS:
			wc->status = IB_WC_SUCCESS;
			break;
		case T4_ERR_STAG:
			wc->status = IB_WC_LOC_ACCESS_ERR;
			break;
		case T4_ERR_PDID:
			wc->status = IB_WC_LOC_PROT_ERR;
			break;
		case T4_ERR_QPID:
		case T4_ERR_ACCESS:
			wc->status = IB_WC_LOC_ACCESS_ERR;
			break;
		case T4_ERR_WRAP:
			wc->status = IB_WC_GENERAL_ERR;
			break;
		case T4_ERR_BOUND:
			wc->status = IB_WC_LOC_LEN_ERR;
			break;
		case T4_ERR_INVALIDATE_SHARED_MR:
		case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND:
			wc->status = IB_WC_MW_BIND_ERR;
			break;
		case T4_ERR_CRC:
		case T4_ERR_MARKER:
		case T4_ERR_PDU_LEN_ERR:
		case T4_ERR_OUT_OF_RQE:
		case T4_ERR_DDP_VERSION:
		case T4_ERR_RDMA_VERSION:
		case T4_ERR_DDP_QUEUE_NUM:
		case T4_ERR_MSN:
		case T4_ERR_TBIT:
		case T4_ERR_MO:
		case T4_ERR_MSN_RANGE:
		case T4_ERR_IRD_OVERFLOW:
		case T4_ERR_OPCODE:
		case T4_ERR_INTERNAL_ERR:
			wc->status = IB_WC_FATAL_ERR;
			break;
		case T4_ERR_SWFLUSH:
			wc->status = IB_WC_WR_FLUSH_ERR;
			break;
		default:
			pr_err("Unexpected cqe_status 0x%x for QPID=0x%0x\n",
			       CQE_STATUS(&cqe), CQE_QPID(&cqe));
			wc->status = IB_WC_FATAL_ERR;
		}
	}
out:
	return ret;
}
Exemplo n.º 7
0
void client_test(char *ip, char *port) {


    struct addrinfo *addr;
    struct rdma_cm_event *event = NULL;
    struct rdma_cm_id *conn= NULL;
    struct rdma_event_channel *ec = NULL;
    struct timeval t1, t2, t3, t11, t12;
    struct timeval dt, dt1, dt2, dt11, dt12, dt13;

    gettimeofday(&t1, NULL);

    TEST_NZ(getaddrinfo(ip, port, NULL, &addr));
    TEST_Z(ec = rdma_create_event_channel());
    gettimeofday(&t11, NULL);

    TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
    gettimeofday(&t12, NULL);

    TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
    freeaddrinfo(addr);
    gettimeofday(&t2, NULL);

    while (rdma_get_cm_event(ec, &event) == 0) {
        struct rdma_cm_event event_copy;

        memcpy(&event_copy, event, sizeof(*event));
        rdma_ack_cm_event(event);

        if (on_event(&event_copy)) {
            s_ctx->ec = ec;
            s_ctx->id = conn;


            on_connection(event_copy.id);//send our memory information to server using post_send
            poll_cq(NULL);//wait for send_completion
            poll_cq(NULL);//wait for recv_completion

            break;
        }
    }

    gettimeofday(&t3, NULL);
    timersub(&t3, &t1, &dt);
    timersub(&t3, &t2, &dt2);
    timersub(&t2, &t1, &dt1);
    timersub(&t2, &t12, &dt13);
    timersub(&t12, &t11, &dt12);
    timersub(&t11, &t1, &dt11);
    long usec = dt.tv_usec + 10000 * dt.tv_sec;

    printf("[dt]:\t%ld us.\n", usec);
    printf("[dt1]:\t%ld us.\n", dt1.tv_usec+1000000 *dt1.tv_sec);
    printf("Including the following steps: \n");
    printf("[dt11]:\t%ld us.\n", dt11.tv_usec+1000000 *dt11.tv_sec);
    printf("[dt12]:\t%ld us.\n", dt12.tv_usec+1000000 *dt12.tv_sec);
    printf("[dt13]:\t%ld us.\n", dt13.tv_usec+1000000 *dt13.tv_sec);
    printf("[dt2] takes %ld micro_secs.\n", dt2.tv_usec+1000000*dt2.tv_sec);
    printf("[dt]:total time\t[dt1]:pre_setup\t[dt2]:send/recv\t.\n");
    printf("[dt11]:create_event_channel\t[dt12]:create_id\t[dt13]:resolve_address.\n");
    return;

};
Exemplo n.º 8
0
void iterate_interface()
{
	post_one_read();
	poll_cq(1);

}