void on_completion(struct ibv_wc *wc) { struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id; if (wc->status != IBV_WC_SUCCESS) die("on_completion: status is not IBV_WC_SUCCESS."); if (wc->opcode & IBV_WC_RECV) { conn->recv_state++; printf("RECV: Recieved: TYPE=%d\n", conn->recv_msg->type); if (conn->recv_msg->type == MSG_MR) { memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr)); post_receives(conn); /* only rearm for MSG_MR */ if (conn->send_state == SS_INIT) /* received peer's MR before sending ours, so send ours back */ send_mr(conn); } } else { conn->send_state++; printf("SEND: Sent out: TYPE=%d\n", conn->send_msg->type); } if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) { /* struct ibv_send_wr wr, *bad_wr = NULL; struct ibv_sge sge; if (s_mode == M_WRITE) printf(" -> received MSG_MR. writing message to remote memory...\n"); else printf(" -> received MSG_MR. reading message from remote memory...\n"); memset(&wr, 0, sizeof(wr)); wr.wr_id = (uintptr_t)conn; wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ; wr.sg_list = &sge; wr.num_sge = 1; wr.send_flags = IBV_SEND_SIGNALED; wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr; wr.wr.rdma.rkey = conn->peer_mr.rkey; sge.addr = (uintptr_t)conn->rdma_local_region; sge.length = RDMA_BUFFER_SIZE; sge.lkey = conn->rdma_local_mr->lkey; TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr)); printf("PSEND: Posted send request: MSG=%s\n", conn->rdma_local_region); */ conn->send_msg->type = MSG_DONE; send_message(conn); } else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) { printf(" -> remote buffer: %s\n", get_peer_message_region(conn)); rdma_disconnect(conn->id); } printf("== STATE: send=%d / recv=%d ==\n", conn->send_state, conn->recv_state); }
void on_completion(struct ibv_wc *wc) { struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id; if (wc->status != IBV_WC_SUCCESS) die("on_completion: status is not IBV_WC_SUCCESS."); if (wc->opcode & IBV_WC_RECV) { conn->recv_state++; if (conn->recv_msg->type == MSG_MR) { memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr)); post_receives(conn); /* only rearm for MSG_MR */ if (conn->send_state == SS_INIT) /* received peer's MR before sending ours, so send ours back */ send_mr(conn); } } else { conn->send_state++; printf("send completed successfully.\n"); } if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) { struct ibv_send_wr wr, *bad_wr = NULL; struct ibv_sge sge; if (s_mode == M_WRITE) printf("received MSG_MR. writing message to remote memory...\n"); else printf("received MSG_MR. reading message from remote memory...\n"); memset(&wr, 0, sizeof(wr)); wr.wr_id = (uintptr_t)conn; wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ; wr.sg_list = &sge; wr.num_sge = 1; wr.send_flags = IBV_SEND_SIGNALED; wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr; wr.wr.rdma.rkey = conn->peer_mr.rkey; sge.addr = (uintptr_t)conn->rdma_local_region; sge.length = RDMA_BUFFER_SIZE; sge.lkey = conn->rdma_local_mr->lkey; /* CODE TO MESS UP PSN */ srand48(getpid()); struct ibv_qp_attr attr; attr.sq_psn = lrand48() & 0xffffff; attr.rq_psn = lrand48() & 0xffffff; if (ibv_modify_qp(conn->qp, &attr, IBV_QP_RQ_PSN | IBV_QP_SQ_PSN)) { fprintf(stderr, "Failed to set the PSN."); return; } /* END CODE TO MESS UP PSN */ TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr)); conn->send_msg->type = MSG_DONE; send_message(conn); } else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) { printf("remote buffer: %s\n", get_peer_message_region(conn)); rdma_disconnect(conn->id); } }
void on_completion(struct ibv_wc *wc) { struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id; if (wc->status != IBV_WC_SUCCESS) die("on_completion: status is not IBV_WC_SUCCESS."); if (wc->opcode == IBV_WC_RDMA_WRITE) { e[idx] = get_dtime(); int *v = (int*)conn->rdma_local_region; printf("RDMA Write: %f (v=%d)\n", e[idx] - s[idx], v[0]); idx++; return; } else if (wc->opcode & IBV_WC_RDMA_READ) { // printf("RDMA Read: %f\n", e - s); return; } if (wc->opcode & IBV_WC_RECV) { conn->recv_state++; if (conn->recv_msg->type == MSG_MR) { memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr)); post_receives(conn); /* only rearm for MSG_MR */ if (conn->send_state == SS_INIT) /* received peer's MR before sending ours, so send ours back */ send_mr(conn); } } else { conn->send_state++; printf("send completed successfully.\n"); } if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) { struct ibv_send_wr wr, *bad_wr = NULL; struct ibv_sge sge; if (s_mode == M_WRITE) printf("received MSG_MR. writing message to remote memory...\n"); else printf("received MSG_MR. reading message from remote memory...\n"); memset(&wr, 0, sizeof(wr)); wr.wr_id = (uintptr_t)conn; wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ; wr.sg_list = &sge; wr.num_sge = 1; wr.send_flags = IBV_SEND_SIGNALED; wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr; wr.wr.rdma.rkey = conn->peer_mr.rkey; sge.addr = (uintptr_t)conn->rdma_local_region; sge.length = RDMA_BUFFER_SIZE; sge.lkey = conn->rdma_local_mr->lkey; int *v = (int*)conn->peer_mr.addr; v[0] = 15; int i; for (i = 0; i < CNUM; i++) { s[i] = get_dtime(); TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr)); } v[0] = 11; conn->send_msg->type = MSG_DONE; send_message(conn); } else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) { printf("remote buffer: %s\n", get_peer_message_region(conn)); rdma_disconnect(conn->id); } }