RDMACMSocket::~RDMACMSocket() {
rdma_disconnect(this->client_id);
rdma_dereg_mr(this->verbs_mr);
rdma_destroy_ep(this->client_id);
this->verbs_buf.free();
}
void RDMACMSocket::post_recv(const Buffer& buf) {
if (rdma_post_recv(this->client_id, buf.addr, buf.addr, buf.size, this->verbs_mr) < 0) {
rdma_dereg_mr(this->verbs_mr);
this->verbs_buf.free();
rdma_destroy_ep(this->client_id);
perror("rdma_post_recv");
exit(1);
}
}
RDMAWriteSocket::~RDMAWriteSocket() {
delete this->rsock;
rdma_dereg_mr(this->write_mr);
this->write_buf.free();
}
static int rc_test(void)
{
struct rdma_addrinfo hints, *res;
struct ibv_qp_init_attr attr;
struct ibv_wc wc;
int ret;
memset(&hints, 0, sizeof hints);
hints.ai_flags = RAI_PASSIVE;
hints.ai_port_space = RDMA_PS_TCP;
ret = rdma_getaddrinfo(NULL, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo %d\n", errno);
return ret;
}
memset(&attr, 0, sizeof attr);
attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = sizeof send_msg;
attr.sq_sig_all = 1;
ret = rdma_create_ep(&listen_id, res, NULL, &attr);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep %d\n", errno);
return ret;
}
ret = rdma_listen(listen_id, 0);
if (ret) {
printf("rdma_listen %d\n", errno);
return ret;
}
ret = rdma_get_request(listen_id, &id);
if (ret) {
printf("rdma_get_request %d\n", errno);
return ret;
}
mr = rdma_reg_msgs(id, recv_msg, sizeof recv_msg);
if (!mr) {
printf("rdma_reg_msgs %d\n", errno);
return ret;
}
ret = rdma_post_recv(id, NULL, recv_msg, sizeof recv_msg, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
ret = rdma_accept(id, NULL);
if (ret) {
printf("rdma_accept %d\n", errno);
return ret;
}
ret = rdma_get_recv_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_recv_comp %d\n", ret);
return ret;
}
ret = rdma_post_send(id, NULL, send_msg, sizeof send_msg, NULL, IBV_SEND_INLINE);
if (ret) {
printf("rdma_post_send %d\n", errno);
return ret;
}
ret = rdma_get_send_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_send_comp %d\n", ret);
return ret;
}
rdma_disconnect(id);
rdma_dereg_mr(mr);
rdma_destroy_ep(id);
rdma_destroy_ep(listen_id);
return 0;
}
static int xrc_test(void)
{
struct rdma_cm_id *conn_id, *lookup_id;
struct ibv_qp_init_attr attr;
struct rdma_conn_param param;
struct rdma_cm_event *event;
struct ibv_wc wc;
int ret;
conn_id = xrc_listen_recv();
if (!conn_id)
return -1;
ret = xrc_create_srq_listen(rdma_get_local_addr(conn_id),
sizeof(struct sockaddr_storage));
if (ret)
return -1;
memset(&attr, 0, sizeof attr);
attr.qp_type = IBV_QPT_XRC_RECV;
attr.ext.xrc_recv.xrcd = srq_id->srq->ext.xrc.xrcd;
ret = rdma_create_qp(conn_id, NULL, &attr);
if (ret) {
printf("Unable to create xrc recv qp %d\n", errno);
return ret;
}
ret = rdma_accept(conn_id, NULL);
if (ret) {
printf("rdma_accept failed for xrc recv qp %d\n", errno);
return ret;
}
ret = rdma_get_request(srq_id, &lookup_id);
if (ret) {
printf("rdma_get_request %d\n", errno);
return ret;
}
mr = rdma_reg_msgs(srq_id, recv_msg, sizeof recv_msg);
if (!mr) {
printf("ibv_reg_msgs %d\n", errno);
return ret;
}
ret = rdma_post_recv(srq_id, NULL, recv_msg, sizeof recv_msg, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
memset(¶m, 0, sizeof param);
param.qp_num = srq_id->srq->ext.xrc.srq_num;
ret = rdma_accept(lookup_id, ¶m);
if (ret) {
printf("rdma_accept failed for srqn lookup %d\n", errno);
return ret;
}
rdma_destroy_id(lookup_id);
ret = rdma_get_recv_comp(srq_id, &wc);
if (ret <= 0) {
printf("rdma_get_recv_comp %d\n", ret);
return ret;
}
ret = rdma_get_cm_event(conn_id->channel, &event);
if (ret || event->event != RDMA_CM_EVENT_DISCONNECTED) {
printf("Failed to get disconnect event\n");
return -1;
}
rdma_ack_cm_event(event);
rdma_disconnect(conn_id);
rdma_destroy_ep(conn_id);
rdma_dereg_mr(mr);
rdma_destroy_ep(srq_id);
rdma_destroy_ep(listen_id);
return 0;
}
static int run(void)
{
struct rdma_addrinfo hints, *res;
struct ibv_qp_init_attr attr;
struct ibv_wc wc;
int ret;
memset(&hints, 0, sizeof hints);
hints.ai_port_space = RDMA_PS_TCP;
ret = rdma_getaddrinfo(server, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo %d\n", errno);
return ret;
}
memset(&attr, 0, sizeof attr);
attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = 16;
attr.qp_context = id;
attr.sq_sig_all = 1;
ret = rdma_create_ep(&id, res, NULL, &attr);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep %d\n", errno);
return ret;
}
mr = rdma_reg_msgs(id, recv_msg, 16);
if (!mr) {
printf("rdma_reg_msgs %d\n", errno);
return ret;
}
ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
ret = rdma_connect(id, NULL);
if (ret) {
printf("rdma_connect %d\n", errno);
return ret;
}
s = get_dtime();
ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
if (ret) {
printf("rdma_post_send %d\n", errno);
return ret;
}
e = get_dtime();
ret = rdma_get_recv_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_recv_comp %d\n", ret);
return ret;
}
printf("time %f\n", e - s);
rdma_disconnect(id);
rdma_dereg_mr(mr);
rdma_destroy_ep(id);
return 0;
}
