int on_connection(struct rdma_cm_id *id)
{
on_connect(id->context);
send_mr(id->context);
return 0;
}
void on_completion(struct ibv_wc *wc)
{
struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id;
if (wc->status != IBV_WC_SUCCESS)
die("on_completion: status is not IBV_WC_SUCCESS.");
if (wc->opcode & IBV_WC_RECV) {
conn->recv_state++;
printf("RECV: Recieved: TYPE=%d\n", conn->recv_msg->type);
if (conn->recv_msg->type == MSG_MR) {
memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr));
post_receives(conn); /* only rearm for MSG_MR */
if (conn->send_state == SS_INIT) /* received peer's MR before sending ours, so send ours back */
send_mr(conn);
}
} else {
conn->send_state++;
printf("SEND: Sent out: TYPE=%d\n", conn->send_msg->type);
}
if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) {
/*
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
if (s_mode == M_WRITE)
printf(" -> received MSG_MR. writing message to remote memory...\n");
else
printf(" -> received MSG_MR. reading message from remote memory...\n");
memset(&wr, 0, sizeof(wr));
wr.wr_id = (uintptr_t)conn;
wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr;
wr.wr.rdma.rkey = conn->peer_mr.rkey;
sge.addr = (uintptr_t)conn->rdma_local_region;
sge.length = RDMA_BUFFER_SIZE;
sge.lkey = conn->rdma_local_mr->lkey;
TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
printf("PSEND: Posted send request: MSG=%s\n", conn->rdma_local_region);
*/
conn->send_msg->type = MSG_DONE;
send_message(conn);
} else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) {
printf(" -> remote buffer: %s\n", get_peer_message_region(conn));
rdma_disconnect(conn->id);
}
printf("== STATE: send=%d / recv=%d ==\n", conn->send_state, conn->recv_state);
}
int client_on_connection(struct rdma_cm_id *id)
{
on_connect(id->context);
send_mr(id->context);
/// if built connection, return 1, and break the while in wait_rdma_event()
return 1;
}
static void
on_completion(struct ibv_wc *wc, bool is_server)
{
IbvConnection *conn = (IbvConnection *)(uintptr_t)wc->wr_id;
check_msgsanity(wc);
switch (wc->opcode) {
case IBV_WC_RECV:
switch (conn->recv_msg->type) {
case MSG_MR:
memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr));
// server receives MR before sending ours, so send ours back.
if (is_server) {
send_mr(conn);
}
conn->rdma_state = STATE_READY;
break;
case MSG_DONE:
break;
default:
fprintf(stderr, "received a message of unknown type.\n");
exit(EXIT_FAILURE);
}
post_receives(conn); // rearm for next message.
break;
case IBV_WC_SEND:
// nop.
break;
case IBV_WC_RDMA_WRITE:
switch (conn->rdma_state) {
case STATE_READY:
break;
case STATE_BUSY0:
conn->rdma_state = STATE_BUSY1;
break;
case STATE_BUSY1:
conn->rdma_state = STATE_READY;
break;
default:
fprintf(stderr, "invalid conn->rdma_state:%d\n", conn->rdma_state);
exit(1);
}
break;
default:
fprintf(stderr, "completion with unexpected opcode : 0x%x\n", wc->opcode);
exit(1);
}
}
void on_completion(struct ibv_wc *wc)
{
struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id;
if (wc->status != IBV_WC_SUCCESS)
die("on_completion: status is not IBV_WC_SUCCESS.");
if (wc->opcode & IBV_WC_RECV) {
conn->recv_state++;
if (conn->recv_msg->type == MSG_MR) {
memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr));
post_receives(conn); /* only rearm for MSG_MR */
if (conn->send_state == SS_INIT) /* received peer's MR before sending ours, so send ours back */
send_mr(conn);
}
} else {
conn->send_state++;
printf("send completed successfully.\n");
}
if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) {
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
if (s_mode == M_WRITE)
printf("received MSG_MR. writing message to remote memory...\n");
else
printf("received MSG_MR. reading message from remote memory...\n");
memset(&wr, 0, sizeof(wr));
wr.wr_id = (uintptr_t)conn;
wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr;
wr.wr.rdma.rkey = conn->peer_mr.rkey;
sge.addr = (uintptr_t)conn->rdma_local_region;
sge.length = RDMA_BUFFER_SIZE;
sge.lkey = conn->rdma_local_mr->lkey;
/* CODE TO MESS UP PSN */
srand48(getpid());
struct ibv_qp_attr attr;
attr.sq_psn = lrand48() & 0xffffff;
attr.rq_psn = lrand48() & 0xffffff;
if (ibv_modify_qp(conn->qp, &attr, IBV_QP_RQ_PSN | IBV_QP_SQ_PSN)) {
fprintf(stderr, "Failed to set the PSN.");
return;
}
/* END CODE TO MESS UP PSN */
TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
conn->send_msg->type = MSG_DONE;
send_message(conn);
} else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) {
printf("remote buffer: %s\n", get_peer_message_region(conn));
rdma_disconnect(conn->id);
}
}
void on_connection(struct rdma_cm_id *id)
{
send_mr(id->context);
return;//exit while loop
}
static int run(int argc, char **argv)
{
struct addrinfo *addr;
//struct rdma_cm_event *event = NULL;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
if (argc != 4)
usage(argv[0]);
if (strcmp(argv[1], "write") == 0)
set_mode(M_WRITE);
else if (strcmp(argv[1], "read") == 0)
set_mode(M_READ);
else
usage(argv[0]);
TEST_NZ(getaddrinfo(argv[2], argv[3], NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
/*create rdma socket*/
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
/* int rdma_resolve_addr (struct rdma_cm_id *id, struct sockaddr *src_addr,
struct sockaddr dst_addr, int timeout_ms)
id RDMA identifier
src_addr Source address information. This parameter may be NULL.
dst_addr Destination address information
timeout_ms Time to wait for resolution to complete
Description:
Resolve destination and optional source addresses from IP addresses
to an RDMA address. If suc- cessful,
the specified rdma_cm_id will be bound to a local device.
*/
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
sprintf(get_local_message_region(cmid->context), "message from active/client side with pid %d", getpid());
/*--------------------*/
/* int rdma_resolve_route (struct rdma_cm_id *id, int timeout_ms);
id RDMA identifier
timeout_ms Time to wait for resolution to complete
Description:
Resolves an RDMA route to the destination address in order
to establish a connection. The destination address must have
already been resolved by calling rdma_resolve_addr.
*/
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
/* -------------------- */
print_path_rec(cmid);
/* int rdma_connect (struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
id RDMA identifier
conn_param connection parameters
Description:
For an rdma_cm_id of type RDMA_PS_TCP, this call initiates a connection
request to a remote destination. For an rdma_cm_id of type RDMA_PS_UDP,
it initiates a lookup of the remote QP providing the datagram service
*/
build_params(&cm_params);
printf("Connecting ...\n");
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
printf("Connected !\n");
/* --------------------- */
/*TODO: do something */
on_connect(cmid->context);
send_mr(cmid->context);
/*--------------------*/
rdma_disconnect(cmid);
rdma_destroy_id(cmid);
rdma_destroy_event_channel(ec);
return 0;
/*=================*/
/*=================*/
/*
while (rdma_get_cm_event(ec, &event) == 0) {
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
*/
}
void on_completion(struct ibv_wc *wc)
{
struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id;
if (wc->status != IBV_WC_SUCCESS)
die("on_completion: status is not IBV_WC_SUCCESS.");
if (wc->opcode == IBV_WC_RDMA_WRITE) {
e[idx] = get_dtime();
int *v = (int*)conn->rdma_local_region;
printf("RDMA Write: %f (v=%d)\n", e[idx] - s[idx], v[0]);
idx++;
return;
} else if (wc->opcode & IBV_WC_RDMA_READ) {
// printf("RDMA Read: %f\n", e - s);
return;
}
if (wc->opcode & IBV_WC_RECV) {
conn->recv_state++;
if (conn->recv_msg->type == MSG_MR) {
memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr));
post_receives(conn); /* only rearm for MSG_MR */
if (conn->send_state == SS_INIT) /* received peer's MR before sending ours, so send ours back */
send_mr(conn);
}
} else {
conn->send_state++;
printf("send completed successfully.\n");
}
if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) {
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
if (s_mode == M_WRITE)
printf("received MSG_MR. writing message to remote memory...\n");
else
printf("received MSG_MR. reading message from remote memory...\n");
memset(&wr, 0, sizeof(wr));
wr.wr_id = (uintptr_t)conn;
wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr;
wr.wr.rdma.rkey = conn->peer_mr.rkey;
sge.addr = (uintptr_t)conn->rdma_local_region;
sge.length = RDMA_BUFFER_SIZE;
sge.lkey = conn->rdma_local_mr->lkey;
int *v = (int*)conn->peer_mr.addr;
v[0] = 15;
int i;
for (i = 0; i < CNUM; i++) {
s[i] = get_dtime();
TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
}
v[0] = 11;
conn->send_msg->type = MSG_DONE;
send_message(conn);
} else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) {
printf("remote buffer: %s\n", get_peer_message_region(conn));
rdma_disconnect(conn->id);
}
}
