int main(int argc, char **argv)
{
struct sockaddr_in addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *listener = NULL;
struct rdma_event_channel *ec = NULL;
uint16_t port = 0;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_bind_addr(listener, (struct sockaddr *)&addr));
TEST_NZ(rdma_listen(listener, 10)); /* backlog=10 is arbitrary */
port = ntohs(rdma_get_src_port(listener));
printf("listening on port %d.\n", port);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_id(listener);
rdma_destroy_event_channel(ec);
return 0;
}
static void build_verbs(IbvConnection *conn, struct ibv_context *verbs)
{
conn->ibvctx = verbs;
TEST_Z(conn->pd = ibv_alloc_pd(conn->ibvctx));
TEST_Z(conn->comp_channel = ibv_create_comp_channel(conn->ibvctx));
TEST_Z(conn->cq = ibv_create_cq(conn->ibvctx, 10, NULL, conn->comp_channel, 0)); /* cqe=10 is arbitrary */
TEST_NZ(ibv_req_notify_cq(conn->cq, 0));
TEST_NZ(pthread_create(&conn->cq_poller_thread, NULL, poll_cq, conn));
}
void * poll_cq(void *ctx)
{
struct ibv_cq *cq;
struct ibv_wc wc;
while (1) {
if (!paused) {
// rdma_debug("get cq event ...");
TEST_NZ(ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx));
ibv_ack_cq_events(cq, 1);
TEST_NZ(ibv_req_notify_cq(cq, 0));
while (ibv_poll_cq(cq, 1, &wc)) {
// rdma_debug("handle cq ...");
on_completion(&wc);
}
} else {
// rdma_debug("wait signal ...");
pthread_mutex_lock(&mutex);
pthread_cond_wait(&resume_cond, &mutex);
pthread_mutex_unlock(&mutex);
}
}
return NULL;
}
int on_addr_resolved(struct rdma_cm_id *id)
{
struct ibv_qp_init_attr qp_attr;
struct connection *conn;
printf("address resolved.\n");
build_context(id->verbs);
build_qp_attr(&qp_attr);
TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr));
id->context = conn = (struct connection *)malloc(sizeof(struct connection));
conn->id = id;
conn->qp = id->qp;
conn->num_completions = 0;
register_memory(conn);
post_receives(conn);
TEST_NZ(rdma_resolve_route(id, TIMEOUT_IN_MS));
return 0;
}
int main(int argc, char **argv)
{
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
if (argc != 3)
die("usage: client <server-address> <server-port>");
TEST_NZ(getaddrinfo(argv[1], argv[2], NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_event_channel(ec);
return 0;
}
int SRLockClient::start_client () {
SRClientContext ctx;
char temp_char;
ctx.ib_port = SERVER_IB_PORT;
srand (generate_random_seed()); // initialize random seed
TEST_NZ (establish_tcp_connection(SERVER_ADDR.c_str(), SERVER_TCP_PORT, &(ctx.sockfd)));
DEBUG_COUT("[Comm] Client connected to LM on sock " << ctx.sockfd);
TEST_NZ (ctx.create_context());
DEBUG_COUT("[Info] Context Created " << ctx.sockfd);
TEST_NZ (RDMACommon::connect_qp (&(ctx.qp), ctx.ib_port, ctx.port_attr.lid, ctx.sockfd));
DEBUG_COUT("[Conn] QP connected!");
start_operation(ctx);
// Sync so server will know that client is done mucking with its memory
DEBUG_COUT("[Info] Client is done, and is ready to destroy its resources!");
TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
TEST_NZ(ctx.destroy_context());
}
//static int run(int argc, char **argv)
int ibrdma_send(char* host, char* port, void* data, uint64_t size)
{
struct addrinfo *addr;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
TEST_NZ(getaddrinfo(host, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
on_connect(cmid->context);
/* Init MSG send to start RDMA*/
init_tfile(data, size);
send_init(cmid->context);
/*----------------------------*/
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
rdma_destroy_id(cmid);
rdma_destroy_event_channel(ec);
return 0;
}
int ibrdma_transfer(struct transfer_info *tfi, int num_tfi) {
struct addrinfo *addr;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
TEST_NZ(getaddrinfo(host, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
on_connect(cmid->context);
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
rdma_destroy_id(&cmid);
rdma_destroy_event_channel(&ec);
return 0;
}
static int create_rdma(void *ctx)
{
// struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *id = NULL;
rdma_cm_event_handler event_handler = NULL;
/*
if (strcmp(argv[1], "write") == 0)
set_mode(M_WRITE);
else if (strcmp(argv[1], "read") == 0)
set_mode(M_READ);
else
usage(argv[0]);
*/
// TEST_NZ(getaddrinfo(s_ip, s_port, NULL, &addr));
struct sockaddr_in addr = { 0 };
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(ip);
addr.sin_port = htons(port);
//TEST_Z(ec = rdma_create_event_channel());
TEST_NZ( id = rdma_create_id( event_handler, NULL, RDMA_PS_TCP, IB_QPT_RC ) ); //WHAT QP?
TEST_NZ( rdma_resolve_addr(id, NULL, (struct sockaddr*) &addr, TIMEOUT_IN_MS) );
// kfree(addr);
time_stamp(0);
while ( event_handler(id, event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
// rdma_ack_cm_event(event);
if (on_event(id, &event_copy))
break;
}
rdma_destroy_id(id);
time_stamp(9);
time_calculate();
return 0;
}
void * poll_cq2(void *ctx)
{
struct ibv_cq *cq;
struct ibv_wc wc;
while (1) {
TEST_NZ(ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx));
ibv_ack_cq_events(cq, 1);
TEST_NZ(ibv_req_notify_cq(cq, 0));
while (ibv_poll_cq(cq, 1, &wc))
on_completion(&wc);
}
return NULL;
}
//static int run(int argc, char **argv)
//int RDMA_Connect(struct RDMA_communicator *comm, struct RDMA_param *param)
int RDMA_Active_Init(struct RDMA_communicator *comm, struct RDMA_param *param)
{
struct addrinfo *addr;
// struct rdma_cm_id *cm_id= NULL;
// struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
char port[8];
// int i,j;
sprintf(port, "%d", RDMA_PORT);
TEST_NZ(getaddrinfo(param->host, port, NULL, &addr));
TEST_Z(comm->ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(comm->ec, &(comm->cm_id), NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(comm->cm_id, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(comm->cm_id);
TEST_NZ(rdma_resolve_route(comm->cm_id, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(comm->cm_id, &cm_params));
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ESTABLISHED));
// on_connect(cm_id->context);
return 0;
}
static void
kickoff_rdma_with_offset(uintptr_t offset, IbvConnection *conn, int length)
{
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
memset(&wr, 0, sizeof(wr));
wr.wr_id = (uintptr_t)conn;
wr.opcode = IBV_WR_RDMA_WRITE;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr + offset;
wr.wr.rdma.rkey = conn->peer_mr.rkey;
sge.addr = (uintptr_t)conn->rdma_local_region + offset;
sge.length = length;
sge.lkey = conn->rdma_local_mr->lkey;
if (RDMA_BUFFER_SIZE < (offset + length)) {
WARN(0, "kickoff_rdma_with_offset: offset + length (=%d) exceeds RDMA_BUFFER_SIZE (=%d).\n",
offset + length, RDMA_BUFFER_SIZE);
exit(1);
}
TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
}
int IPTradTrxManagerContext::destroy_context () {
//for (int s = 0; s < SERVER_CNT; s++)
// TEST_NZ (res_mng_ctxs[s].destroy_context());
TEST_NZ (client_ctx.destroy_context());
return 0;
}
int Process::post_send(void *context){
Connection *conn = (Connection *) context;
std::cout<<"SEND LOCATION CONN -> ID"<<conn->identifier<<"\n";
std::cout<<"SEND POINTER ID"<<listener<<std::endl<<std::flush;
assert(conn != nullptr);
assert(conn->identifier != nullptr);
struct ibv_send_wr wr, *bad_wr = nullptr;
struct ibv_sge sge;
assert(&message != nullptr);
calc_message_numerical(&message);
assert(&message != nullptr);
assert(conn->send_region != nullptr);
memcpy(conn->send_region, message.x, message.size*sizeof(char));
printf("connected. posting send...\n");
memsetzero(&wr);
wr.wr_id = (uintptr_t)conn;
wr.opcode = IBV_WR_SEND;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
sge.addr = (uintptr_t)conn->send_region;
sge.length = message.size;
sge.lkey = conn->send_memory_region->lkey;
TEST_NZ(ibv_post_send(conn->queue_pair, &wr, &bad_wr));
return 0;
}
void build_connection(struct rdma_cm_id *id)
{
rdma_conn_t *conn;
struct ibv_qp_init_attr qp_attr;
build_context(id->verbs);
build_qp_attr(&qp_attr);
TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr));
conn = malloc(sizeof(rdma_conn_t));
id->context = conn;
rdma_conn = conn;
conn->id = id;
conn->qp = id->qp;
conn->send_state = SS_INIT;
conn->recv_state = RS_INIT;
conn->connected = 0;
register_memory(conn);
post_receives(conn);
}
void cfio_rdma_client_write_data(
int remote_offset,
int length,
int local_offset)
{
// rdma_debug("write data ...");
if (remote_offset < 0 || remote_offset + length > DATA_REGION_SIZE) {
die("RDMA out of region");
}
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
memset(&wr, 0, sizeof(wr));
rdma_conn_t *conn = rdma_conn;
wr.wr_id = (uintptr_t)(conn);
wr.opcode = IBV_WR_RDMA_WRITE;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
wr.wr.rdma.remote_addr = (uintptr_t)((char *)conn->peer_data_mr.addr + remote_offset);
wr.wr.rdma.rkey = conn->peer_data_mr.rkey;
sge.addr = (uintptr_t)(conn->data_region + local_offset);
sge.length = length;
sge.lkey = conn->data_mr->lkey;
++ request_stack_size;
TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
}
void build_connection(struct rdma_cm_id *id)
{
struct connection *conn;
struct ibv_qp_init_attr qp_attr;
//init semaphores
sem_init(&read_ops, 0, 0);
sem_init(&done_ops, 0, 0);
sem_init(&write_ops, 0, 1);
build_context(id->verbs);
build_qp_attr(&qp_attr);
TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr));
id->context = conn = (struct connection *)malloc(sizeof(struct connection));
conn->id = id;
conn->qp = id->qp;
conn->send_state = SS_INIT;
conn->recv_state = RS_INIT;
conn->connected = 0;
register_memory(conn);
post_receives(conn);
}
void _rdma_write_offset(void *context, void* buf, uint64_t offset)
{
struct connection *conn = (struct connection *)context;
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
//printf("7\n");
memset(&wr, 0, sizeof(wr));
wr.wr_id = (uintptr_t)conn;
wr.opcode = IBV_WR_RDMA_WRITE;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr + offset;
wr.wr.rdma.rkey = conn->peer_mr.rkey;
sge.addr = (uintptr_t)conn->rdma_remote_region;
sge.length = RDMA_BUFFER_SIZE;
sge.lkey = conn->rdma_remote_mr->lkey;
time_stamp(2);
sem_wait(&write_ops);
memcpy(conn->rdma_remote_region, (char*)buf, RDMA_BUFFER_SIZE);
TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
}
int on_connection(void *context)
{
struct connection *conn = (struct connection *)context;
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
snprintf(conn->send_region, BUFFER_SIZE, "message from active/client side with pid %d", getpid());
printf("connected. posting send...\n");
memset(&wr, 0, sizeof(wr));
wr.wr_id = (uintptr_t)conn;
wr.opcode = IBV_WR_SEND;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
sge.addr = (uintptr_t)conn->send_region;
sge.length = BUFFER_SIZE;
sge.lkey = conn->send_mr->lkey;
TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
return 0;
}
int RDMAServer::initialize_data_structures(){
global_items_region = new ItemVersion[ITEM_CNT * MAX_ITEM_VERSIONS];
//OrdersVersion* RDMAServer::orders_region = new OrdersVersion[MAX_ORDERS_CNT * MAX_ORDERS_VERSIONS];
global_orders_region = new OrdersVersion[MAX_BUFFER_SIZE]; // TODO
//OrderLineVersion* RDMAServer::order_line_region = new OrderLineVersion[ORDERLINE_PER_ORDER * MAX_ORDERS_CNT * MAX_ORDERLINE_VERSIONS];
global_order_line_region = new OrderLineVersion[MAX_BUFFER_SIZE]; // TODO
//CCXactsVersion* RDMAServer::cc_xacts_region = new CCXactsVersion[MAX_CCXACTS_CNT * MAX_CCXACTS_VERSIONS];
global_cc_xacts_region = new CCXactsVersion[MAX_BUFFER_SIZE]; // TODO
global_timestamp_region = new Timestamp[1];
global_lock_items_region = new uint64_t[ITEM_CNT];
RDMAServer::global_timestamp_region[0].value = 0ULL; // the timestamp counter is initially set to 0
DEBUG_COUT("[Info] Timestamp set to " << RDMAServer::global_timestamp_region[0].value);
TEST_NZ(load_tables_from_files(global_items_region));
DEBUG_COUT("[Info] Tables loaded successfully");
int i;
uint32_t stat;
uint32_t version;
for (i=0; i < ITEM_CNT; i++) {
stat = (uint32_t)0; // 0 for free, 1 for locked
version = (uint32_t)0; // the first version of each item is 0
global_lock_items_region[i] = Lock::set_lock(stat, version);
}
DEBUG_COUT("[Info] All locks set free");
return 0;
}
int RDMA_Active_Finalize(struct RDMA_communicator *comm)
{
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_DISCONNECTED));
rdma_destroy_id(comm->cm_id);
rdma_destroy_event_channel(comm->ec);
return 0;
}
void write_remote(struct connection * conn, uint32_t len){
uint32_t size =len&(~(1U<<31));
snprintf(conn->send_region, send_buffer_size, "message from active/client side with pid %d", getpid());
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
memset(&wr,0,sizeof(wr));
wr.wr_id = (uintptr_t)conn;
wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
wr.send_flags = IBV_SEND_SIGNALED;
wr.imm_data = htonl(len);
wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_addr;
wr.wr.rdma.rkey = conn->peer_rkey;
if (size>0){
wr.sg_list = &sge;
wr.num_sge = 1;
sge.addr = (uintptr_t)conn->send_region;
sge.length = size;
sge.lkey = conn->send_region_mr->lkey;
}
TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
}
int client_on_route_resolved(struct rdma_cm_id *id)
{
struct rdma_conn_param cm_params;
build_params(&cm_params);
TEST_NZ(rdma_connect(id, &cm_params));
return 0;
}
int Coordinator::start () {
CoordinatorContext ctx[SERVER_CNT];
struct sockaddr_in returned_addr;
socklen_t len = sizeof(returned_addr);
char temp_char;
srand (generate_random_seed()); // initialize random seed
// Call socket(), bind() and listen()
TEST_NZ (server_socket_setup(&server_sockfd, SERVER_CNT));
// accept connections from Cohorts
std::cout << "[Info] Waiting for " << SERVER_CNT << " cohort(s) on port " << TRX_MANAGER_TCP_PORT << std::endl;
for (int s = 0; s < SERVER_CNT; s++) {
ctx[s].sockfd = accept (server_sockfd, (struct sockaddr *) &returned_addr, &len);
if (ctx[s].sockfd < 0){
std::cerr << "ERROR on accept() for RM #" << s << std::endl;
return -1;
}
std::cout << "[Conn] Received Cohort #" << s << " on sock " << ctx[s].sockfd << std::endl;
// create all resources
ctx[s].ib_port = TRX_MANAGER_IB_PORT;
TEST_NZ (ctx[s].create_context());
DEBUG_COUT("[Info] Context for cohort " << s << " created");
// connect the QPs
TEST_NZ (RDMACommon::connect_qp (&(ctx[s].qp), ctx[s].ib_port, ctx[s].port_attr.lid, ctx[s].sockfd));
DEBUG_COUT("[Conn] QPed to cohort " << s);
}
std::cout << "[Info] Established connection to all " << SERVER_CNT << " resource-manager(s)." << std::endl;
start_benchmark(ctx);
DEBUG_COUT("[Info] Coordinator is done, and is ready to destroy its resources!");
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
DEBUG_COUT("[Conn] Notified cohort " << i << " it's done");
TEST_NZ ( ctx[i].destroy_context());
}
}
int IPCohort::start_server () {
IPCohortContext ctx;
char temp_char;
TEST_NZ (establish_tcp_connection(TRX_MANAGER_ADDR.c_str(), TRX_MANAGER_TCP_PORT, &ctx.sockfd));
TEST_NZ (ctx.create_context());
srand (time(NULL)); // initialize random seed
start_benchmark(ctx);
/* Sync so server will know that client is done mucking with its memory */
DEBUG_COUT("[Info] Cohort client is done, and is ready to destroy its resources!");
TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
TEST_NZ(ctx.destroy_context());
}
inline void cfio_rdma_client_wait(void *ctx)
{
struct ibv_cq *cq;
struct ibv_wc wc;
while (request_stack_size) {
// rdma_debug("get cq event ...");
TEST_NZ(ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx));
// rdma_debug("ibv_ack_cq_events...");
ibv_ack_cq_events(cq, 1);
TEST_NZ(ibv_req_notify_cq(cq, 0));
while (ibv_poll_cq(cq, 1, &wc)) {
// rdma_debug("handle cq ...");
on_completion(&wc);
}
}
}
void on_completion(struct ibv_wc *wc)
{
struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id;
printf("== STATE: send=%d / recv=%d ==\n", conn->send_state, conn->recv_state);
if (wc->status != IBV_WC_SUCCESS)
die("on_completion: status is not IBV_WC_SUCCESS.");
if (wc->opcode & IBV_WC_RECV) {
conn->recv_state++;
printf("RECV: Recieved: TYPE=%d\n", conn->recv_msg->type);
if (conn->recv_msg->type == MSG_MR) {
memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr));
post_receives(conn); /* only rearm for MSG_MR */
if (conn->send_state == SS_INIT) /* received peer's MR before sending ours, so send ours back */
send_mr(conn);
}
} else {
conn->send_state++;
printf("SEND: Sent out: TYPE=%d\n", conn->send_msg->type);
}
if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) {
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
if (s_mode == M_WRITE)
printf(" -> received MSG_MR. writing message to remote memory...\n");
else
printf(" -> received MSG_MR. reading message from remote memory...\n");
memset(&wr, 0, sizeof(wr));
wr.wr_id = (uintptr_t)conn;
wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr;
wr.wr.rdma.rkey = conn->peer_mr.rkey;
sge.addr = (uintptr_t)conn->rdma_local_region;
sge.length = RDMA_BUFFER_SIZE;
sge.lkey = conn->rdma_local_mr->lkey;
TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr));
printf("PSEND: Posted send request: MSG=%s\n", conn->rdma_local_region);
conn->send_msg->type = MSG_DONE;
send_message(conn);
} else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) {
printf(" -> remote buffer: %s\n", get_peer_message_region(conn));
rdma_disconnect(conn->id);
}
}
int on_addr_resolved(struct rdma_cm_id *id)
{
printf("address resolved. a\n");
build_connection(id);
sprintf(get_local_message_region(id->context), "message from active/client side with pid %d", getpid());
TEST_NZ(rdma_resolve_route(id, TIMEOUT_IN_MS));
return 0;
}
int on_route_resolved(struct rdma_cm_id *id)
{
struct rdma_conn_param cm_params;
printf("route resolved.\n");
build_params(&cm_params);
TEST_NZ(rdma_connect(id, &cm_params));
return 0;
}
int on_addr_resolved(struct rdma_cm_id *id)
{
printf("address resolved.\n");
build_context(id->verbs);
build_connection(id);
TEST_NZ(rdma_resolve_route(id, TIMEOUT_IN_MS));
return 0;
}