int main(int argc, char **argv)
{
struct sockaddr_in addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *listener = NULL;
struct rdma_event_channel *ec = NULL;
uint16_t port = 0;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_bind_addr(listener, (struct sockaddr *)&addr));
TEST_NZ(rdma_listen(listener, 10)); /* backlog=10 is arbitrary */
port = ntohs(rdma_get_src_port(listener));
printf("listening on port %d.\n", port);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_id(listener);
rdma_destroy_event_channel(ec);
return 0;
}
static void build_verbs(IbvConnection *conn, struct ibv_context *verbs)
{
conn->ibvctx = verbs;
TEST_Z(conn->pd = ibv_alloc_pd(conn->ibvctx));
TEST_Z(conn->comp_channel = ibv_create_comp_channel(conn->ibvctx));
TEST_Z(conn->cq = ibv_create_cq(conn->ibvctx, 10, NULL, conn->comp_channel, 0)); /* cqe=10 is arbitrary */
TEST_NZ(ibv_req_notify_cq(conn->cq, 0));
TEST_NZ(pthread_create(&conn->cq_poller_thread, NULL, poll_cq, conn));
}
void register_memory(struct connection *conn)
{
conn->send_region = malloc(send_buffer_size);
conn->recv_region = malloc(recv_buffer_size);
memset(conn->recv_region,0, recv_buffer_size);
conn->send_msg = malloc(sizeof(struct message));
conn->recv_msg = malloc(sizeof(struct message));
TEST_Z(conn->send_region_mr = ibv_reg_mr(
s_ctx->pd,
conn->send_region,
send_buffer_size,
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->recv_region_mr = ibv_reg_mr(
s_ctx->pd,
conn->recv_region,
recv_buffer_size,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
TEST_Z(conn->send_msg_mr = ibv_reg_mr(
s_ctx->pd,
conn->send_msg,
sizeof(struct message),
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->recv_msg_mr = ibv_reg_mr(
s_ctx->pd,
conn->recv_msg,
sizeof(struct message),
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
}
void register_memory(IbvConnection *conn)
{
conn->send_msg = (message *)malloc(sizeof(struct message));
conn->recv_msg = (message *)malloc(sizeof(struct message));
conn->rdma_local_region = (char *)calloc(RDMA_BUFFER_SIZE, 1);
conn->rdma_remote_region = (char *)calloc(RDMA_BUFFER_SIZE, 1);
TEST_Z(conn->send_mr = ibv_reg_mr(
conn->pd,
conn->send_msg,
sizeof(struct message),
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->recv_mr = ibv_reg_mr(
conn->pd,
conn->recv_msg,
sizeof(struct message),
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE ));
TEST_Z(conn->rdma_local_mr = ibv_reg_mr(
conn->pd,
conn->rdma_local_region,
RDMA_BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->rdma_remote_mr = ibv_reg_mr(
conn->pd,
conn->rdma_remote_region,
RDMA_BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
}
void register_memory(rdma_conn_t *conn)
{
conn->send_msg = malloc(sizeof(rdma_msg_t));
conn->recv_msg = malloc(sizeof(rdma_msg_t));
conn->data_region = data_region;
conn->addr_region = addr_region;
TEST_Z(conn->send_mr = ibv_reg_mr(
s_ctx->pd,
conn->send_msg,
sizeof(rdma_msg_t),
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->recv_mr = ibv_reg_mr(
s_ctx->pd,
conn->recv_msg,
sizeof(rdma_msg_t),
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->addr_mr = ibv_reg_mr(
s_ctx->pd,
conn->addr_region,
ADDR_REGION_SIZE,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
TEST_Z(conn->data_mr = ibv_reg_mr(
s_ctx->pd,
conn->data_region,
DATA_REGION_SIZE,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ));
}
void register_memory(struct connection *conn)
{
conn->send_msg = malloc(sizeof(struct message));
conn->recv_msg = malloc(sizeof(struct message));
conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE);
conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE);
TEST_Z(conn->send_mr = ibv_reg_mr(
s_ctx->pd,
conn->send_msg,
sizeof(struct message),
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->recv_mr = ibv_reg_mr(
s_ctx->pd,
conn->recv_msg,
sizeof(struct message),
IBV_ACCESS_LOCAL_WRITE | ((s_mode == M_WRITE) ? IBV_ACCESS_REMOTE_WRITE : IBV_ACCESS_REMOTE_READ)));
TEST_Z(conn->rdma_local_mr = ibv_reg_mr(
s_ctx->pd,
conn->rdma_local_region,
RDMA_BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->rdma_remote_mr = ibv_reg_mr(
s_ctx->pd,
conn->rdma_remote_region,
RDMA_BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE | ((s_mode == M_WRITE) ? IBV_ACCESS_REMOTE_WRITE : IBV_ACCESS_REMOTE_READ)));
}
void *rmem_table_alloc(struct rmem_table *rmem, size_t size, tag_t tag)
{
size_t req_size = size + DATA_OFFSET;
struct alloc_entry *entry = NULL;
struct list_head *free_node;
int bucket;
/* Warn if the tag is not unique, then return the original allocation as
if the caller had used rmem_table_lookup. This can happen if the user
fails before committing any allocations, and doesn't know it already
allocated something. */
entry = find_entry(rmem, tag);
if (entry != NULL) {
LOG(5, ("Requested tag %d is not unique\n", tag));
return entry->start + DATA_OFFSET;
}
free_node = rmem->free_list.next;
// next_free == start node means we've reached the end
while (free_node != &rmem->free_list) {
entry = entry_of_free_list(free_node);
// make sure entry is actually free
TEST_Z(entry->free);
if (entry->size >= req_size)
break;
free_node = free_node->next;
}
if (free_node == &rmem->free_list) {
// make sure we haven't run out of memory
if (rmem->alloc_size + req_size > RMEM_SIZE) {
LOG(5, ("Out of memory\n"));
return NULL;
}
TEST_Z(entry = (struct alloc_entry*)malloc(sizeof(struct alloc_entry)));
list_append(&rmem->list, &entry->list);
entry->free_list.next = &rmem->free_list;
entry->free_list.prev = free_node->prev;
entry->start = rmem->mem + rmem->alloc_size;
rmem->alloc_size += req_size;
entry->size = req_size;
entry->free = 0;
entry->tag = tag;
} else {
entry->tag = tag;
reserve_entry(rmem, entry, req_size);
}
rmem->nblocks++;
bucket = tag % NUM_BUCKETS;
list_append(&rmem->htable[bucket], &entry->htable);
memcpy(entry->start, &entry, sizeof(struct alloc_entry *));
return entry->start + DATA_OFFSET;
}
int BClientContext::register_memory() {
int mr_flags =
IBV_ACCESS_LOCAL_WRITE
| IBV_ACCESS_REMOTE_READ
| IBV_ACCESS_REMOTE_WRITE;
TEST_Z(recv_memory_mr = ibv_reg_mr(pd, &recv_memory_msg, sizeof(struct BMemoryKeys), mr_flags));
TEST_Z(lock_result_mr = ibv_reg_mr(pd, &lock_result, sizeof(uint64_t), mr_flags));
return 0;
}
int ibrdma_transfer(struct transfer_info *tfi, int num_tfi) {
struct addrinfo *addr;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
TEST_NZ(getaddrinfo(host, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
on_connect(cmid->context);
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
rdma_destroy_id(&cmid);
rdma_destroy_event_channel(&ec);
return 0;
}
static inline void reserve_entry(struct rmem_table *rmem,
struct alloc_entry *entry, size_t req_size)
{
size_t free_size = entry->size - req_size;
struct alloc_entry *free_entry;
void *entry_end = entry->start + req_size, *free_entry_end;
struct list_head *last_free = entry->free_list.prev;
struct list_head *next_free = entry->free_list.next;
struct list_head *free_node;
entry->size = req_size;
entry->free = 0;
list_delete(&entry->free_list);
if (free_size < MIN_SIZE || entry->list.next == &rmem->list) {
free_node = next_free;
} else if (entry->list.next == entry->free_list.next) {
free_entry = entry_of_free_list(entry->free_list.next);
free_entry_end = free_entry->start + free_entry->size;
free_entry->start = entry_end;
free_entry->size = (uintptr_t)free_entry_end - (uintptr_t)entry_end;
free_node = &free_entry->free_list;
} else {
TEST_Z(free_entry = (struct alloc_entry*)malloc(sizeof(struct alloc_entry)));
free_entry->free = 1;
free_entry->tag = 0;
free_entry->size = free_size;
free_entry->start = entry->start + req_size;
list_insert(&entry->list, &free_entry->list);
list_insert(last_free, &free_entry->free_list);
free_node = &free_entry->free_list;
}
set_free_ptrs(rmem, free_node);
}
//static int run(int argc, char **argv)
//int RDMA_Connect(struct RDMA_communicator *comm, struct RDMA_param *param)
int RDMA_Active_Init(struct RDMA_communicator *comm, struct RDMA_param *param)
{
struct addrinfo *addr;
// struct rdma_cm_id *cm_id= NULL;
// struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
char port[8];
// int i,j;
sprintf(port, "%d", RDMA_PORT);
TEST_NZ(getaddrinfo(param->host, port, NULL, &addr));
TEST_Z(comm->ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(comm->ec, &(comm->cm_id), NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(comm->cm_id, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(comm->cm_id);
TEST_NZ(rdma_resolve_route(comm->cm_id, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(comm->cm_id, &cm_params));
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ESTABLISHED));
// on_connect(cm_id->context);
return 0;
}
int main(int argc, char **argv)
{
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
if (argc != 3)
die("usage: client <server-address> <server-port>");
TEST_NZ(getaddrinfo(argv[1], argv[2], NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_event_channel(ec);
return 0;
}
static void on_pre_conn(struct rdma_cm_id *id)
{
struct conn_context *ctx = (struct conn_context *)malloc(sizeof(struct conn_context));
id->context = ctx;
ctx->file_name[0] = '\0'; // take this to mean we don't have the file name
posix_memalign((void **)&ctx->buffer, sysconf(_SC_PAGESIZE), BUFFER_SIZE);
TEST_Z(ctx->buffer_mr = ibv_reg_mr(rc_get_pd(), ctx->buffer, BUFFER_SIZE, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
posix_memalign((void **)&ctx->msg, sysconf(_SC_PAGESIZE), sizeof(*ctx->msg));
TEST_Z(ctx->msg_mr = ibv_reg_mr(rc_get_pd(), ctx->msg, sizeof(*ctx->msg), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
post_receive(id);
}
//static int run(int argc, char **argv)
int ibrdma_send(char* host, char* port, void* data, uint64_t size)
{
struct addrinfo *addr;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
TEST_NZ(getaddrinfo(host, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
on_connect(cmid->context);
/* Init MSG send to start RDMA*/
init_tfile(data, size);
send_init(cmid->context);
/*----------------------------*/
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
rdma_destroy_id(cmid);
rdma_destroy_event_channel(ec);
return 0;
}
void register_memory(struct connection *conn)
{
conn->send_region = malloc(BUFFER_SIZE);
conn->recv_region = malloc(BUFFER_SIZE);
TEST_Z(conn->send_mr = ibv_reg_mr(
s_ctx->pd,
conn->send_region,
BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
TEST_Z(conn->recv_mr = ibv_reg_mr(
s_ctx->pd,
conn->recv_region,
BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
}
void register_memory(struct connection *conn)
{
conn->send_msg = malloc(sizeof(struct control_msg));
conn->recv_msg = malloc(sizeof(struct control_msg));
// conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE);
// conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE);
// conn->rdma_msg_region = malloc(RDMA_BUFFER_SIZE);
TEST_Z(conn->send_mr = ibv_reg_mr(
s_ctx->pd,
conn->send_msg,
sizeof(struct control_msg),
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->recv_mr = ibv_reg_mr(
s_ctx->pd,
conn->recv_msg,
sizeof(struct control_msg),
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ));
// TEST_Z(conn->rdma_msg_mr = ibv_reg_mr(
// s_ctx->pd,
// conn->rdma_msg_region,
// RDMA_BUFFER_SIZE,
// IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ));
// IBV_ACCESS_LOCAL_WRITE | ((s_mode == M_WRITE) ? IBV_ACCESS_REMOTE_WRITE : IBV_ACCESS_REMOTE_READ)));
/*
TEST_Z(conn->rdma_local_mr = ibv_reg_mr(
s_ctx->pd,
conn->rdma_local_region,
RDMA_BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->rdma_remote_mr = ibv_reg_mr(
s_ctx->pd,
conn->rdma_remote_region,
RDMA_BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE | ((s_mode == M_WRITE) ? IBV_ACCESS_REMOTE_WRITE : IBV_ACCESS_REMOTE_READ)));
*/
return;
}
void callback_read(msk_trans_t *trans, void *arg) {
struct datamr *datamr = arg;
if (trans->state == MSK_CONNECTED && *datamr->count < SEND_COUNT)
TEST_Z(msk_post_RW(trans, datamr->data, datamr->mr, datamr->rloc, callback_read, datamr));
*datamr->count += 1;
pthread_cond_signal(datamr->cond);
}
void build_context(struct ibv_context *verbs)
{
if (s_ctx) {
if (s_ctx->ctx != verbs) {
die("cannot handle events in more than one context.");
}
return;
}
s_ctx = (rdma_ctx_t *)malloc(sizeof(rdma_ctx_t));
s_ctx->ctx = verbs;
TEST_Z(s_ctx->pd = ibv_alloc_pd(s_ctx->ctx));
TEST_Z(s_ctx->comp_channel = ibv_create_comp_channel(s_ctx->ctx));
TEST_Z(s_ctx->cq = ibv_create_cq(s_ctx->ctx, 10, NULL, s_ctx->comp_channel, 0)); /* cqe=10 is arbitrary */
TEST_NZ(ibv_req_notify_cq(s_ctx->cq, 0));
}
void build_msg_region(struct connection *conn, char* addr, int size)
{
conn->rdma_msg_region = addr;
TEST_Z(conn->rdma_msg_mr = ibv_reg_mr(
s_ctx->pd,
conn->rdma_msg_region,
size,
IBV_ACCESS_LOCAL_WRITE));
}
static void build_context(struct ibv_context *verbs)
{
if (s_ctx) {
if (s_ctx->ctx != verbs)
die("cannot handle events in more than one context.");
return;
}
s_ctx = (struct context *)malloc(sizeof(struct context));
s_ctx->ctx = verbs;
TEST_Z(s_ctx->pd = ibv_alloc_pd(s_ctx->ctx));
TEST_Z(s_ctx->comp_channel = ibv_create_comp_channel(s_ctx->ctx));
TEST_Z(s_ctx->cq = ibv_create_cq(s_ctx->ctx, 10, NULL, s_ctx->comp_channel, 0)); /* cqe=10 is arbitrary */
TEST_NZ(ibv_req_notify_cq(s_ctx->cq, 0));
// TEST_NZ(pthread_create(&s_ctx->cq_poller_thread, NULL, poll_cq, NULL));
}
void Connector::build_context(struct ibv_context* verb_)
{
if (s_ctx_ && s_ctx_->ctx_ != verb_) {
log_(ERROR, "cannot handle events in more than one context.")
exit(EXIT_FAILURE);
}
s_ctx_ = (struct context*)malloc(sizeof(struct context) );
s_ctx_->ctx_ = verb_;
TEST_Z(s_ctx_->pd_ = ibv_alloc_pd(s_ctx_->ctx_) );
TEST_Z(s_ctx_->comp_channel_ = ibv_create_comp_channel(s_ctx_->ctx_) );
TEST_Z(s_ctx_->cq_ = ibv_create_cq(s_ctx_->ctx_, MAX_QP__CQ_LENGTH, NULL, s_ctx_->comp_channel_, 0) );
TEST_NZ(ibv_req_notify_cq(s_ctx_->cq_, 0) )
// TODO
// TEST_NZ(pthread_create(pthread_v.back(), NULL, &Connector::bst_poll_cq, (void*)(this) ) )
pthread_v.push_back(new pthread_t() );
wrap_Connector* wrap_ = new wrap_Connector(this, s_ctx_);
TEST_NZ(pthread_create(pthread_v.back(), NULL, call_poll_cq_w_wrap, wrap_) )
}
void register_memory(struct connection *conn)
{
conn->send_msg = malloc(sizeof(struct message));
conn->recv_msg = malloc(sizeof(struct message));
conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE);
conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE);
/*
time_stamp(10);
memset(conn->rdma_local_region, 0, RDMA_BUFFER_SIZE);
memset(conn->rdma_remote_region, 0, RDMA_BUFFER_SIZE);
time_stamp(11);
*/
TEST_Z(conn->send_mr = ibv_reg_mr(
s_ctx->pd,
conn->send_msg,
sizeof(struct message),
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->recv_mr = ibv_reg_mr(
s_ctx->pd,
conn->recv_msg,
sizeof(struct message),
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
TEST_Z(conn->rdma_local_mr = ibv_reg_mr(
s_ctx->pd,
conn->rdma_local_region,
RDMA_BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE));
TEST_Z(conn->rdma_remote_mr = ibv_reg_mr(
s_ctx->pd,
conn->rdma_remote_region,
RDMA_BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
}
static void register_rdma_region(struct connection *conn, void* addr, uint64_t size)
{
if (conn->rdma_msg_mr != NULL) {
ibv_dereg_mr(conn->rdma_msg_mr);
}
// printf("Regist size=%d\n", size);
TEST_Z(conn->rdma_msg_mr = ibv_reg_mr(
s_ctx->pd,
addr,
size,
IBV_ACCESS_LOCAL_WRITE
| IBV_ACCESS_REMOTE_READ
| IBV_ACCESS_REMOTE_WRITE));
return;
}
static void register_rdma_msg_mr(int mr_index, void* addr, uint64_t size)
{
if (rdma_msg_mr[mr_index] != NULL) {
ibv_dereg_mr(rdma_msg_mr[mr_index]);
}
TEST_Z(rdma_msg_mr[mr_index] = ibv_reg_mr(
s_ctx->pd,
addr,
size,
IBV_ACCESS_LOCAL_WRITE
| IBV_ACCESS_REMOTE_READ
| IBV_ACCESS_REMOTE_WRITE));
return;
}
void cfio_rdma_client_send_addr()
{
rdma_conn_t *conn = rdma_conn;
#ifdef REGISTER_ON_THE_FLY
ibv_dereg_mr(conn->send_mr);
TEST_Z(conn->send_mr = ibv_reg_mr(
s_ctx->pd,
conn->send_msg,
sizeof(rdma_msg_t),
IBV_ACCESS_LOCAL_WRITE));
#endif
conn->send_msg->type = MSG_ADDR;
memcpy((char *)&(conn->send_msg->addr), conn->addr_region, ADDR_REGION_SIZE);
send_message(conn);
}
void connect_to_server()
{
/// recv addr and port from server
int server_rank = cfio_map_get_server_of_client(rank);
char server_ip[NI_MAXHOST];
unsigned short server_port_int;
char server_port[11];
MPI_Status sta;
MPI_Recv(server_ip, NI_MAXHOST, MPI_CHAR, server_rank, rank, MPI_COMM_WORLD, &sta);
MPI_Recv(&server_port_int, 1, MPI_UNSIGNED_SHORT, server_rank, rank + 1, MPI_COMM_WORLD, &sta);
sprintf(server_port, "%hu", server_port_int);
struct addrinfo *addr;
TEST_NZ(getaddrinfo(server_ip, server_port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cm_id, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cm_id, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
}
int main(int argc, char **argv)
{
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
if (argc != 4)
usage(argv[0]);
if (strcmp(argv[1], "write") == 0)
set_mode(M_WRITE);
else if (strcmp(argv[1], "read") == 0)
set_mode(M_READ);
else
usage(argv[0]);
TEST_NZ(getaddrinfo(argv[2], argv[3], NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_event_channel(ec);
return 0;
}
int ibrdma_transfer(struct transfer_info tfi, int num_tfi) {
struct addrinfo *addr;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
int i,j;
/*Allocation buffer space for reading from local fs to memory*/
struct transfer_file *ffile = tfi.tfiles;
int nf = tfi.tfiles;
char* host = tfi.ib_host;
char* port; sprintf(port,"%d",tfi.ib_port);
for (i = 0; i < NUM_FILE_BUF_C; i++) {
tfi.fbufs[i].fbuf = (char *)malloc(FILE_BUF_SIZE_C);
tfi.fbufs[i].size = 0;
}
TEST_NZ(getaddrinfo(host, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
on_connect(cmid->context);
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
rdma_destroy_id(&cmid);
rdma_destroy_event_channel(&ec);
return 0;
}
void client_test(char *ip, char *port) {
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
TEST_NZ(getaddrinfo(ip, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy)) {
s_ctx->ec = ec;
s_ctx->id = conn;
on_connection(event_copy.id);//send our memory information to server using post_send
//printf("wait for msg_send_completion\n");
poll_cq(NULL);//wait for send_completion
//printf("wait for msg_recv_completion\n");
poll_cq(NULL);//wait for recv_completion
break;
}
}
return;
};
int main(int argc, char **argv) {
msk_trans_t *trans;
uint8_t *mrbuf;
struct ibv_mr *mr;
msk_data_t *wdata;
msk_trans_attr_t attr;
memset(&attr, 0, sizeof(msk_trans_attr_t));
attr.server = -1; // put an incorrect value to check if we're either client or server
// sane values for optional or non-configurable elements
attr.rq_depth = RECV_NUM+2;
attr.sq_depth = RECV_NUM+2;
attr.max_recv_sge = NUM_SGE;
attr.max_send_sge = NUM_SGE;
attr.port = "1235";
// attr.disconnect_callback = callback_disconnect;
// argument handling
static struct option long_options[] = {
{ "client", required_argument, 0, 'c' },
{ "port", required_argument, 0, 'p' },
{ "server", no_argument, 0, 's' },
{ "help", no_argument, 0, 'h' },
{ 0, 0, 0, 0 }
};
int option_index = 0;
int op;
while ((op = getopt_long(argc, argv, "@hvsS:c:p:", long_options, &option_index)) != -1) { /* */
switch(op) {
case '@':
printf("%s compiled on %s at %s\n", argv[0], __DATE__, __TIME__);
printf("Release = %s\n", VERSION);
printf("Release comment = %s\n", VERSION_COMMENT);
printf("Git HEAD = %s\n", _GIT_HEAD_COMMIT ) ;
printf("Git Describe = %s\n", _GIT_DESCRIBE ) ;
exit(0);
case 'h':
print_help(argv);
exit(0);
case 'v':
attr.debug = attr.debug * 2 + 1;
break;
case 'c':
attr.server = 0;
attr.node = optarg;
break;
case 's':
attr.server = 10;
attr.node = "::";
break;
case 'S':
attr.server = 10;
attr.node = optarg;
break;
case 'p':
attr.port = optarg;
break;
default:
ERROR_LOG("Failed to parse arguments");
print_help(argv);
exit(EINVAL);
}
}
if (attr.server == -1) {
ERROR_LOG("must be either a client or a server!");
print_help(argv);
exit(EINVAL);
}
TEST_Z(msk_init(&trans, &attr));
if (!trans)
exit(-1);
if (trans->server) {
TEST_Z(msk_bind_server(trans));
TEST_NZ(trans = msk_accept_one(trans));
} else { //client
TEST_Z(msk_connect(trans));
TEST_NZ(trans);
}
TEST_NZ(mrbuf = malloc((RECV_NUM*NUM_SGE+1)*CHUNK_SIZE));
memset(mrbuf, 0, (RECV_NUM*NUM_SGE+1)*CHUNK_SIZE);
TEST_NZ(mr = msk_reg_mr(trans, mrbuf, (RECV_NUM*NUM_SGE+1)*CHUNK_SIZE, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ));
pthread_mutex_t lock;
pthread_cond_t cond;
pthread_mutex_init(&lock, NULL);
pthread_cond_init(&cond, NULL);
msk_data_t *rdata;
struct datalock datalock;
TEST_NZ(rdata = malloc(RECV_NUM*NUM_SGE*sizeof(msk_data_t)));
int i;
for (i=0; i<RECV_NUM*NUM_SGE; i++) {
rdata[i].data=mrbuf+i*CHUNK_SIZE;
rdata[i].size = 0;
rdata[i].max_size=CHUNK_SIZE;
rdata[i].mr = mr;
if ((i-1) % NUM_SGE != 0)
rdata[i].next = &rdata[i+1];
else
rdata[i].next = NULL;
}
datalock.lock = &lock;
datalock.cond = &cond;
TEST_NZ(wdata = malloc(sizeof(msk_data_t)));
wdata->data = mrbuf+RECV_NUM*NUM_SGE*CHUNK_SIZE;
wdata->mr = mr;
wdata->max_size = CHUNK_SIZE;
pthread_mutex_lock(&lock);
if (trans->server) // server receives, client sends
TEST_Z(msk_post_n_recv(trans, rdata, NUM_SGE, callback_recv, NULL, &datalock));
if (trans->server) {
TEST_Z(msk_finalize_accept(trans));
} else {
TEST_Z(msk_finalize_connect(trans));
}
if (trans->server) {
TEST_Z(pthread_cond_wait(&cond, &lock));
printf("Got something:\n %s (%d), %s (%d)\n", rdata[0].data, rdata[0].size, rdata[1].data, rdata[1].size);
} else {
memcpy(rdata[0].data, "012345678", 10);
rdata[0].size = 10;
memcpy(rdata[1].data, "0123456", 8);
rdata[1].size = 8;
TEST_Z(msk_post_n_send(trans, rdata, NUM_SGE, callback_recv, NULL, &datalock));
TEST_Z(pthread_cond_wait(&cond, &lock));
printf("Done with send\n");
}
pthread_mutex_unlock(&lock);
msk_dereg_mr(mr);
msk_destroy_trans(&trans);
free(rdata);
free(wdata);
free(mrbuf);
return 0;
}
