static int run_client(void)
{
struct rdma_addrinfo hints;
int i, ret, ret2;
printf("cmatose: starting client\n");
memset(&hints, 0, sizeof hints);
hints.ai_port_space = RDMA_PS_TCP;
ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai);
if (ret) {
perror("cmatose: getaddrinfo error");
return ret;
}
printf("cmatose: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id, test.rai->ai_src_addr,
test.rai->ai_dst_addr, 2000);
if (ret) {
perror("cmatose: failure getting addr");
connect_error();
return ret;
}
}
ret = connect_events();
if (ret)
goto disc;
if (message_count) {
printf("receiving data transfers\n");
ret = poll_cqs(RECV_CQ_INDEX);
if (ret)
goto disc;
printf("sending replies\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i]);
if (ret)
goto disc;
}
printf("data transfers complete\n");
}
ret = 0;
if (migrate) {
ret = migrate_channel(NULL);
if (ret)
goto out;
}
disc:
ret2 = disconnect_events();
if (ret2)
ret = ret2;
out:
return ret;
}
static int krping_bind_client(struct krping_cb *cb)
{
struct sockaddr_in sin;
int ret;
memset(&sin, 0, sizeof(sin));
sin.sin_len = sizeof sin;
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = cb->addr.s_addr;
sin.sin_port = cb->port;
ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *) &sin,
2000);
if (ret) {
log(LOG_ERR, "rdma_resolve_addr error %d\n", ret);
return ret;
}
krping_wait(cb, ROUTE_RESOLVED);
if (cb->state != ROUTE_RESOLVED) {
log(LOG_ERR,
"addr/route resolution did not resolve: state %d\n",
cb->state);
return EINTR;
}
DEBUG_LOG(PFX "rdma_resolve_addr - rdma_resolve_route successful\n");
return 0;
}
static int rping_bind_client(struct rping_cb *cb)
{
int ret;
if (cb->sin.ss_family == AF_INET)
((struct sockaddr_in *) &cb->sin)->sin_port = cb->port;
else
((struct sockaddr_in6 *) &cb->sin)->sin6_port = cb->port;
ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *) &cb->sin, 2000);
if (ret) {
perror("rdma_resolve_addr");
return ret;
}
sem_wait(&cb->sem);
if (cb->state != ROUTE_RESOLVED) {
fprintf(stderr, "waiting for addr/route resolution state %d\n",
cb->state);
return -1;
}
DEBUG_LOG("rdma_resolve_addr - rdma_resolve_route successful\n");
return 0;
}
void IBConnection::connect(const std::string& hostname,
const std::string& service)
{
struct addrinfo hints;
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
struct addrinfo* res;
int err = getaddrinfo(hostname.c_str(), service.c_str(), &hints, &res);
if (err)
throw InfinibandException("getaddrinfo failed");
L_(debug) << "[" << index_ << "] "
<< "resolution of server address and route";
for (struct addrinfo* t = res; t; t = t->ai_next) {
err =
rdma_resolve_addr(cm_id_, nullptr, t->ai_addr, RESOLVE_TIMEOUT_MS);
if (!err)
break;
}
if (err)
throw InfinibandException("rdma_resolve_addr failed");
freeaddrinfo(res);
}
//static int run(int argc, char **argv)
//int RDMA_Connect(struct RDMA_communicator *comm, struct RDMA_param *param)
int RDMA_Active_Init(struct RDMA_communicator *comm, struct RDMA_param *param)
{
struct addrinfo *addr;
// struct rdma_cm_id *cm_id= NULL;
// struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
char port[8];
// int i,j;
sprintf(port, "%d", RDMA_PORT);
TEST_NZ(getaddrinfo(param->host, port, NULL, &addr));
TEST_Z(comm->ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(comm->ec, &(comm->cm_id), NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(comm->cm_id, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(comm->cm_id);
TEST_NZ(rdma_resolve_route(comm->cm_id, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(comm->cm_id, &cm_params));
TEST_NZ(wait_for_event(comm->ec, RDMA_CM_EVENT_ESTABLISHED));
// on_connect(cm_id->context);
return 0;
}
int main(int argc, char **argv)
{
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
if (argc != 3)
die("usage: client <server-address> <server-port>");
TEST_NZ(getaddrinfo(argv[1], argv[2], NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_event_channel(ec);
return 0;
}
//static int run(int argc, char **argv)
int ibrdma_send(char* host, char* port, void* data, uint64_t size)
{
struct addrinfo *addr;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
TEST_NZ(getaddrinfo(host, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
on_connect(cmid->context);
/* Init MSG send to start RDMA*/
init_tfile(data, size);
send_init(cmid->context);
/*----------------------------*/
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
rdma_destroy_id(cmid);
rdma_destroy_event_channel(ec);
return 0;
}
/**
* starts the process of connecting to the target
* sleeps untill the connection is established or rejected
*/
int iser_connect(struct iser_conn *ib_conn,
struct sockaddr_in *src_addr,
struct sockaddr_in *dst_addr,
int non_blocking)
{
struct sockaddr *src, *dst;
int err = 0;
sprintf(ib_conn->name, "%pI4:%d",
&dst_addr->sin_addr.s_addr, dst_addr->sin_port);
/* the device is known only --after-- address resolution */
ib_conn->device = NULL;
iser_err("connecting to: %pI4, port 0x%x\n",
&dst_addr->sin_addr, dst_addr->sin_port);
ib_conn->state = ISER_CONN_PENDING;
iser_conn_get(ib_conn); /* ref ib conn's cma id */
ib_conn->cma_id = rdma_create_id(iser_cma_handler,
(void *)ib_conn,
RDMA_PS_TCP);
if (IS_ERR(ib_conn->cma_id)) {
err = PTR_ERR(ib_conn->cma_id);
iser_err("rdma_create_id failed: %d\n", err);
goto id_failure;
}
src = (struct sockaddr *)src_addr;
dst = (struct sockaddr *)dst_addr;
err = rdma_resolve_addr(ib_conn->cma_id, src, dst, 1000);
if (err) {
iser_err("rdma_resolve_addr failed: %d\n", err);
goto addr_failure;
}
if (!non_blocking) {
wait_event_interruptible(ib_conn->wait,
(ib_conn->state != ISER_CONN_PENDING));
if (ib_conn->state != ISER_CONN_UP) {
err = -EIO;
goto connect_failure;
}
}
mutex_lock(&ig.connlist_mutex);
list_add(&ib_conn->conn_list, &ig.connlist);
mutex_unlock(&ig.connlist_mutex);
return 0;
id_failure:
ib_conn->cma_id = NULL;
addr_failure:
ib_conn->state = ISER_CONN_DOWN;
connect_failure:
iser_conn_release(ib_conn, 1);
return err;
}
int ibrdma_transfer(struct transfer_info *tfi, int num_tfi) {
struct addrinfo *addr;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
TEST_NZ(getaddrinfo(host, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
on_connect(cmid->context);
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
rdma_destroy_id(&cmid);
rdma_destroy_event_channel(&ec);
return 0;
}
static int create_rdma(void *ctx)
{
// struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *id = NULL;
rdma_cm_event_handler event_handler = NULL;
/*
if (strcmp(argv[1], "write") == 0)
set_mode(M_WRITE);
else if (strcmp(argv[1], "read") == 0)
set_mode(M_READ);
else
usage(argv[0]);
*/
// TEST_NZ(getaddrinfo(s_ip, s_port, NULL, &addr));
struct sockaddr_in addr = { 0 };
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(ip);
addr.sin_port = htons(port);
//TEST_Z(ec = rdma_create_event_channel());
TEST_NZ( id = rdma_create_id( event_handler, NULL, RDMA_PS_TCP, IB_QPT_RC ) ); //WHAT QP?
TEST_NZ( rdma_resolve_addr(id, NULL, (struct sockaddr*) &addr, TIMEOUT_IN_MS) );
// kfree(addr);
time_stamp(0);
while ( event_handler(id, event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
// rdma_ack_cm_event(event);
if (on_event(id, &event_copy))
break;
}
rdma_destroy_id(id);
time_stamp(9);
time_calculate();
return 0;
}
static int run_client(void)
{
int i, ret;
printf("udaddy: starting client\n");
if (src_addr) {
ret = get_addr(src_addr, &test.src_in);
if (ret)
return ret;
}
ret = get_addr(dst_addr, &test.dst_in);
if (ret)
return ret;
test.dst_in.sin_port = port;
printf("udaddy: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id,
src_addr ? test.src_addr : NULL,
test.dst_addr, 2000);
if (ret) {
perror("udaddy: failure getting addr");
connect_error();
return ret;
}
}
ret = connect_events();
if (ret)
goto out;
if (message_count) {
printf("initiating data transfers\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i], 0);
if (ret)
goto out;
}
printf("receiving data transfers\n");
ret = poll_cqs();
if (ret)
goto out;
printf("data transfers complete\n");
}
out:
return ret;
}
static struct rdma_cm_id *
rpcrdma_create_id(struct rpcrdma_xprt *xprt,
struct rpcrdma_ia *ia, struct sockaddr *addr)
{
struct rdma_cm_id *id;
int rc;
init_completion(&ia->ri_done);
id = rdma_create_id(rpcrdma_conn_upcall, xprt, RDMA_PS_TCP, IB_QPT_RC);
if (IS_ERR(id)) {
rc = PTR_ERR(id);
dprintk("RPC: %s: rdma_create_id() failed %i\n",
__func__, rc);
return id;
}
ia->ri_async_rc = -ETIMEDOUT;
rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT);
if (rc) {
dprintk("RPC: %s: rdma_resolve_addr() failed %i\n",
__func__, rc);
goto out;
}
wait_for_completion_interruptible_timeout(&ia->ri_done,
msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
rc = ia->ri_async_rc;
if (rc)
goto out;
ia->ri_async_rc = -ETIMEDOUT;
rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT);
if (rc) {
dprintk("RPC: %s: rdma_resolve_route() failed %i\n",
__func__, rc);
goto out;
}
wait_for_completion_interruptible_timeout(&ia->ri_done,
msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
rc = ia->ri_async_rc;
if (rc)
goto out;
return id;
out:
rdma_destroy_id(id);
return ERR_PTR(rc);
}
int ibw_connect(struct ibw_conn *conn, struct sockaddr_in *serv_addr, void *conn_userdata)
{
struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
struct ibw_conn_priv *pconn = NULL;
int rc;
assert(conn!=NULL);
conn->conn_userdata = conn_userdata;
pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
DEBUG(DEBUG_DEBUG, ("ibw_connect: addr=%s, port=%u\n", inet_ntoa(serv_addr->sin_addr),
ntohs(serv_addr->sin_port)));
/* clean previous - probably half - initialization */
if (ibw_conn_priv_destruct(pconn)) {
DEBUG(DEBUG_ERR, ("ibw_connect/ibw_pconn_destruct failed for cm_id=%p\n", pconn->cm_id));
return -1;
}
/* init cm */
#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP);
#else
rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn);
#endif
if (rc) {
rc = errno;
sprintf(ibw_lasterr, "ibw_connect/rdma_create_id error %d\n", rc);
talloc_free(conn);
return -1;
}
DEBUG(DEBUG_DEBUG, ("ibw_connect: rdma_create_id succeeded, cm_id=%p\n", pconn->cm_id));
rc = rdma_resolve_addr(pconn->cm_id, NULL, (struct sockaddr *) serv_addr, 2000);
if (rc) {
sprintf(ibw_lasterr, "rdma_resolve_addr error %d\n", rc);
DEBUG(DEBUG_ERR, (ibw_lasterr));
talloc_free(conn);
return -1;
}
/* continued at RDMA_CM_EVENT_ADDR_RESOLVED */
return 0;
}
static int run_client(void)
{
int i, ret;
printf("udaddy: starting client\n");
ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai);
if (ret) {
perror("udaddy: getaddrinfo error");
return ret;
}
printf("udaddy: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id, test.rai->ai_src_addr,
test.rai->ai_dst_addr, 2000);
if (ret) {
perror("udaddy: failure getting addr");
connect_error();
return ret;
}
}
ret = connect_events();
if (ret)
goto out;
if (message_count) {
printf("initiating data transfers\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i], 0);
if (ret)
goto out;
}
printf("receiving data transfers\n");
ret = poll_cqs();
if (ret)
goto out;
printf("data transfers complete\n");
}
out:
return ret;
}
void connect_to_server()
{
/// recv addr and port from server
int server_rank = cfio_map_get_server_of_client(rank);
char server_ip[NI_MAXHOST];
unsigned short server_port_int;
char server_port[11];
MPI_Status sta;
MPI_Recv(server_ip, NI_MAXHOST, MPI_CHAR, server_rank, rank, MPI_COMM_WORLD, &sta);
MPI_Recv(&server_port_int, 1, MPI_UNSIGNED_SHORT, server_rank, rank + 1, MPI_COMM_WORLD, &sta);
sprintf(server_port, "%hu", server_port_int);
struct addrinfo *addr;
TEST_NZ(getaddrinfo(server_ip, server_port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cm_id, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cm_id, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
}
int main(int argc, char **argv)
{
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
if (argc != 4)
usage(argv[0]);
if (strcmp(argv[1], "write") == 0)
set_mode(M_WRITE);
else if (strcmp(argv[1], "read") == 0)
set_mode(M_READ);
else
usage(argv[0]);
TEST_NZ(getaddrinfo(argv[2], argv[3], NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_event_channel(ec);
return 0;
}
int ibrdma_transfer(struct transfer_info tfi, int num_tfi) {
struct addrinfo *addr;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
int i,j;
/*Allocation buffer space for reading from local fs to memory*/
struct transfer_file *ffile = tfi.tfiles;
int nf = tfi.tfiles;
char* host = tfi.ib_host;
char* port; sprintf(port,"%d",tfi.ib_port);
for (i = 0; i < NUM_FILE_BUF_C; i++) {
tfi.fbufs[i].fbuf = (char *)malloc(FILE_BUF_SIZE_C);
tfi.fbufs[i].size = 0;
}
TEST_NZ(getaddrinfo(host, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
build_params(&cm_params);
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
on_connect(cmid->context);
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_DISCONNECTED));
rdma_destroy_id(&cmid);
rdma_destroy_event_channel(&ec);
return 0;
}
void client_test(char *ip, char *port) {
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
TEST_NZ(getaddrinfo(ip, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy)) {
s_ctx->ec = ec;
s_ctx->id = conn;
on_connection(event_copy.id);//send our memory information to server using post_send
//printf("wait for msg_send_completion\n");
poll_cq(NULL);//wait for send_completion
//printf("wait for msg_recv_completion\n");
poll_cq(NULL);//wait for recv_completion
break;
}
}
return;
};
static int run(int argc, char **argv)
{
struct addrinfo *addr;
//struct rdma_cm_event *event = NULL;
struct rdma_cm_id *cmid= NULL;
struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
if (argc != 4)
usage(argv[0]);
if (strcmp(argv[1], "write") == 0)
set_mode(M_WRITE);
else if (strcmp(argv[1], "read") == 0)
set_mode(M_READ);
else
usage(argv[0]);
TEST_NZ(getaddrinfo(argv[2], argv[3], NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
/*create rdma socket*/
TEST_NZ(rdma_create_id(ec, &cmid, NULL, RDMA_PS_TCP));
/* int rdma_resolve_addr (struct rdma_cm_id *id, struct sockaddr *src_addr,
struct sockaddr dst_addr, int timeout_ms)
id RDMA identifier
src_addr Source address information. This parameter may be NULL.
dst_addr Destination address information
timeout_ms Time to wait for resolution to complete
Description:
Resolve destination and optional source addresses from IP addresses
to an RDMA address. If suc- cessful,
the specified rdma_cm_id will be bound to a local device.
*/
TEST_NZ(rdma_resolve_addr(cmid, NULL, addr->ai_addr, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ADDR_RESOLVED));
freeaddrinfo(addr);
build_connection(cmid);
sprintf(get_local_message_region(cmid->context), "message from active/client side with pid %d", getpid());
/*--------------------*/
/* int rdma_resolve_route (struct rdma_cm_id *id, int timeout_ms);
id RDMA identifier
timeout_ms Time to wait for resolution to complete
Description:
Resolves an RDMA route to the destination address in order
to establish a connection. The destination address must have
already been resolved by calling rdma_resolve_addr.
*/
TEST_NZ(rdma_resolve_route(cmid, TIMEOUT_IN_MS));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ROUTE_RESOLVED));
/* -------------------- */
print_path_rec(cmid);
/* int rdma_connect (struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
id RDMA identifier
conn_param connection parameters
Description:
For an rdma_cm_id of type RDMA_PS_TCP, this call initiates a connection
request to a remote destination. For an rdma_cm_id of type RDMA_PS_UDP,
it initiates a lookup of the remote QP providing the datagram service
*/
build_params(&cm_params);
printf("Connecting ...\n");
TEST_NZ(rdma_connect(cmid, &cm_params));
TEST_NZ(wait_for_event(ec, RDMA_CM_EVENT_ESTABLISHED));
printf("Connected !\n");
/* --------------------- */
/*TODO: do something */
on_connect(cmid->context);
send_mr(cmid->context);
/*--------------------*/
rdma_disconnect(cmid);
rdma_destroy_id(cmid);
rdma_destroy_event_channel(ec);
return 0;
/*=================*/
/*=================*/
/*
while (rdma_get_cm_event(ec, &event) == 0) {
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
*/
}
void network_init() {
/* Set up RDMA CM structures */
cm_channel = rdma_create_event_channel();
assert(cm_channel);
err = rdma_create_id(cm_channel, &cm_id, 0, RDMA_PS_TCP);
assert(err == 0);
/* Resolve server address and route */
n = getaddrinfo(server_ip, server_port_string, &hints, &res);
assert(n >= 0);
for (t = res; t; t = t->ai_next) {
err = rdma_resolve_addr(cm_id, 0, t->ai_addr, RESOLVE_TIMEOUT_MS);
if (!err) break;
}
assert(err == 0);
err = rdma_get_cm_event(cm_channel, &event);
assert(err == 0);
assert(event->event == RDMA_CM_EVENT_ADDR_RESOLVED);
rdma_ack_cm_event(event);
err = rdma_resolve_route(cm_id, RESOLVE_TIMEOUT_MS);
assert(err == 0);
err = rdma_get_cm_event(cm_channel, &event);
assert(err == 0);
assert(event->event == RDMA_CM_EVENT_ROUTE_RESOLVED);
rdma_ack_cm_event(event);
/* Create verbs objects now that we know which device to use */
pd = ibv_alloc_pd(cm_id->verbs);
assert(pd);
comp_chan = ibv_create_comp_channel(cm_id->verbs);
assert(comp_chan);
cq = ibv_create_cq(cm_id->verbs, 10, 0, comp_chan, 0);
assert(cq);
err = ibv_req_notify_cq(cq, 0);
assert(err == 0);
mr_data = ibv_reg_mr(pd, data, BUFFER_SIZE,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
assert(mr_data);
mr_ack_buffer = ibv_reg_mr(pd, &ack_buffer, sizeof(ack_buffer),
IBV_ACCESS_LOCAL_WRITE);
assert(mr_ack_buffer);
qp_attr.cap.max_send_wr = 10;
qp_attr.cap.max_send_sge = 10;
qp_attr.cap.max_recv_wr = 10;
qp_attr.cap.max_recv_sge = 10;
qp_attr.send_cq = cq;
qp_attr.recv_cq = cq;
qp_attr.qp_type = IBV_QPT_RC;
err = rdma_create_qp(cm_id, pd, &qp_attr);
assert(err == 0);
/* Post receive for data before connecting */
sge_data.addr = (uintptr_t)data;
sge_data.length = BUFFER_SIZE;
sge_data.lkey = mr_data->lkey;
recv_wr.sg_list = &sge_data;
recv_wr.num_sge = 1;
err = ibv_post_recv(cm_id->qp, &recv_wr, &bad_recv_wr);
assert(err == 0);
/* Construct connection params */
client_pdata.data_va = htonll((uintptr_t)data);
client_pdata.data_rkey = htonl(mr_data->rkey);
conn_param.private_data = &client_pdata;
conn_param.private_data_len = sizeof(client_pdata);
conn_param.initiator_depth = 1;
conn_param.retry_count = 7;
/* Connect to server */
err = rdma_connect(cm_id, &conn_param);
assert(err == 0);
err = rdma_get_cm_event(cm_channel, &event);
assert(err == 0);
assert(event->event == RDMA_CM_EVENT_ESTABLISHED);
memcpy(&server_pdata, event->param.conn.private_data, sizeof(server_pdata));
rdma_ack_cm_event(event);
printf("My index == %d\n", server_pdata.index);
/* Construct connection params */
client_pdata.index = server_pdata.index;
client_pdata.ack_buffer_va = server_pdata.ack_buffer_va;
client_pdata.ack_buffer_rkey = server_pdata.ack_buffer_rkey;
}
static struct rdma_cm_id *
rpcrdma_create_id(struct rpcrdma_xprt *xprt,
struct rpcrdma_ia *ia, struct sockaddr *addr)
{
struct rdma_cm_id *id;
int rc;
init_completion(&ia->ri_done);
id = rdma_create_id(&init_net, rpcrdma_conn_upcall, xprt, RDMA_PS_TCP,
IB_QPT_RC);
if (IS_ERR(id)) {
rc = PTR_ERR(id);
dprintk("RPC: %s: rdma_create_id() failed %i\n",
__func__, rc);
return id;
}
ia->ri_async_rc = -ETIMEDOUT;
rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT);
if (rc) {
dprintk("RPC: %s: rdma_resolve_addr() failed %i\n",
__func__, rc);
goto out;
}
wait_for_completion_interruptible_timeout(&ia->ri_done,
msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
/* FIXME:
* Until xprtrdma supports DEVICE_REMOVAL, the provider must
* be pinned while there are active NFS/RDMA mounts to prevent
* hangs and crashes at umount time.
*/
if (!ia->ri_async_rc && !try_module_get(id->device->owner)) {
dprintk("RPC: %s: Failed to get device module\n",
__func__);
ia->ri_async_rc = -ENODEV;
}
rc = ia->ri_async_rc;
if (rc)
goto out;
ia->ri_async_rc = -ETIMEDOUT;
rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT);
if (rc) {
dprintk("RPC: %s: rdma_resolve_route() failed %i\n",
__func__, rc);
goto put;
}
wait_for_completion_interruptible_timeout(&ia->ri_done,
msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
rc = ia->ri_async_rc;
if (rc)
goto put;
return id;
put:
module_put(id->device->owner);
out:
rdma_destroy_id(id);
return ERR_PTR(rc);
}
static int p9_rdma_bind_privport(struct p9_trans_rdma *rdma)
{
struct sockaddr_in cl = {
.sin_family = AF_INET,
.sin_addr.s_addr = htonl(INADDR_ANY),
};
int port, err = -EINVAL;
for (port = P9_DEF_MAX_RESVPORT; port >= P9_DEF_MIN_RESVPORT; port--) {
cl.sin_port = htons((ushort)port);
err = rdma_bind_addr(rdma->cm_id, (struct sockaddr *)&cl);
if (err != -EADDRINUSE)
break;
}
return err;
}
/**
* trans_create_rdma - Transport method for creating atransport instance
* @client: client instance
* @addr: IP address string
* @args: Mount options string
*/
static int
rdma_create_trans(struct p9_client *client, const char *addr, char *args)
{
int err;
struct p9_rdma_opts opts;
struct p9_trans_rdma *rdma;
struct rdma_conn_param conn_param;
struct ib_qp_init_attr qp_attr;
struct ib_device_attr devattr;
struct ib_cq_init_attr cq_attr = {};
/* Parse the transport specific mount options */
err = parse_opts(args, &opts);
if (err < 0)
return err;
/* Create and initialize the RDMA transport structure */
rdma = alloc_rdma(&opts);
if (!rdma)
return -ENOMEM;
/* Create the RDMA CM ID */
rdma->cm_id = rdma_create_id(p9_cm_event_handler, client, RDMA_PS_TCP,
IB_QPT_RC);
if (IS_ERR(rdma->cm_id))
goto error;
/* Associate the client with the transport */
client->trans = rdma;
/* Bind to a privileged port if we need to */
if (opts.privport) {
err = p9_rdma_bind_privport(rdma);
if (err < 0) {
pr_err("%s (%d): problem binding to privport: %d\n",
__func__, task_pid_nr(current), -err);
goto error;
}
}
/* Resolve the server's address */
rdma->addr.sin_family = AF_INET;
rdma->addr.sin_addr.s_addr = in_aton(addr);
rdma->addr.sin_port = htons(opts.port);
err = rdma_resolve_addr(rdma->cm_id, NULL,
(struct sockaddr *)&rdma->addr,
rdma->timeout);
if (err)
goto error;
err = wait_for_completion_interruptible(&rdma->cm_done);
if (err || (rdma->state != P9_RDMA_ADDR_RESOLVED))
goto error;
/* Resolve the route to the server */
err = rdma_resolve_route(rdma->cm_id, rdma->timeout);
if (err)
goto error;
err = wait_for_completion_interruptible(&rdma->cm_done);
if (err || (rdma->state != P9_RDMA_ROUTE_RESOLVED))
goto error;
/* Query the device attributes */
err = ib_query_device(rdma->cm_id->device, &devattr);
if (err)
goto error;
/* Create the Completion Queue */
cq_attr.cqe = opts.sq_depth + opts.rq_depth + 1;
rdma->cq = ib_create_cq(rdma->cm_id->device, cq_comp_handler,
cq_event_handler, client,
&cq_attr);
if (IS_ERR(rdma->cq))
goto error;
ib_req_notify_cq(rdma->cq, IB_CQ_NEXT_COMP);
/* Create the Protection Domain */
rdma->pd = ib_alloc_pd(rdma->cm_id->device);
if (IS_ERR(rdma->pd))
goto error;
/* Cache the DMA lkey in the transport */
rdma->dma_mr = NULL;
if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
rdma->lkey = rdma->cm_id->device->local_dma_lkey;
else {
rdma->dma_mr = ib_get_dma_mr(rdma->pd, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(rdma->dma_mr))
goto error;
rdma->lkey = rdma->dma_mr->lkey;
}
/* Create the Queue Pair */
memset(&qp_attr, 0, sizeof qp_attr);
qp_attr.event_handler = qp_event_handler;
qp_attr.qp_context = client;
qp_attr.cap.max_send_wr = opts.sq_depth;
qp_attr.cap.max_recv_wr = opts.rq_depth;
qp_attr.cap.max_send_sge = P9_RDMA_SEND_SGE;
qp_attr.cap.max_recv_sge = P9_RDMA_RECV_SGE;
qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
qp_attr.qp_type = IB_QPT_RC;
qp_attr.send_cq = rdma->cq;
qp_attr.recv_cq = rdma->cq;
err = rdma_create_qp(rdma->cm_id, rdma->pd, &qp_attr);
if (err)
goto error;
rdma->qp = rdma->cm_id->qp;
/* Request a connection */
memset(&conn_param, 0, sizeof(conn_param));
conn_param.private_data = NULL;
conn_param.private_data_len = 0;
conn_param.responder_resources = P9_RDMA_IRD;
conn_param.initiator_depth = P9_RDMA_ORD;
err = rdma_connect(rdma->cm_id, &conn_param);
if (err)
goto error;
err = wait_for_completion_interruptible(&rdma->cm_done);
if (err || (rdma->state != P9_RDMA_CONNECTED))
goto error;
client->status = Connected;
return 0;
error:
rdma_destroy_trans(rdma);
return -ENOTCONN;
}
/**
* trans_create_rdma - Transport method for creating atransport instance
* @client: client instance
* @addr: IP address string
* @args: Mount options string
*/
static int
rdma_create_trans(struct p9_client *client, const char *addr, char *args)
{
int err;
struct p9_rdma_opts opts;
struct p9_trans_rdma *rdma;
struct rdma_conn_param conn_param;
struct ib_qp_init_attr qp_attr;
struct ib_device_attr devattr;
/* Parse the transport specific mount options */
err = parse_opts(args, &opts);
if (err < 0)
return err;
/* Create and initialize the RDMA transport structure */
rdma = alloc_rdma(&opts);
if (!rdma)
return -ENOMEM;
/* Create the RDMA CM ID */
rdma->cm_id = rdma_create_id(p9_cm_event_handler, client, RDMA_PS_TCP);
if (IS_ERR(rdma->cm_id))
goto error;
/* Associate the client with the transport */
client->trans = rdma;
/* Resolve the server's address */
rdma->addr.sin_family = AF_INET;
rdma->addr.sin_addr.s_addr = in_aton(addr);
rdma->addr.sin_port = htons(opts.port);
err = rdma_resolve_addr(rdma->cm_id, NULL,
(struct sockaddr *)&rdma->addr,
rdma->timeout);
if (err)
goto error;
err = wait_for_completion_interruptible(&rdma->cm_done);
if (err || (rdma->state != P9_RDMA_ADDR_RESOLVED))
goto error;
/* Resolve the route to the server */
err = rdma_resolve_route(rdma->cm_id, rdma->timeout);
if (err)
goto error;
err = wait_for_completion_interruptible(&rdma->cm_done);
if (err || (rdma->state != P9_RDMA_ROUTE_RESOLVED))
goto error;
/* Query the device attributes */
err = ib_query_device(rdma->cm_id->device, &devattr);
if (err)
goto error;
/* Create the Completion Queue */
rdma->cq = ib_create_cq(rdma->cm_id->device, cq_comp_handler,
cq_event_handler, client,
opts.sq_depth + opts.rq_depth + 1, 0);
if (IS_ERR(rdma->cq))
goto error;
ib_req_notify_cq(rdma->cq, IB_CQ_NEXT_COMP);
/* Create the Protection Domain */
rdma->pd = ib_alloc_pd(rdma->cm_id->device);
if (IS_ERR(rdma->pd))
goto error;
/* Cache the DMA lkey in the transport */
rdma->dma_mr = NULL;
if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
rdma->lkey = rdma->cm_id->device->local_dma_lkey;
else {
rdma->dma_mr = ib_get_dma_mr(rdma->pd, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(rdma->dma_mr))
goto error;
rdma->lkey = rdma->dma_mr->lkey;
}
/* Create the Queue Pair */
memset(&qp_attr, 0, sizeof qp_attr);
qp_attr.event_handler = qp_event_handler;
qp_attr.qp_context = client;
qp_attr.cap.max_send_wr = opts.sq_depth;
qp_attr.cap.max_recv_wr = opts.rq_depth;
qp_attr.cap.max_send_sge = P9_RDMA_SEND_SGE;
qp_attr.cap.max_recv_sge = P9_RDMA_RECV_SGE;
qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
qp_attr.qp_type = IB_QPT_RC;
qp_attr.send_cq = rdma->cq;
qp_attr.recv_cq = rdma->cq;
err = rdma_create_qp(rdma->cm_id, rdma->pd, &qp_attr);
if (err)
goto error;
rdma->qp = rdma->cm_id->qp;
/* Request a connection */
memset(&conn_param, 0, sizeof(conn_param));
conn_param.private_data = NULL;
conn_param.private_data_len = 0;
conn_param.responder_resources = P9_RDMA_IRD;
conn_param.initiator_depth = P9_RDMA_ORD;
err = rdma_connect(rdma->cm_id, &conn_param);
if (err)
goto error;
err = wait_for_completion_interruptible(&rdma->cm_done);
if (err || (rdma->state != P9_RDMA_CONNECTED))
goto error;
client->status = Connected;
return 0;
error:
rdma_destroy_trans(rdma);
return -ENOTCONN;
}
struct xfer_context *xfer_rdma_client_connect(struct xfer_data *data)
{
struct addrinfo *res;
struct addrinfo hints = {
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM
};
char *service;
int n;
int n_retries = 10;
struct rdma_cm_event *event;
struct sockaddr_in sin;
struct xfer_context *ctx = NULL;
struct rdma_conn_param conn_param;
if (asprintf(&service, "%d", data->port) < 0)
goto err4;
n = getaddrinfo(data->servername, service, &hints, &res);
if (n < 0) {
fprintf(stderr, "%d:%s: %s for %s:%d\n",
pid, __func__, gai_strerror(n),
data->servername, data->port);
goto err4;
}
if (data->use_cma) {
sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr;
sin.sin_family = AF_INET;
sin.sin_port = htons(data->port);
retry_addr:
if (rdma_resolve_addr(data->cm_id, NULL,
(struct sockaddr *)&sin, 2000)) {
fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n",
pid, __func__ );
goto err2;
}
if (rdma_get_cm_event(data->cm_channel, &event))
goto err2;
if (event->event == RDMA_CM_EVENT_ADDR_ERROR
&& n_retries-- > 0) {
rdma_ack_cm_event(event);
goto retry_addr;
}
if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
fprintf(stderr, "%d:%s: unexpected CM event %d\n",
pid, __func__, event->event);
goto err2;
}
rdma_ack_cm_event(event);
retry_route:
if (rdma_resolve_route(data->cm_id, 2000)) {
fprintf(stderr, "%d:%s: rdma_resolve_route failed\n",
pid, __func__);
goto err2;
}
if (rdma_get_cm_event(data->cm_channel, &event))
goto err2;
if (event->event == RDMA_CM_EVENT_ROUTE_ERROR
&& n_retries-- > 0) {
rdma_ack_cm_event(event);
goto retry_route;
}
if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
fprintf(stderr, "%d:%s: unexpected CM event %d\n",
pid, __func__, event->event);
rdma_ack_cm_event(event);
goto err2;
}
rdma_ack_cm_event(event);
ctx = xfer_rdma_init_ctx(data->cm_id, data);
if (!ctx) {
fprintf(stderr, "%d:%s: xfer_init_ctx failed\n", pid, __func__);
goto err2;
}
memset(&conn_param, 0, sizeof conn_param);
conn_param.responder_resources = 16;
conn_param.initiator_depth = 16;
conn_param.retry_count = 5;
conn_param.private_data = data->local_priv;
conn_param.private_data_len = data->local_priv_size;
if (rdma_connect(data->cm_id, &conn_param)) {
fprintf(stderr, "%d:%s: rdma_connect failure\n", pid, __func__);
goto err2;
}
if (rdma_get_cm_event(data->cm_channel, &event))
goto err2;
if (event->event != RDMA_CM_EVENT_ESTABLISHED) {
fprintf(stderr, "%d:%s: unexpected CM event %d\n",
pid, __func__, event->event);
goto err1;
}
if (event->param.conn.private_data &&
(event->param.conn.private_data_len > 0)) {
data->remote_priv = malloc(event->param.conn.private_data_len);
if (!data->remote_priv)
goto err1;
memcpy(data->remote_priv, event->param.conn.private_data,
event->param.conn.private_data_len);
}
rdma_ack_cm_event(event);
} else {
// use an alternative to CMA here
}
freeaddrinfo(res);
return ctx;
err1:
rdma_ack_cm_event(event);
err2:
rdma_destroy_id(data->cm_id);
rdma_destroy_event_channel(data->cm_channel);
err4:
if (ctx)
xfer_rdma_destroy_ctx(ctx);
return NULL;
}
//static int run(int argc, char **argv)
//int RDMA_Connect(struct RDMA_communicator *comm, struct RDMA_param *param)
int RDMA_Active_Init(struct RDMA_communicator *comm, struct RDMA_param *param)
{
struct addrinfo *addr;
// struct rdma_cm_id *cm_id= NULL;
// struct rdma_event_channel *ec = NULL;
struct rdma_conn_param cm_params;
char port[8];
// int i,j;
sprintf(port, "%d", RDMA_PORT);
if(getaddrinfo(param->host, port, NULL, &addr)){
fprintf(stderr, "RDMA lib: SEND: ERROR: getaddrinfo failed @ %s:%d", __FILE__, __LINE__);
exit(1);
}
if(!(comm->ec = rdma_create_event_channel())){
fprintf(stderr, "RDMA lib: SEND: ERROR: rdma event channel create failed @ %s:%d", __FILE__, __LINE__);
exit(1);
}
if (rdma_create_id(comm->ec, &(comm->cm_id), NULL, RDMA_PS_TCP)){
fprintf(stderr, "RDMA lib: SEND: ERROR: rdma id create failed @ %s:%d", __FILE__, __LINE__);
exit(1);
}
if (rdma_resolve_addr(comm->cm_id, NULL, addr->ai_addr, TIMEOUT_IN_MS)) {
fprintf(stderr, "RDMA lib: SEND: ERROR: rdma address resolve failed @ %s:%d", __FILE__, __LINE__);
exit(1);
}
if (wait_for_event(comm->ec, RDMA_CM_EVENT_ADDR_RESOLVED)) {
fprintf(stderr, "RDMA lib: SEND: ERROR: event wait failed @ %s:%d", __FILE__, __LINE__);
exit(1);
}
freeaddrinfo(addr);
build_connection(comm->cm_id);
if (rdma_resolve_route(comm->cm_id, TIMEOUT_IN_MS)) {
fprintf(stderr, "RDMA lib: SEND: ERROR: rdma route resolve failed @ %s:%d", __FILE__, __LINE__);
exit(1);
}
if (wait_for_event(comm->ec, RDMA_CM_EVENT_ROUTE_RESOLVED)) {
fprintf(stderr, "RDMA lib: SEND: ERROR: event wait failed @ %s:%d", __FILE__, __LINE__);
exit(1);
}
build_params(&cm_params);
if (rdma_connect(comm->cm_id, &cm_params)) {
fprintf(stderr, "RDMA lib: SEND: ERROR: rdma connection failed @ %s:%d", __FILE__, __LINE__);
exit(1);
}
if (wait_for_event(comm->ec, RDMA_CM_EVENT_ESTABLISHED)) {
fprintf(stderr, "RDMA lib: SEND: ERROR: event wait failed @ %s:%d\n", __FILE__, __LINE__);
exit(1);
}
// on_connect(cm_id->context);
int i ;
for (i = 0; i < RDMA_BUF_NUM_C; i++){ rdma_msg_mr[i] = NULL;}
char *value;
value = getenv("RDMA_CLIENT_NUM_S");
if (value == NULL) {
rdma_buf_size = RDMA_BUF_SIZE_C;
} else {
rdma_buf_size = MAX_RDMA_BUF_SIZE_C / atoi(value);
}
fprintf(stderr, "rdma_buf_size: %d\n", rdma_buf_size);
return 0;
}
void client_test(char *ip, char *port) {
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
struct timeval t1, t2, t3, t11, t12;
struct timeval dt, dt1, dt2, dt11, dt12, dt13;
gettimeofday(&t1, NULL);
TEST_NZ(getaddrinfo(ip, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
gettimeofday(&t11, NULL);
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
gettimeofday(&t12, NULL);
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
gettimeofday(&t2, NULL);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy)) {
s_ctx->ec = ec;
s_ctx->id = conn;
on_connection(event_copy.id);//send our memory information to server using post_send
poll_cq(NULL);//wait for send_completion
poll_cq(NULL);//wait for recv_completion
break;
}
}
gettimeofday(&t3, NULL);
timersub(&t3, &t1, &dt);
timersub(&t3, &t2, &dt2);
timersub(&t2, &t1, &dt1);
timersub(&t2, &t12, &dt13);
timersub(&t12, &t11, &dt12);
timersub(&t11, &t1, &dt11);
long usec = dt.tv_usec + 10000 * dt.tv_sec;
printf("[dt]:\t%ld us.\n", usec);
printf("[dt1]:\t%ld us.\n", dt1.tv_usec+1000000 *dt1.tv_sec);
printf("Including the following steps: \n");
printf("[dt11]:\t%ld us.\n", dt11.tv_usec+1000000 *dt11.tv_sec);
printf("[dt12]:\t%ld us.\n", dt12.tv_usec+1000000 *dt12.tv_sec);
printf("[dt13]:\t%ld us.\n", dt13.tv_usec+1000000 *dt13.tv_sec);
printf("[dt2] takes %ld micro_secs.\n", dt2.tv_usec+1000000*dt2.tv_sec);
printf("[dt]:total time\t[dt1]:pre_setup\t[dt2]:send/recv\t.\n");
printf("[dt11]:create_event_channel\t[dt12]:create_id\t[dt13]:resolve_address.\n");
return;
};
static int run(void)
{
int i, ret;
printf("mckey: starting %s\n", is_sender ? "client" : "server");
if (src_addr) {
ret = get_addr(src_addr, (struct sockaddr *) &test.src_in);
if (ret)
return ret;
}
ret = get_addr(dst_addr, (struct sockaddr *) &test.dst_in);
if (ret)
return ret;
printf("mckey: joining\n");
for (i = 0; i < connections; i++) {
if (src_addr) {
ret = rdma_bind_addr(test.nodes[i].cma_id,
test.src_addr);
if (ret) {
perror("mckey: addr bind failure");
connect_error();
return ret;
}
}
if (unmapped_addr)
ret = addr_handler(&test.nodes[i]);
else
ret = rdma_resolve_addr(test.nodes[i].cma_id,
test.src_addr, test.dst_addr,
2000);
if (ret) {
perror("mckey: resolve addr failure");
connect_error();
return ret;
}
}
ret = connect_events();
if (ret)
goto out;
pthread_create(&test.cmathread, NULL, cma_thread, NULL);
/*
* Pause to give SM chance to configure switches. We don't want to
* handle reliability issue in this simple test program.
*/
sleep(3);
if (message_count) {
if (is_sender) {
printf("initiating data transfers\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i], 0);
if (ret)
goto out;
}
} else {
printf("receiving data transfers\n");
ret = poll_cqs();
if (ret)
goto out;
}
printf("data transfers complete\n");
}
out:
for (i = 0; i < connections; i++) {
ret = rdma_leave_multicast(test.nodes[i].cma_id,
test.dst_addr);
if (ret)
perror("mckey: failure leaving");
}
return ret;
}
static int fio_rdmaio_setup_connect(struct thread_data *td, const char *host,
unsigned short port)
{
struct rdmaio_data *rd = td->io_ops->data;
struct ibv_recv_wr *bad_wr;
int err;
rd->addr.sin_family = AF_INET;
rd->addr.sin_port = htons(port);
if (inet_aton(host, &rd->addr.sin_addr) != 1) {
struct hostent *hent;
hent = gethostbyname(host);
if (!hent) {
td_verror(td, errno, "gethostbyname");
return 1;
}
memcpy(&rd->addr.sin_addr, hent->h_addr, 4);
}
/* resolve route */
err = rdma_resolve_addr(rd->cm_id, NULL, (struct sockaddr *)&rd->addr, 2000);
if (err != 0) {
log_err("fio: rdma_resolve_addr: %d\n", err);
return 1;
}
err = get_next_channel_event(td, rd->cm_channel, RDMA_CM_EVENT_ADDR_RESOLVED);
if (err != 0) {
log_err("fio: get_next_channel_event: %d\n", err);
return 1;
}
/* resolve route */
err = rdma_resolve_route(rd->cm_id, 2000);
if (err != 0) {
log_err("fio: rdma_resolve_route: %d\n", err);
return 1;
}
err = get_next_channel_event(td, rd->cm_channel, RDMA_CM_EVENT_ROUTE_RESOLVED);
if (err != 0) {
log_err("fio: get_next_channel_event: %d\n", err);
return 1;
}
/* create qp and buffer */
if (fio_rdmaio_setup_qp(td) != 0)
return 1;
if (fio_rdmaio_setup_control_msg_buffers(td) != 0)
return 1;
/* post recv buf */
err = ibv_post_recv(rd->qp, &rd->rq_wr, &bad_wr);
if (err != 0) {
log_err("fio: ibv_post_recv fail: %d\n", err);
return 1;
}
return 0;
}
int rdma_client_connect(struct pingpong_context *ctx,struct perftest_parameters *user_param)
{
char *service;
int temp,num_of_retry= NUM_OF_RETRIES;
struct sockaddr_in sin;
struct addrinfo *res;
struct rdma_cm_event *event;
struct rdma_conn_param conn_param;
struct addrinfo hints;
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
if (check_add_port(&service,user_param->port,user_param->servername,&hints,&res)) {
fprintf(stderr, "Problem in resolving basic adress and port\n");
return FAILURE;
}
sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr;
sin.sin_family = PF_INET;
sin.sin_port = htons((unsigned short)user_param->port);
while (1) {
if (num_of_retry == 0) {
fprintf(stderr, "Received %d times ADDR_ERROR\n",NUM_OF_RETRIES);
return FAILURE;
}
if (rdma_resolve_addr(ctx->cm_id, NULL,(struct sockaddr *)&sin,2000)) {
fprintf(stderr, "rdma_resolve_addr failed\n");
return FAILURE;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return FAILURE;
}
if (event->event == RDMA_CM_EVENT_ADDR_ERROR) {
num_of_retry--;
rdma_ack_cm_event(event);
continue;
}
if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
fprintf(stderr, "unexpected CM event %d\n",event->event);
rdma_ack_cm_event(event);
return FAILURE;
}
rdma_ack_cm_event(event);
break;
}
if (user_param->tos != DEF_TOS) {
if (rdma_set_option(ctx->cm_id,RDMA_OPTION_ID,RDMA_OPTION_ID_TOS,&user_param->tos,sizeof(uint8_t))) {
fprintf(stderr, " Set TOS option failed: %d\n",event->event);
return FAILURE;
}
}
while (1) {
if (num_of_retry <= 0) {
fprintf(stderr, "Received %d times ADDR_ERROR - aborting\n",NUM_OF_RETRIES);
return FAILURE;
}
if (rdma_resolve_route(ctx->cm_id,2000)) {
fprintf(stderr, "rdma_resolve_route failed\n");
return FAILURE;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return FAILURE;
}
if (event->event == RDMA_CM_EVENT_ROUTE_ERROR) {
num_of_retry--;
rdma_ack_cm_event(event);
continue;
}
if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
fprintf(stderr, "unexpected CM event %d\n",event->event);
rdma_ack_cm_event(event);
return FAILURE;
}
rdma_ack_cm_event(event);
break;
}
ctx->context = ctx->cm_id->verbs;
temp = user_param->work_rdma_cm;
user_param->work_rdma_cm = ON;
if (ctx_init(ctx,user_param)) {
fprintf(stderr," Unable to create the resources needed by comm struct\n");
return FAILURE;
}
memset(&conn_param, 0, sizeof conn_param);
if (user_param->verb == READ || user_param->verb == ATOMIC) {
conn_param.responder_resources = user_param->out_reads;
conn_param.initiator_depth = user_param->out_reads;
}
user_param->work_rdma_cm = temp;
conn_param.retry_count = user_param->retry_count;
conn_param.rnr_retry_count = 7;
if (user_param->work_rdma_cm == OFF) {
if (post_one_recv_wqe(ctx)) {
fprintf(stderr, "Couldn't post send \n");
return 1;
}
}
if (rdma_connect(ctx->cm_id,&conn_param)) {
fprintf(stderr, "Function rdma_connect failed\n");
return FAILURE;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return FAILURE;
}
if (event->event != RDMA_CM_EVENT_ESTABLISHED) {
rdma_ack_cm_event(event);
fprintf(stderr, "Unexpected CM event bl blka %d\n", event->event);
return FAILURE;
}
if (user_param->connection_type == UD) {
user_param->rem_ud_qpn = event->param.ud.qp_num;
user_param->rem_ud_qkey = event->param.ud.qkey;
ctx->ah[0] = ibv_create_ah(ctx->pd,&event->param.ud.ah_attr);
if (!ctx->ah) {
printf(" Unable to create address handler for UD QP\n");
return FAILURE;
}
if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) {
if (send_qp_num_for_ah(ctx,user_param)) {
printf(" Unable to send my QP number\n");
return FAILURE;
}
}
}
rdma_ack_cm_event(event);
return SUCCESS;
}
