int main(int argc, char *argv[]) {
struct ibv_device *ib_dev;
struct pingpong_context ctx;
struct pingpong_dest *my_dest,*rem_dest;
struct perftest_parameters user_param;
struct perftest_comm user_comm;
int i = 0;
memset(&ctx,0,sizeof(struct pingpong_context));
memset(&user_param, 0, sizeof(struct perftest_parameters));
memset(&user_comm,0,sizeof(struct perftest_comm));
user_param.verb = WRITE;
user_param.tst = BW;
user_param.spec = PL;
user_param.version = VERSION;
// Configure the parameters values according to user arguments or defalut values.
if (parser(&user_param,argv,argc)) {
fprintf(stderr," Parser function exited with Error\n");
return 1;
}
// Finding the IB device selected (or defalut if no selected).
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE deivce\n");
return 1;
}
// Getting the relevant context from the device
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
return 1;
}
// See if MTU and link type are valid and supported.
if (check_link_and_mtu(ctx.context,&user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
// Print basic test information.
ctx_print_test_info(&user_param);
ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
// copy the rellevant user parameters to the comm struct + creating rdma_cm resources.
if (create_comm_struct(&user_comm,&user_param)) {
fprintf(stderr," Unable to create RDMA_CM resources\n");
return 1;
}
// Create (if nessacery) the rdma_cm ids and channel.
if (user_param.work_rdma_cm == ON) {
if (create_rdma_resources(&ctx,&user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (user_param.machine == CLIENT) {
if (rdma_client_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
if (rdma_server_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
}
} else {
// create all the basic IB resources (data buffer, PD, MR, CQ and events channel)
if (ctx_init(&ctx,&user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
}
// Set up the Connection.
if (set_up_connection(&ctx,&user_param,my_dest)) {
fprintf(stderr," Unable to set up socket connection\n");
return FAILURE;
}
// Print this machine QP information
for (i=0; i < user_param.num_of_qps; i++)
ctx_print_pingpong_data(&my_dest[i],&user_comm);
// Init the connection and print the local data.
if (establish_connection(&user_comm)) {
fprintf(stderr," Unable to init the socket connection\n");
return FAILURE;
}
// shaking hands and gather the other side info.
for (i=0; i < user_param.num_of_qps; i++) {
if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
fprintf(stderr," Failed to exchange date between server and clients\n");
return 1;
}
// Print remote machine QP information
user_comm.rdma_params->side = REMOTE;
ctx_print_pingpong_data(&rem_dest[i],&user_comm);
if (user_param.work_rdma_cm == OFF) {
if (pp_connect_ctx(&ctx,my_dest[i].psn,&rem_dest[i],&user_param,i)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
return FAILURE;
}
}
// An additional handshake is required after moving qp to RTR.
if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
fprintf(stderr," Failed to exchange date between server and clients\n");
return FAILURE;
}
}
printf(RESULT_LINE);
printf(RESULT_FMT);
// For half duplex tests, server just waits for client to exit
if (user_param.machine == SERVER && !user_param.duplex) {
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
printf(RESULT_LINE);
return 0;
}
ALLOCATE(tposted,cycles_t,user_param.iters*user_param.num_of_qps);
ALLOCATE(tcompleted,cycles_t,user_param.iters*user_param.num_of_qps);
if (user_param.all == ON) {
for (i = 1; i < 24 ; ++i) {
user_param.size = 1 << i;
if(run_iter(&ctx,&user_param,rem_dest))
return 17;
print_report(&user_param);
}
} else {
if(run_iter(&ctx,&user_param,rem_dest))
return 18;
print_report(&user_param);
}
free(tposted);
free(tcompleted);
// Closing connection.
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
free(my_dest);
free(rem_dest);
printf(RESULT_LINE);
return 0;
}
/*---------------------------------------------------------------------------*/
int run_client_test(struct perf_parameters *user_param)
{
struct session_data sess_data;
struct perf_comm *comm;
struct thread_data *tdata;
char url[256];
int i = 0;
int max_cpus;
pthread_t statistics_thread_id;
struct perf_command command;
int size_log2;
int max_size_log2 = 24;
/* client session attributes */
struct xio_session_attr attr = {
&ses_ops,
NULL,
0
};
xio_init();
g_mhz = get_cpu_mhz(0);
max_cpus = sysconf(_SC_NPROCESSORS_ONLN);
threads_iter = 1;
size_log2 = 0;
tdata = calloc(user_param->threads_num, sizeof(*tdata));
if (tdata == NULL) {
fprintf(fd, "malloc failed\n");
return -1;
}
comm = create_comm_struct(user_param);
if (establish_connection(comm)) {
fprintf(stderr, "failed to establish connection\n");
free(tdata);
destroy_comm_struct(comm);
return -1;
}
if (user_param->output_file) {
fd = fopen(user_param->output_file, "w");
if (fd == NULL) {
fprintf(fd, "file open failed. %s\n",
user_param->output_file);
free(sess_data.tdata);
destroy_comm_struct(comm);
return -1;
}
fprintf(fd, "size, threads, tps, bw[Mbps], lat[usec]\n");
fflush(fd);
}
printf("%s", RESULT_FMT);
printf("%s", RESULT_LINE);
while (threads_iter <= user_param->threads_num) {
data_len = (uint64_t)1 << size_log2;
memset(&sess_data, 0, sizeof(sess_data));
memset(tdata, 0, user_param->threads_num*sizeof(*tdata));
sess_data.tdata = tdata;
command.test_param.machine_type = user_param->machine_type;
command.test_param.test_type = user_param->test_type;
command.test_param.verb = user_param->verb;
command.test_param.data_len = data_len;
command.command = GetTestParams;
ctx_write_data(comm, &command, sizeof(command));
sprintf(url, "rdma://%s:%d", user_param->server_addr,
user_param->server_port);
sess_data.session = xio_session_create(XIO_SESSION_CLIENT,
&attr, url, 0, 0, &sess_data);
if (sess_data.session == NULL) {
int error = xio_errno();
fprintf(stderr,
"session creation failed. reason %d - (%s)\n",
error, xio_strerror(error));
goto cleanup;
}
pthread_create(&statistics_thread_id, NULL,
statistics_thread_cb, &sess_data);
/* spawn threads to handle connection */
for (i = 0; i < threads_iter; i++) {
sess_data.tdata[i].affinity =
((user_param->cpu + i) % max_cpus);
sess_data.tdata[i].cid = i;
sess_data.tdata[i].sdata = &sess_data;
sess_data.tdata[i].user_param = user_param;
sess_data.tdata[i].data_len = data_len;
/* all threads are working on the same session */
sess_data.tdata[i].session = sess_data.session;
pthread_create(&sess_data.tdata[i].thread_id, NULL,
worker_thread, &sess_data.tdata[i]);
}
pthread_join(statistics_thread_id, NULL);
/* join the threads */
for (i = 0; i < threads_iter; i++)
pthread_join(sess_data.tdata[i].thread_id, NULL);
/* close the session */
xio_session_destroy(sess_data.session);
if (sess_data.abort) {
fprintf(stderr, "program aborted\n");
goto cleanup;
}
/* send result to server */
command.results.bytes = data_len;
command.results.threads = threads_iter;
command.results.tps = sess_data.tps;
command.results.avg_bw = sess_data.avg_bw;
command.results.avg_lat = sess_data.avg_lat_us;
command.results.min_lat = sess_data.min_lat_us;
command.results.max_lat = sess_data.max_lat_us;
command.command = GetTestResults;
/* sync point */
ctx_write_data(comm, &command, sizeof(command));
printf(REPORT_FMT,
data_len,
threads_iter,
sess_data.tps,
sess_data.avg_bw,
sess_data.avg_lat_us,
sess_data.min_lat_us,
sess_data.max_lat_us);
if (fd)
fprintf(fd, "%lu, %d, %lu, %.2lf, %.2lf\n",
data_len,
threads_iter,
sess_data.tps,
sess_data.avg_bw,
sess_data.avg_lat_us);
fflush(fd);
/* sync point */
ctx_read_data(comm, NULL, 0, NULL);
if (++size_log2 < max_size_log2)
continue;
threads_iter++;
size_log2 = 0;
}
printf("%s", RESULT_LINE);
cleanup:
if (fd)
fclose(fd);
ctx_hand_shake(comm);
ctx_close_connection(comm);
destroy_comm_struct(comm);
free(tdata);
xio_shutdown();
return 0;
}
int main(int argc, char *argv[]) {
int i = 0;
int size_max_pow = 24;
int ret_val;
struct report_options report;
struct pingpong_context ctx;
struct pingpong_dest *my_dest = NULL;
struct pingpong_dest *rem_dest = NULL;
struct mcast_parameters mcg_params;
struct ibv_device *ib_dev = NULL;
struct perftest_parameters user_param;
struct perftest_comm user_comm;
/* init default values to user's parameters */
memset(&ctx, 0, sizeof(struct pingpong_context));
memset(&user_param, 0, sizeof(struct perftest_parameters));
memset(&user_comm , 0, sizeof(struct perftest_comm));
memset(&mcg_params, 0, sizeof(struct mcast_parameters));
user_param.verb = SEND;
user_param.tst = LAT;
strncpy(user_param.version, VERSION, sizeof(user_param.version));
user_param.r_flag = &report;
// Configure the parameters values according to user arguments or defalut values.
ret_val = parser(&user_param,argv,argc);
if (ret_val) {
if (ret_val != VERSION_EXIT && ret_val != HELP_EXIT)
fprintf(stderr," Parser function exited with Error\n");
return 1;
}
if(user_param.use_xrc || user_param.connection_type == DC) {
user_param.num_of_qps *= 2;
}
//Checking that the user did not run with RawEth. for this we have raw_etherent_bw test.
if (user_param.connection_type == RawEth) {
fprintf(stderr," This test cannot run Raw Ethernet QPs (you have chosen RawEth as connection type\n");
return FAILURE;
}
// Finding the IB device selected (or defalut if no selected).
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
return 1;
}
if (user_param.use_mcg)
GET_STRING(mcg_params.ib_devname,ibv_get_device_name(ib_dev));
// Getting the relevant context from the device
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
return 1;
}
// See if MTU and link type are valid and supported.
if (check_link(ctx.context,&user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
// copy the relevant user parameters to the comm struct + creating rdma_cm resources.
if (create_comm_struct(&user_comm,&user_param)) {
fprintf(stderr," Unable to create RDMA_CM resources\n");
return 1;
}
if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
printf("\n************************************\n");
printf("* Waiting for client to connect... *\n");
printf("************************************\n");
}
// Initialize the connection and print the local data.
if (establish_connection(&user_comm)) {
fprintf(stderr," Unable to init the socket connection\n");
return FAILURE;
}
exchange_versions(&user_comm, &user_param);
check_sys_data(&user_comm, &user_param);
// See if MTU and link type are valid and supported.
if (check_mtu(ctx.context,&user_param, &user_comm)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
// Print basic test information.
ctx_print_test_info(&user_param);
ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
// Allocating arrays needed for the test.
alloc_ctx(&ctx,&user_param);
// Create (if nessacery) the rdma_cm ids and channel.
if (user_param.work_rdma_cm == ON) {
if (user_param.machine == CLIENT) {
if (retry_rdma_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
if (create_rdma_resources(&ctx,&user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (rdma_server_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
}
} else {
// create all the basic IB resources (data buffer, PD, MR, CQ and events channel)
if (ctx_init(&ctx,&user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
}
// Set up the Connection.
if (send_set_up_connection(&ctx,&user_param,my_dest,&mcg_params,&user_comm)) {
fprintf(stderr," Unable to set up socket connection\n");
return 1;
}
for (i=0; i < user_param.num_of_qps; i++)
ctx_print_pingpong_data(&my_dest[i],&user_comm);
user_comm.rdma_params->side = REMOTE;
for (i=0; i < user_param.num_of_qps; i++) {
// shaking hands and gather the other side info.
if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
ctx_print_pingpong_data(&rem_dest[i],&user_comm);
}
if (user_param.work_rdma_cm == OFF)
{
if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0]))
{
fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
fprintf(stderr," Please Try to use a different IP version.\n\n");
return 1;
}
}
if (user_param.use_mcg) {
memcpy(mcg_params.base_mgid.raw,mcg_params.mgid.raw,16);
memcpy(mcg_params.mgid.raw,rem_dest[0].gid.raw,16);
mcg_params.base_mlid = mcg_params.mlid;
mcg_params.is_2nd_mgid_used = ON;
if (!strcmp(link_layer_str(user_param.link_type),"IB")) {
// Request for Mcast group create registery in SM.
if (join_multicast_group(SUBN_ADM_METHOD_SET,&mcg_params)) {
fprintf(stderr," Failed to Join Mcast request\n");
return 1;
}
}
/*
* The next stall in code (50 ms sleep) is a work around for fixing the
* the bug this test had in Multicast for the past 1 year.
* It appears, that when a switch involved, it takes ~ 10 ms for the join
* request to propogate on the IB fabric, thus we need to wait for it.
* what happened before this fix was reaching the post_send
* code segment in about 350 ns from here, and the switch(es) dropped
* the packet because join request wasn't finished.
*/
usleep(50000);
}
if (user_param.work_rdma_cm == OFF) {
// Prepare IB resources for rtr/rts.
if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
return 1;
}
}
// shaking hands and gather the other side info.
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
if (user_param.use_event) {
if (ibv_req_notify_cq(ctx.send_cq, 0)) {
fprintf(stderr, "Couldn't request RCQ notification\n");
return 1;
}
if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
fprintf(stderr, "Couldn't request RCQ notification\n");
return 1;
}
}
if (user_param.output == FULL_VERBOSITY) {
printf(RESULT_LINE);
printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
}
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
if (user_param.test_method == RUN_ALL) {
if (user_param.connection_type == UD)
size_max_pow = (int)UD_MSG_2_EXP(MTU_SIZE(user_param.curr_mtu)) + 1;
for (i = 1; i < size_max_pow ; ++i) {
user_param.size = (uint64_t)1 << i;
// Post recevie recv_wqes fo current message size
if (ctx_set_recv_wqes(&ctx,&user_param)) {
fprintf(stderr," Failed to post receive recv_wqes\n");
return 1;
}
// Sync between the client and server so the client won't send packets
// Before the server has posted his receive wqes (in UC/UD it will result in a deadlock).
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
if(run_iter_lat_send(&ctx, &user_param))
return 17;
user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
}
} else {
// Post recevie recv_wqes fo current message size
if (ctx_set_recv_wqes(&ctx,&user_param)) {
fprintf(stderr," Failed to post receive recv_wqes\n");
return 1;
}
// Sync between the client and server so the client won't send packets
// Before the server has posted his receive wqes (in UC/UD it will result in a deadlock).
if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
if(run_iter_lat_send(&ctx, &user_param))
return 17;
user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
}
if (user_param.output == FULL_VERBOSITY) {
printf(RESULT_LINE);
}
if (ctx_close_connection(&user_comm,my_dest,rem_dest)) {
fprintf(stderr,"Failed to close connection between server and client\n");
fprintf(stderr," Trying to close this side resources\n");
}
return send_destroy_ctx(&ctx,&user_param,&mcg_params);
}
int main(int argc, char *argv[]) {
int i = 0;
struct report_options report = {};
struct pingpong_context ctx;
struct ibv_device *ib_dev;
struct perftest_parameters user_param;
struct pingpong_dest my_dest,rem_dest;
struct perftest_comm user_comm;
/* init default values to user's parameters */
memset(&ctx,0,sizeof(struct pingpong_context));
memset(&user_param,0,sizeof(struct perftest_parameters));
memset(&user_comm,0,sizeof(struct perftest_comm));
memset(&my_dest,0,sizeof(struct pingpong_dest));
memset(&rem_dest,0,sizeof(struct pingpong_dest));
user_param.verb = READ;
user_param.tst = LAT;
user_param.r_flag = &report;
user_param.version = VERSION;
// Configure the parameters values according to user arguments or defalut values.
if (parser(&user_param,argv,argc)) {
fprintf(stderr," Parser function exited with Error\n");
return FAILURE;
}
// Finding the IB device selected (or defalut if no selected).
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE deivce\n");
return FAILURE;
}
// Getting the relevant context from the device
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
return 1;
}
// See if MTU and link type are valid and supported.
if (check_link_and_mtu(ctx.context,&user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
// Print basic test information.
ctx_print_test_info(&user_param);
// copy the rellevant user parameters to the comm struct + creating rdma_cm resources.
if (create_comm_struct(&user_comm,&user_param)) {
fprintf(stderr," Unable to create RDMA_CM resources\n");
return 1;
}
// Create (if nessacery) the rdma_cm ids and channel.
if (user_param.work_rdma_cm == ON) {
if (create_rdma_resources(&ctx,&user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (user_param.machine == CLIENT) {
if (rdma_client_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
if (rdma_server_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
}
} else {
// create all the basic IB resources (data buffer, PD, MR, CQ and events channel)
if (ctx_init(&ctx,&user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
}
// Set up the Connection.
if (set_up_connection(&ctx,&user_param,&my_dest)) {
fprintf(stderr," Unable to set up socket connection\n");
return 1;
}
ctx_print_pingpong_data(&my_dest,&user_comm);
// Init the connection and print the local data.
if (establish_connection(&user_comm)) {
fprintf(stderr," Unable to init the socket connection\n");
return 1;
}
// shaking hands and gather the other side info.
if (ctx_hand_shake(&user_comm,&my_dest,&rem_dest)) {
fprintf(stderr,"Failed to exchange date between server and clients\n");
return 1;
}
user_comm.rdma_params->side = REMOTE;
ctx_print_pingpong_data(&rem_dest,&user_comm);
if (user_param.work_rdma_cm == OFF) {
if (pp_connect_ctx(&ctx,my_dest.psn,&rem_dest,my_dest.out_reads,&user_param)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
return 1;
}
}
// An additional handshake is required after moving qp to RTR.
if (ctx_hand_shake(&user_comm,&my_dest,&rem_dest)) {
fprintf(stderr,"Failed to exchange date between server and clients\n");
return 1;
}
ALLOCATE(tstamp,cycles_t,user_param.iters);
// Only Client post read request.
if (user_param.machine == SERVER) {
if (ctx_close_connection(&user_comm,&my_dest,&rem_dest)) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
printf(RESULT_LINE);
return 0; // destroy_ctx(&ctx,&user_param);
}
if (user_param.use_event) {
if (ibv_req_notify_cq(ctx.send_cq, 0)) {
fprintf(stderr, "Couldn't request CQ notification\n");
return 1;
}
}
printf(RESULT_LINE);
printf(RESULT_FMT_LAT);
if (user_param.all == ON) {
for (i = 1; i < 24 ; ++i) {
user_param.size = 1 << i;
if(run_iter(&ctx,&user_param,&rem_dest))
return 17;
print_report(&user_param);
}
} else {
if(run_iter(&ctx,&user_param,&rem_dest))
return 18;
print_report(&user_param);
}
if (ctx_close_connection(&user_comm,&my_dest,&rem_dest)) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
printf(RESULT_LINE);
return 0; // destroy_ctx(&ctx,&user_param);
}
int main(int argc, char *argv[])
{
struct ibv_device *ib_dev = NULL;
struct pingpong_context ctx;
struct pingpong_dest *my_dest = NULL;
struct pingpong_dest *rem_dest = NULL;
struct perftest_parameters user_param;
struct perftest_comm user_comm;
struct mcast_parameters mcg_params;
struct bw_report_data my_bw_rep, rem_bw_rep;
int ret_parser,i = 0;
int size_max_pow = 24;
/* init default values to user's parameters */
memset(&ctx, 0,sizeof(struct pingpong_context));
memset(&user_param, 0 , sizeof(struct perftest_parameters));
memset(&mcg_params, 0 , sizeof(struct mcast_parameters));
memset(&user_comm, 0,sizeof(struct perftest_comm));
user_param.verb = SEND;
user_param.tst = BW;
strncpy(user_param.version, VERSION, sizeof(user_param.version));
/* Configure the parameters values according to user arguments or defalut values. */
ret_parser = parser(&user_param,argv,argc);
if (ret_parser) {
if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
fprintf(stderr," Parser function exited with Error\n");
return 1;
}
if((user_param.connection_type == DC || user_param.use_xrc) && user_param.duplex) {
user_param.num_of_qps *= 2;
}
/* Checking that the user did not run with RawEth. for this we have raw_etherent_bw test. */
if (user_param.connection_type == RawEth) {
fprintf(stderr," This test cannot run Raw Ethernet QPs (you have chosen RawEth as connection type\n");
fprintf(stderr," For this we have raw_ethernet_bw test in this package.\n");
return FAILURE;
}
/* Finding the IB device selected (or defalut if no selected). */
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
return 1;
}
if (user_param.use_mcg)
GET_STRING(mcg_params.ib_devname,ibv_get_device_name(ib_dev));
/* Getting the relevant context from the device */
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
return 1;
}
/* See if MTU and link type are valid and supported. */
if (check_link(ctx.context,&user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
if (create_comm_struct(&user_comm,&user_param)) {
fprintf(stderr," Unable to create RDMA_CM resources\n");
return 1;
}
if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
printf("\n************************************\n");
printf("* Waiting for client to connect... *\n");
printf("************************************\n");
}
/* Initialize the connection and print the local data. */
if (establish_connection(&user_comm)) {
fprintf(stderr," Unable to init the socket connection\n");
return FAILURE;
}
exchange_versions(&user_comm, &user_param);
check_sys_data(&user_comm, &user_param);
/* See if MTU and link type are valid and supported. */
if (check_mtu(ctx.context,&user_param, &user_comm)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
/* Print basic test information. */
ctx_print_test_info(&user_param);
ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
if (user_param.transport_type == IBV_TRANSPORT_IWARP)
ctx.send_rcredit = 1;
/* Allocating arrays needed for the test. */
alloc_ctx(&ctx,&user_param);
/* Create (if nessacery) the rdma_cm ids and channel. */
if (user_param.work_rdma_cm == ON) {
if (user_param.machine == CLIENT) {
if (retry_rdma_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
if (create_rdma_resources(&ctx,&user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (rdma_server_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
}
} else {
/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
if (ctx_init(&ctx,&user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
}
/* Set up the Connection. */
if (send_set_up_connection(&ctx,&user_param,my_dest,&mcg_params,&user_comm)) {
fprintf(stderr," Unable to set up socket connection\n");
return 1;
}
if (ctx.send_rcredit)
ctx_alloc_credit(&ctx,&user_param,my_dest);
for (i=0; i < user_param.num_of_qps; i++)
ctx_print_pingpong_data(&my_dest[i],&user_comm);
user_comm.rdma_params->side = REMOTE;
for (i=0; i < user_param.num_of_qps; i++) {
/* shaking hands and gather the other side info. */
if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
ctx_print_pingpong_data(&rem_dest[i],&user_comm);
}
if (user_param.work_rdma_cm == OFF) {
if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
fprintf(stderr," Please Try to use a different IP version.\n\n");
return 1;
}
}
/* If credit for available recieve buffers is necessary,
* the credit sending is done via RDMA WRITE ops and the ctx_hand_shake above
* is used to exchange the rkeys and buf addresses for the RDMA WRITEs
*/
if (ctx.send_rcredit)
ctx_set_credit_wqes(&ctx,&user_param,rem_dest);
/* Joining the Send side port the Mcast gid */
if (user_param.use_mcg && (user_param.machine == CLIENT || user_param.duplex)) {
memcpy(mcg_params.mgid.raw, rem_dest[0].gid.raw, 16);
if (set_mcast_group(&ctx,&user_param,&mcg_params)) {
fprintf(stderr," Unable to Join Sender to Mcast gid\n");
return 1;
}
/*
* The next stall in code (50 ms sleep) is a work around for fixing the
* the bug this test had in Multicast for the past 1 year.
* It appears, that when a switch involved, it takes ~ 10 ms for the join
* request to propogate on the IB fabric, thus we need to wait for it.
* what happened before this fix was client reaching the post_send
* code segment in about 350 ns from here, and the switch(es) dropped
* the packet because join request wasn't finished.
*/
usleep(50000);
}
if (user_param.work_rdma_cm == OFF) {
/* Prepare IB resources for rtr/rts. */
if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
return 1;
}
}
/* shaking hands and gather the other side info. */
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
if (user_param.use_event) {
if (ibv_req_notify_cq(ctx.send_cq, 0)) {
fprintf(stderr, " Couldn't request CQ notification\n");
return 1;
}
if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
fprintf(stderr, " Couldn't request CQ notification\n");
return 1;
}
}
if (user_param.output == FULL_VERBOSITY) {
if (user_param.report_per_port) {
printf(RESULT_LINE_PER_PORT);
printf((user_param.report_fmt == MBS ? RESULT_FMT_PER_PORT : RESULT_FMT_G_PER_PORT));
}
else {
printf(RESULT_LINE);
printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
}
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
}
if (user_param.test_method == RUN_ALL) {
if (user_param.connection_type == UD)
size_max_pow = (int)UD_MSG_2_EXP(MTU_SIZE(user_param.curr_mtu)) + 1;
for (i = 1; i < size_max_pow ; ++i) {
user_param.size = (uint64_t)1 << i;
if (user_param.machine == CLIENT || user_param.duplex)
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
if (user_param.machine == SERVER || user_param.duplex) {
if (ctx_set_recv_wqes(&ctx,&user_param)) {
fprintf(stderr," Failed to post receive recv_wqes\n");
return 1;
}
}
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
if (ctx.send_rcredit) {
int j;
for (j = 0; j < user_param.num_of_qps; j++)
ctx.credit_buf[j] = 0;
}
if (user_param.duplex) {
if(run_iter_bi(&ctx,&user_param))
return 17;
} else if (user_param.machine == CLIENT) {
if(run_iter_bw(&ctx,&user_param)) {
return 17;
}
} else {
if(run_iter_bw_server(&ctx,&user_param)) {
return 17;
}
}
print_report_bw(&user_param,&my_bw_rep);
if (user_param.duplex && user_param.test_type != DURATION) {
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
}
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
/* Check if last iteration ended well in UC/UD */
if (user_param.check_alive_exited) {
break;
}
}
} else if (user_param.test_method == RUN_REGULAR) {
if (user_param.machine == CLIENT || user_param.duplex)
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
if (user_param.machine == SERVER || user_param.duplex) {
if (ctx_set_recv_wqes(&ctx,&user_param)) {
fprintf(stderr," Failed to post receive recv_wqes\n");
return 1;
}
}
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
if (user_param.duplex) {
if(run_iter_bi(&ctx,&user_param))
return 17;
} else if (user_param.machine == CLIENT) {
if(run_iter_bw(&ctx,&user_param)) {
return 17;
}
} else if(run_iter_bw_server(&ctx,&user_param)) {
return 17;
}
print_report_bw(&user_param,&my_bw_rep);
if (user_param.duplex && user_param.test_type != DURATION) {
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
}
if (user_param.report_both && user_param.duplex) {
printf(RESULT_LINE);
printf("\n Local results: \n");
printf(RESULT_LINE);
printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
print_full_bw_report(&user_param, &my_bw_rep, NULL);
printf(RESULT_LINE);
printf("\n Remote results: \n");
printf(RESULT_LINE);
printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
print_full_bw_report(&user_param, &rem_bw_rep, NULL);
}
} else if (user_param.test_method == RUN_INFINITELY) {
if (user_param.machine == CLIENT)
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
else if (user_param.machine == SERVER) {
if (ctx_set_recv_wqes(&ctx,&user_param)) {
fprintf(stderr," Failed to post receive recv_wqes\n");
return 1;
}
}
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
if (user_param.machine == CLIENT) {
if(run_iter_bw_infinitely(&ctx,&user_param)) {
fprintf(stderr," Error occured while running infinitely! aborting ...\n");
return 1;
}
} else if (user_param.machine == SERVER) {
if(run_iter_bw_infinitely_server(&ctx,&user_param)) {
fprintf(stderr," Error occured while running infinitely on server! aborting ...\n");
return 1;
}
}
}
if (user_param.output == FULL_VERBOSITY) {
if (user_param.report_per_port)
printf(RESULT_LINE_PER_PORT);
else
printf(RESULT_LINE);
}
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr," Failed to close connection between server and client\n");
fprintf(stderr," Trying to close this side resources\n");
}
/* Destory all test resources, including Mcast if exists */
if (send_destroy_ctx(&ctx,&user_param,&mcg_params)) {
fprintf(stderr,"Couldn't Destory all SEND resources\n");
return FAILURE;
}
if (user_param.work_rdma_cm == ON) {
user_comm.rdma_params->work_rdma_cm = ON;
if (destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params)) {
fprintf(stderr,"Failed to destroy resources\n");
return 1;
}
}
if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
fprintf(stderr,"Error: BW result is below bw limit\n");
return 1;
}
if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) {
fprintf(stderr,"Error: Msg rate is below msg_rate limit\n");
return 1;
}
return 0;
}
int main(int argc, char *argv[])
{
int ret_parser,i = 0;
struct ibv_device *ib_dev = NULL;
struct pingpong_context ctx;
struct pingpong_dest *my_dest = NULL;
struct pingpong_dest *rem_dest = NULL;
struct perftest_parameters user_param;
struct perftest_comm user_comm;
struct bw_report_data my_bw_rep, rem_bw_rep;
/* init default values to user's parameters */
memset(&ctx,0,sizeof(struct pingpong_context));
memset(&user_param , 0 , sizeof(struct perftest_parameters));
memset(&user_comm,0,sizeof(struct perftest_comm));
user_param.verb = READ;
user_param.tst = BW;
strncpy(user_param.version, VERSION, sizeof(user_param.version));
ret_parser = parser(&user_param,argv,argc);
if (ret_parser) {
if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
fprintf(stderr," Parser function exited with Error\n");
return 1;
}
if((user_param.connection_type == DC || user_param.use_xrc) && user_param.duplex) {
user_param.num_of_qps *= 2;
}
ib_dev =ctx_find_dev(user_param.ib_devname);
if (!ib_dev)
return 7;
/* Getting the relevant context from the device */
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
return 1;
}
/* See if MTU and link type are valid and supported. */
if (check_link(ctx.context,&user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
if (create_comm_struct(&user_comm,&user_param)) {
fprintf(stderr," Unable to create RDMA_CM resources\n");
return 1;
}
if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
printf("\n************************************\n");
printf("* Waiting for client to connect... *\n");
printf("************************************\n");
}
/* Initialize the connection and print the local data. */
if (establish_connection(&user_comm)) {
fprintf(stderr," Unable to init the socket connection\n");
return FAILURE;
}
exchange_versions(&user_comm, &user_param);
check_sys_data(&user_comm, &user_param);
/* See if MTU and link type are valid and supported. */
if (check_mtu(ctx.context,&user_param, &user_comm)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
/* Print basic test information. */
ctx_print_test_info(&user_param);
ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
/* Allocating arrays needed for the test. */
alloc_ctx(&ctx,&user_param);
/* Create (if nessacery) the rdma_cm ids and channel. */
if (user_param.work_rdma_cm == ON) {
if (user_param.machine == CLIENT) {
if (retry_rdma_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
if (create_rdma_resources(&ctx,&user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (rdma_server_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
}
} else {
/* create all the basic IB resources. */
if (ctx_init(&ctx,&user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
}
/* Set up the Connection. */
if (set_up_connection(&ctx,&user_param,my_dest)) {
fprintf(stderr," Unable to set up socket connection\n");
return FAILURE;
}
/* Print this machine QP information */
for (i=0; i < user_param.num_of_qps; i++)
ctx_print_pingpong_data(&my_dest[i],&user_comm);
user_comm.rdma_params->side = REMOTE;
for (i=0; i < user_param.num_of_qps; i++) {
/* shaking hands and gather the other side info. */
if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
ctx_print_pingpong_data(&rem_dest[i],&user_comm);
}
if (user_param.work_rdma_cm == OFF) {
if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
fprintf(stderr," Please Try to use a different IP version.\n\n");
return 1;
}
}
if (user_param.work_rdma_cm == OFF) {
if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
return 1;
}
}
/* An additional handshake is required after moving qp to RTR. */
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to exchange data between server and clients\n");
return 1;
}
if (user_param.output == FULL_VERBOSITY) {
if (user_param.report_per_port) {
printf(RESULT_LINE_PER_PORT);
printf((user_param.report_fmt == MBS ? RESULT_FMT_PER_PORT : RESULT_FMT_G_PER_PORT));
}
else {
printf(RESULT_LINE);
printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
}
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
}
/* For half duplex tests, server just waits for client to exit */
if (user_param.machine == SERVER && !user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
return FAILURE;
}
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
print_full_bw_report(&user_param, &rem_bw_rep, NULL);
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
if (user_param.output == FULL_VERBOSITY) {
if (user_param.report_per_port)
printf(RESULT_LINE_PER_PORT);
else
printf(RESULT_LINE);
}
if (user_param.work_rdma_cm == ON) {
if (destroy_ctx(&ctx,&user_param)) {
fprintf(stderr, "Failed to destroy resources\n");
return 1;
}
user_comm.rdma_params->work_rdma_cm = ON;
return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
}
return destroy_ctx(&ctx,&user_param);
}
if (user_param.use_event) {
if (ibv_req_notify_cq(ctx.send_cq, 0)) {
fprintf(stderr, "Couldn't request CQ notification\n");
return 1;
}
}
if (user_param.test_method == RUN_ALL) {
for (i = 1; i < 24 ; ++i) {
user_param.size = (uint64_t)1 << i;
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
if(perform_warm_up(&ctx,&user_param)) {
fprintf(stderr,"Problems with warm up\n");
return 1;
}
if(user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
return 1;
}
}
if(run_iter_bw(&ctx,&user_param))
return 17;
if (user_param.duplex && (atof(user_param.version) >= 4.6)) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
return 1;
}
}
print_report_bw(&user_param,&my_bw_rep);
if (user_param.duplex) {
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
}
}
} else if (user_param.test_method == RUN_REGULAR) {
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
if(perform_warm_up(&ctx,&user_param)) {
fprintf(stderr,"Problems with warm up\n");
return 1;
}
if(user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
return 1;
}
}
if(run_iter_bw(&ctx,&user_param)) {
fprintf(stderr," Failed to complete run_iter_bw function successfully\n");
return 1;
}
print_report_bw(&user_param,&my_bw_rep);
if (user_param.duplex) {
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
}
if (user_param.report_both && user_param.duplex) {
printf(RESULT_LINE);
printf("\n Local results: \n");
printf(RESULT_LINE);
printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
print_full_bw_report(&user_param, &my_bw_rep, NULL);
printf(RESULT_LINE);
printf("\n Remote results: \n");
printf(RESULT_LINE);
printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
print_full_bw_report(&user_param, &rem_bw_rep, NULL);
}
} else if (user_param.test_method == RUN_INFINITELY) {
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
if(run_iter_bw_infinitely(&ctx,&user_param)) {
fprintf(stderr," Error occured while running! aborting ...\n");
return 1;
}
}
if (user_param.output == FULL_VERBOSITY) {
if (user_param.report_per_port)
printf(RESULT_LINE_PER_PORT);
else
printf(RESULT_LINE);
}
/* For half duplex tests, server just waits for client to exit */
if (user_param.machine == CLIENT && !user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
return FAILURE;
}
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
}
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
fprintf(stderr,"Error: BW result is below bw limit\n");
return 1;
}
if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) {
fprintf(stderr,"Error: Msg rate is below msg_rate limit\n");
return 1;
}
if (user_param.work_rdma_cm == ON) {
if (destroy_ctx(&ctx,&user_param)) {
fprintf(stderr, "Failed to destroy resources\n");
return 1;
}
user_comm.rdma_params->work_rdma_cm = ON;
return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
}
return destroy_ctx(&ctx,&user_param);
}