int main(int argc, char *argv[]) {
int i = 0;
struct report_options report = {};
struct pingpong_context ctx;
struct ibv_device *ib_dev;
struct perftest_parameters user_param;
struct pingpong_dest my_dest,rem_dest;
struct perftest_comm user_comm;
/* init default values to user's parameters */
memset(&ctx,0,sizeof(struct pingpong_context));
memset(&user_param,0,sizeof(struct perftest_parameters));
memset(&user_comm,0,sizeof(struct perftest_comm));
memset(&my_dest,0,sizeof(struct pingpong_dest));
memset(&rem_dest,0,sizeof(struct pingpong_dest));
user_param.verb = READ;
user_param.tst = LAT;
user_param.r_flag = &report;
user_param.version = VERSION;
// Configure the parameters values according to user arguments or defalut values.
if (parser(&user_param,argv,argc)) {
fprintf(stderr," Parser function exited with Error\n");
return FAILURE;
}
// Finding the IB device selected (or defalut if no selected).
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE deivce\n");
return FAILURE;
}
// Getting the relevant context from the device
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
return 1;
}
// See if MTU and link type are valid and supported.
if (check_link_and_mtu(ctx.context,&user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
// Print basic test information.
ctx_print_test_info(&user_param);
// copy the rellevant user parameters to the comm struct + creating rdma_cm resources.
if (create_comm_struct(&user_comm,&user_param)) {
fprintf(stderr," Unable to create RDMA_CM resources\n");
return 1;
}
// Create (if nessacery) the rdma_cm ids and channel.
if (user_param.work_rdma_cm == ON) {
if (create_rdma_resources(&ctx,&user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (user_param.machine == CLIENT) {
if (rdma_client_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
if (rdma_server_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
}
} else {
// create all the basic IB resources (data buffer, PD, MR, CQ and events channel)
if (ctx_init(&ctx,&user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
}
// Set up the Connection.
if (set_up_connection(&ctx,&user_param,&my_dest)) {
fprintf(stderr," Unable to set up socket connection\n");
return 1;
}
ctx_print_pingpong_data(&my_dest,&user_comm);
// Init the connection and print the local data.
if (establish_connection(&user_comm)) {
fprintf(stderr," Unable to init the socket connection\n");
return 1;
}
// shaking hands and gather the other side info.
if (ctx_hand_shake(&user_comm,&my_dest,&rem_dest)) {
fprintf(stderr,"Failed to exchange date between server and clients\n");
return 1;
}
user_comm.rdma_params->side = REMOTE;
ctx_print_pingpong_data(&rem_dest,&user_comm);
if (user_param.work_rdma_cm == OFF) {
if (pp_connect_ctx(&ctx,my_dest.psn,&rem_dest,my_dest.out_reads,&user_param)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
return 1;
}
}
// An additional handshake is required after moving qp to RTR.
if (ctx_hand_shake(&user_comm,&my_dest,&rem_dest)) {
fprintf(stderr,"Failed to exchange date between server and clients\n");
return 1;
}
ALLOCATE(tstamp,cycles_t,user_param.iters);
// Only Client post read request.
if (user_param.machine == SERVER) {
if (ctx_close_connection(&user_comm,&my_dest,&rem_dest)) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
printf(RESULT_LINE);
return 0; // destroy_ctx(&ctx,&user_param);
}
if (user_param.use_event) {
if (ibv_req_notify_cq(ctx.send_cq, 0)) {
fprintf(stderr, "Couldn't request CQ notification\n");
return 1;
}
}
printf(RESULT_LINE);
printf(RESULT_FMT_LAT);
if (user_param.all == ON) {
for (i = 1; i < 24 ; ++i) {
user_param.size = 1 << i;
if(run_iter(&ctx,&user_param,&rem_dest))
return 17;
print_report(&user_param);
}
} else {
if(run_iter(&ctx,&user_param,&rem_dest))
return 18;
print_report(&user_param);
}
if (ctx_close_connection(&user_comm,&my_dest,&rem_dest)) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
printf(RESULT_LINE);
return 0; // destroy_ctx(&ctx,&user_param);
}
int main(int argc, char *argv[]) {
struct ibv_device *ib_dev;
struct pingpong_context ctx;
struct pingpong_dest *my_dest,*rem_dest;
struct perftest_parameters user_param;
struct perftest_comm user_comm;
int i = 0;
memset(&ctx,0,sizeof(struct pingpong_context));
memset(&user_param, 0, sizeof(struct perftest_parameters));
memset(&user_comm,0,sizeof(struct perftest_comm));
user_param.verb = WRITE;
user_param.tst = BW;
user_param.spec = PL;
user_param.version = VERSION;
// Configure the parameters values according to user arguments or defalut values.
if (parser(&user_param,argv,argc)) {
fprintf(stderr," Parser function exited with Error\n");
return 1;
}
// Finding the IB device selected (or defalut if no selected).
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE deivce\n");
return 1;
}
// Getting the relevant context from the device
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
return 1;
}
// See if MTU and link type are valid and supported.
if (check_link_and_mtu(ctx.context,&user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
// Print basic test information.
ctx_print_test_info(&user_param);
ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
// copy the rellevant user parameters to the comm struct + creating rdma_cm resources.
if (create_comm_struct(&user_comm,&user_param)) {
fprintf(stderr," Unable to create RDMA_CM resources\n");
return 1;
}
// Create (if nessacery) the rdma_cm ids and channel.
if (user_param.work_rdma_cm == ON) {
if (create_rdma_resources(&ctx,&user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (user_param.machine == CLIENT) {
if (rdma_client_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
if (rdma_server_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
}
} else {
// create all the basic IB resources (data buffer, PD, MR, CQ and events channel)
if (ctx_init(&ctx,&user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
}
// Set up the Connection.
if (set_up_connection(&ctx,&user_param,my_dest)) {
fprintf(stderr," Unable to set up socket connection\n");
return FAILURE;
}
// Print this machine QP information
for (i=0; i < user_param.num_of_qps; i++)
ctx_print_pingpong_data(&my_dest[i],&user_comm);
// Init the connection and print the local data.
if (establish_connection(&user_comm)) {
fprintf(stderr," Unable to init the socket connection\n");
return FAILURE;
}
// shaking hands and gather the other side info.
for (i=0; i < user_param.num_of_qps; i++) {
if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
fprintf(stderr," Failed to exchange date between server and clients\n");
return 1;
}
// Print remote machine QP information
user_comm.rdma_params->side = REMOTE;
ctx_print_pingpong_data(&rem_dest[i],&user_comm);
if (user_param.work_rdma_cm == OFF) {
if (pp_connect_ctx(&ctx,my_dest[i].psn,&rem_dest[i],&user_param,i)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
return FAILURE;
}
}
// An additional handshake is required after moving qp to RTR.
if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
fprintf(stderr," Failed to exchange date between server and clients\n");
return FAILURE;
}
}
printf(RESULT_LINE);
printf(RESULT_FMT);
// For half duplex tests, server just waits for client to exit
if (user_param.machine == SERVER && !user_param.duplex) {
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
printf(RESULT_LINE);
return 0;
}
ALLOCATE(tposted,cycles_t,user_param.iters*user_param.num_of_qps);
ALLOCATE(tcompleted,cycles_t,user_param.iters*user_param.num_of_qps);
if (user_param.all == ON) {
for (i = 1; i < 24 ; ++i) {
user_param.size = 1 << i;
if(run_iter(&ctx,&user_param,rem_dest))
return 17;
print_report(&user_param);
}
} else {
if(run_iter(&ctx,&user_param,rem_dest))
return 18;
print_report(&user_param);
}
free(tposted);
free(tcompleted);
// Closing connection.
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
return 1;
}
free(my_dest);
free(rem_dest);
printf(RESULT_LINE);
return 0;
}
/*
* Main function. implements raw_ethernet_send_lat
*/
int main(int argc, char *argv[])
{
struct ibv_device *ib_dev = NULL;
struct pingpong_context ctx;
struct raw_ethernet_info my_dest_info,rem_dest_info;
int ret_parser;
struct perftest_parameters user_param;
#ifdef HAVE_RAW_ETH_EXP
struct ibv_exp_flow *flow_create_result = NULL;
struct ibv_exp_flow_attr *flow_rules = NULL;
struct ibv_exp_flow *flow_promisc = NULL;
#else
struct ibv_flow *flow_create_result = NULL;
struct ibv_flow_attr *flow_rules = NULL;
#endif
struct report_options report;
//allocate memory space for user parameters
memset(&ctx, 0, sizeof(struct pingpong_context));
memset(&user_param, 0, sizeof(struct perftest_parameters));
memset(&my_dest_info, 0 , sizeof(struct raw_ethernet_info));
memset(&rem_dest_info, 0 , sizeof(struct raw_ethernet_info));
/* init default values to user's parameters that's relvant for this test:
* Raw Ethernet Send Latency Test
*/
user_param.verb = SEND;
user_param.tst = LAT;
strncpy(user_param.version, VERSION, sizeof(user_param.version));
user_param.connection_type = RawEth;
user_param.r_flag = &report;
if (check_flow_steering_support()) {
return 1;
}
/* Configure the parameters values according to user
arguments or default values. */
ret_parser = parser(&user_param, argv,argc);
//check for parsing errors
if (ret_parser) {
if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
fprintf(stderr," Parser function exited with Error\n");
DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
return 1;
}
//this is a bidirectional test, so we need to let the init functions
//to think we are in duplex mode
//TODO: ask Ido if that's ok, or should I add another field in user_param
user_param.duplex = 1;
// Find the selected IB device (or default if the user didn't select one).
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
return 1;
}
// Getting the relevant context from the device
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
return 1;
}
// See if MTU and link type are valid and supported.
if (check_link_and_mtu(ctx.context, &user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
return FAILURE;
}
// Allocating arrays needed for the test.
alloc_ctx(&ctx, &user_param);
// Print basic test information.
ctx_print_test_info(&user_param);
//set up the connection, return the required flow rules (notice that user_param->duplex == TRUE)
//so the function will setup like it's a bidirectional test
if (send_set_up_connection(&flow_rules, &ctx, &user_param, &my_dest_info, &rem_dest_info)) {
fprintf(stderr," Unable to set up socket connection\n");
return 1;
}
//print specifications of the test
print_spec(flow_rules,&user_param);
// Create (if necessary) the rdma_cm ids and channel.
if (user_param.work_rdma_cm == ON) {
//create resources
if (create_rdma_resources(&ctx, &user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (user_param.machine == CLIENT) {
//Connects the client to a QP on the other machine with rdma_cm
if (rdma_client_connect(&ctx, &user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else if (rdma_server_connect(&ctx, &user_param)) {
//Assigning a server to listen on rdma_cm port and connect to it.
fprintf(stderr,"Unable to perform rdma_server function\n");
return FAILURE;
}
} else {
// initalize IB resources (data buffer, PD, MR, CQ and events channel)
if (ctx_init(&ctx, &user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
}
//attaching the qp to the spec
#ifdef HAVE_RAW_ETH_EXP
flow_create_result = ibv_exp_create_flow(ctx.qp[0], flow_rules);
#else
flow_create_result = ibv_create_flow(ctx.qp[0], flow_rules);
#endif
if (!flow_create_result){
perror("error");
fprintf(stderr, "Couldn't attach QP\n");
return FAILURE;
}
#ifdef HAVE_RAW_ETH_EXP
if (user_param.use_promiscuous) {
struct ibv_exp_flow_attr attr = {
.type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT,
.num_of_specs = 0,
.port = user_param.ib_port,
.flags = 0
};
if ((flow_promisc = ibv_exp_create_flow(ctx.qp[0], &attr)) == NULL) {
perror("error");
fprintf(stderr, "Couldn't attach promiscous rule QP\n");
}
}
#endif
//build ONE Raw Ethernet packets on ctx buffer
create_raw_eth_pkt(&user_param,&ctx, &my_dest_info , &rem_dest_info);
if (user_param.output == FULL_VERBOSITY) {
printf(RESULT_LINE);
printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
}
// Prepare IB resources for rtr(ready to read)/rts(ready to send)
if (user_param.work_rdma_cm == OFF) {
if (ctx_connect(&ctx, NULL, &user_param, NULL)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
return 1;
}
}
//Post Send send_wqes for current message size
ctx_set_send_wqes(&ctx,&user_param,NULL);
// Post receive recv_wqes for current message size
if (ctx_set_recv_wqes(&ctx,&user_param)) {
fprintf(stderr," Failed to post receive recv_wqes\n");
return 1;
}
//latency test function for SEND verb latency test.
if (run_iter_lat_send(&ctx, &user_param))
{
return 17;
}
//print report (like print_report_bw) in the correct format
// (as set before: FMT_LAT or FMT_LAT_DUR)
user_param.test_type == ITERATIONS ? print_report_lat(&user_param) :
print_report_lat_duration(&user_param);
//destory promisc flow
#ifdef HAVE_RAW_ETH_EXP
if (user_param.use_promiscuous) {
if (ibv_exp_destroy_flow(flow_promisc)) {
perror("error");
fprintf(stderr, "Couldn't Destory promisc flow\n");
return FAILURE;
}
}
#endif
//destroy flow
#ifdef HAVE_RAW_ETH_EXP
if (ibv_exp_destroy_flow(flow_create_result)) {
#else
if (ibv_destroy_flow(flow_create_result)) {
#endif
perror("error");
fprintf(stderr, "Couldn't Destory flow\n");
return FAILURE;
}
free(flow_rules);
//Deallocate all perftest resources.
if (destroy_ctx(&ctx, &user_param)) {
fprintf(stderr,"Failed to destroy_ctx\n");
DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
return 1;
}
if (user_param.output == FULL_VERBOSITY)
printf(RESULT_LINE);
DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
return 0;
}
