int main(int argc, char *argv[]) { int i = 0; struct report_options report = {}; struct pingpong_context ctx; struct ibv_device *ib_dev; struct perftest_parameters user_param; struct pingpong_dest my_dest,rem_dest; struct perftest_comm user_comm; /* init default values to user's parameters */ memset(&ctx,0,sizeof(struct pingpong_context)); memset(&user_param,0,sizeof(struct perftest_parameters)); memset(&user_comm,0,sizeof(struct perftest_comm)); memset(&my_dest,0,sizeof(struct pingpong_dest)); memset(&rem_dest,0,sizeof(struct pingpong_dest)); user_param.verb = READ; user_param.tst = LAT; user_param.r_flag = &report; user_param.version = VERSION; // Configure the parameters values according to user arguments or defalut values. if (parser(&user_param,argv,argc)) { fprintf(stderr," Parser function exited with Error\n"); return FAILURE; } // Finding the IB device selected (or defalut if no selected). ib_dev = ctx_find_dev(user_param.ib_devname); if (!ib_dev) { fprintf(stderr," Unable to find the Infiniband/RoCE deivce\n"); return FAILURE; } // Getting the relevant context from the device ctx.context = ibv_open_device(ib_dev); if (!ctx.context) { fprintf(stderr, " Couldn't get context for the device\n"); return 1; } // See if MTU and link type are valid and supported. if (check_link_and_mtu(ctx.context,&user_param)) { fprintf(stderr, " Couldn't get context for the device\n"); return FAILURE; } // Print basic test information. ctx_print_test_info(&user_param); // copy the rellevant user parameters to the comm struct + creating rdma_cm resources. if (create_comm_struct(&user_comm,&user_param)) { fprintf(stderr," Unable to create RDMA_CM resources\n"); return 1; } // Create (if nessacery) the rdma_cm ids and channel. if (user_param.work_rdma_cm == ON) { if (create_rdma_resources(&ctx,&user_param)) { fprintf(stderr," Unable to create the rdma_resources\n"); return FAILURE; } if (user_param.machine == CLIENT) { if (rdma_client_connect(&ctx,&user_param)) { fprintf(stderr,"Unable to perform rdma_client function\n"); return FAILURE; } } else { if (rdma_server_connect(&ctx,&user_param)) { fprintf(stderr,"Unable to perform rdma_client function\n"); return FAILURE; } } } else { // create all the basic IB resources (data buffer, PD, MR, CQ and events channel) if (ctx_init(&ctx,&user_param)) { fprintf(stderr, " Couldn't create IB resources\n"); return FAILURE; } } // Set up the Connection. if (set_up_connection(&ctx,&user_param,&my_dest)) { fprintf(stderr," Unable to set up socket connection\n"); return 1; } ctx_print_pingpong_data(&my_dest,&user_comm); // Init the connection and print the local data. if (establish_connection(&user_comm)) { fprintf(stderr," Unable to init the socket connection\n"); return 1; } // shaking hands and gather the other side info. if (ctx_hand_shake(&user_comm,&my_dest,&rem_dest)) { fprintf(stderr,"Failed to exchange date between server and clients\n"); return 1; } user_comm.rdma_params->side = REMOTE; ctx_print_pingpong_data(&rem_dest,&user_comm); if (user_param.work_rdma_cm == OFF) { if (pp_connect_ctx(&ctx,my_dest.psn,&rem_dest,my_dest.out_reads,&user_param)) { fprintf(stderr," Unable to Connect the HCA's through the link\n"); return 1; } } // An additional handshake is required after moving qp to RTR. if (ctx_hand_shake(&user_comm,&my_dest,&rem_dest)) { fprintf(stderr,"Failed to exchange date between server and clients\n"); return 1; } ALLOCATE(tstamp,cycles_t,user_param.iters); // Only Client post read request. if (user_param.machine == SERVER) { if (ctx_close_connection(&user_comm,&my_dest,&rem_dest)) { fprintf(stderr,"Failed to close connection between server and client\n"); return 1; } printf(RESULT_LINE); return 0; // destroy_ctx(&ctx,&user_param); } if (user_param.use_event) { if (ibv_req_notify_cq(ctx.send_cq, 0)) { fprintf(stderr, "Couldn't request CQ notification\n"); return 1; } } printf(RESULT_LINE); printf(RESULT_FMT_LAT); if (user_param.all == ON) { for (i = 1; i < 24 ; ++i) { user_param.size = 1 << i; if(run_iter(&ctx,&user_param,&rem_dest)) return 17; print_report(&user_param); } } else { if(run_iter(&ctx,&user_param,&rem_dest)) return 18; print_report(&user_param); } if (ctx_close_connection(&user_comm,&my_dest,&rem_dest)) { fprintf(stderr,"Failed to close connection between server and client\n"); return 1; } printf(RESULT_LINE); return 0; // destroy_ctx(&ctx,&user_param); }
int main(int argc, char *argv[]) { struct ibv_device *ib_dev; struct pingpong_context ctx; struct pingpong_dest *my_dest,*rem_dest; struct perftest_parameters user_param; struct perftest_comm user_comm; int i = 0; memset(&ctx,0,sizeof(struct pingpong_context)); memset(&user_param, 0, sizeof(struct perftest_parameters)); memset(&user_comm,0,sizeof(struct perftest_comm)); user_param.verb = WRITE; user_param.tst = BW; user_param.spec = PL; user_param.version = VERSION; // Configure the parameters values according to user arguments or defalut values. if (parser(&user_param,argv,argc)) { fprintf(stderr," Parser function exited with Error\n"); return 1; } // Finding the IB device selected (or defalut if no selected). ib_dev = ctx_find_dev(user_param.ib_devname); if (!ib_dev) { fprintf(stderr," Unable to find the Infiniband/RoCE deivce\n"); return 1; } // Getting the relevant context from the device ctx.context = ibv_open_device(ib_dev); if (!ctx.context) { fprintf(stderr, " Couldn't get context for the device\n"); return 1; } // See if MTU and link type are valid and supported. if (check_link_and_mtu(ctx.context,&user_param)) { fprintf(stderr, " Couldn't get context for the device\n"); return FAILURE; } // Print basic test information. ctx_print_test_info(&user_param); ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps); memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps); ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps); memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps); // copy the rellevant user parameters to the comm struct + creating rdma_cm resources. if (create_comm_struct(&user_comm,&user_param)) { fprintf(stderr," Unable to create RDMA_CM resources\n"); return 1; } // Create (if nessacery) the rdma_cm ids and channel. if (user_param.work_rdma_cm == ON) { if (create_rdma_resources(&ctx,&user_param)) { fprintf(stderr," Unable to create the rdma_resources\n"); return FAILURE; } if (user_param.machine == CLIENT) { if (rdma_client_connect(&ctx,&user_param)) { fprintf(stderr,"Unable to perform rdma_client function\n"); return FAILURE; } } else { if (rdma_server_connect(&ctx,&user_param)) { fprintf(stderr,"Unable to perform rdma_client function\n"); return FAILURE; } } } else { // create all the basic IB resources (data buffer, PD, MR, CQ and events channel) if (ctx_init(&ctx,&user_param)) { fprintf(stderr, " Couldn't create IB resources\n"); return FAILURE; } } // Set up the Connection. if (set_up_connection(&ctx,&user_param,my_dest)) { fprintf(stderr," Unable to set up socket connection\n"); return FAILURE; } // Print this machine QP information for (i=0; i < user_param.num_of_qps; i++) ctx_print_pingpong_data(&my_dest[i],&user_comm); // Init the connection and print the local data. if (establish_connection(&user_comm)) { fprintf(stderr," Unable to init the socket connection\n"); return FAILURE; } // shaking hands and gather the other side info. for (i=0; i < user_param.num_of_qps; i++) { if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) { fprintf(stderr," Failed to exchange date between server and clients\n"); return 1; } // Print remote machine QP information user_comm.rdma_params->side = REMOTE; ctx_print_pingpong_data(&rem_dest[i],&user_comm); if (user_param.work_rdma_cm == OFF) { if (pp_connect_ctx(&ctx,my_dest[i].psn,&rem_dest[i],&user_param,i)) { fprintf(stderr," Unable to Connect the HCA's through the link\n"); return FAILURE; } } // An additional handshake is required after moving qp to RTR. if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) { fprintf(stderr," Failed to exchange date between server and clients\n"); return FAILURE; } } printf(RESULT_LINE); printf(RESULT_FMT); // For half duplex tests, server just waits for client to exit if (user_param.machine == SERVER && !user_param.duplex) { if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) { fprintf(stderr,"Failed to close connection between server and client\n"); return 1; } printf(RESULT_LINE); return 0; } ALLOCATE(tposted,cycles_t,user_param.iters*user_param.num_of_qps); ALLOCATE(tcompleted,cycles_t,user_param.iters*user_param.num_of_qps); if (user_param.all == ON) { for (i = 1; i < 24 ; ++i) { user_param.size = 1 << i; if(run_iter(&ctx,&user_param,rem_dest)) return 17; print_report(&user_param); } } else { if(run_iter(&ctx,&user_param,rem_dest)) return 18; print_report(&user_param); } free(tposted); free(tcompleted); // Closing connection. if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) { fprintf(stderr,"Failed to close connection between server and client\n"); return 1; } free(my_dest); free(rem_dest); printf(RESULT_LINE); return 0; }
/* * Main function. implements raw_ethernet_send_lat */ int main(int argc, char *argv[]) { struct ibv_device *ib_dev = NULL; struct pingpong_context ctx; struct raw_ethernet_info my_dest_info,rem_dest_info; int ret_parser; struct perftest_parameters user_param; #ifdef HAVE_RAW_ETH_EXP struct ibv_exp_flow *flow_create_result = NULL; struct ibv_exp_flow_attr *flow_rules = NULL; struct ibv_exp_flow *flow_promisc = NULL; #else struct ibv_flow *flow_create_result = NULL; struct ibv_flow_attr *flow_rules = NULL; #endif struct report_options report; //allocate memory space for user parameters memset(&ctx, 0, sizeof(struct pingpong_context)); memset(&user_param, 0, sizeof(struct perftest_parameters)); memset(&my_dest_info, 0 , sizeof(struct raw_ethernet_info)); memset(&rem_dest_info, 0 , sizeof(struct raw_ethernet_info)); /* init default values to user's parameters that's relvant for this test: * Raw Ethernet Send Latency Test */ user_param.verb = SEND; user_param.tst = LAT; strncpy(user_param.version, VERSION, sizeof(user_param.version)); user_param.connection_type = RawEth; user_param.r_flag = &report; if (check_flow_steering_support()) { return 1; } /* Configure the parameters values according to user arguments or default values. */ ret_parser = parser(&user_param, argv,argc); //check for parsing errors if (ret_parser) { if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT) fprintf(stderr," Parser function exited with Error\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } //this is a bidirectional test, so we need to let the init functions //to think we are in duplex mode //TODO: ask Ido if that's ok, or should I add another field in user_param user_param.duplex = 1; // Find the selected IB device (or default if the user didn't select one). ib_dev = ctx_find_dev(user_param.ib_devname); if (!ib_dev) { fprintf(stderr," Unable to find the Infiniband/RoCE device\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } // Getting the relevant context from the device ctx.context = ibv_open_device(ib_dev); if (!ctx.context) { fprintf(stderr, " Couldn't get context for the device\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } // See if MTU and link type are valid and supported. if (check_link_and_mtu(ctx.context, &user_param)) { fprintf(stderr, " Couldn't get context for the device\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return FAILURE; } // Allocating arrays needed for the test. alloc_ctx(&ctx, &user_param); // Print basic test information. ctx_print_test_info(&user_param); //set up the connection, return the required flow rules (notice that user_param->duplex == TRUE) //so the function will setup like it's a bidirectional test if (send_set_up_connection(&flow_rules, &ctx, &user_param, &my_dest_info, &rem_dest_info)) { fprintf(stderr," Unable to set up socket connection\n"); return 1; } //print specifications of the test print_spec(flow_rules,&user_param); // Create (if necessary) the rdma_cm ids and channel. if (user_param.work_rdma_cm == ON) { //create resources if (create_rdma_resources(&ctx, &user_param)) { fprintf(stderr," Unable to create the rdma_resources\n"); return FAILURE; } if (user_param.machine == CLIENT) { //Connects the client to a QP on the other machine with rdma_cm if (rdma_client_connect(&ctx, &user_param)) { fprintf(stderr,"Unable to perform rdma_client function\n"); return FAILURE; } } else if (rdma_server_connect(&ctx, &user_param)) { //Assigning a server to listen on rdma_cm port and connect to it. fprintf(stderr,"Unable to perform rdma_server function\n"); return FAILURE; } } else { // initalize IB resources (data buffer, PD, MR, CQ and events channel) if (ctx_init(&ctx, &user_param)) { fprintf(stderr, " Couldn't create IB resources\n"); return FAILURE; } } //attaching the qp to the spec #ifdef HAVE_RAW_ETH_EXP flow_create_result = ibv_exp_create_flow(ctx.qp[0], flow_rules); #else flow_create_result = ibv_create_flow(ctx.qp[0], flow_rules); #endif if (!flow_create_result){ perror("error"); fprintf(stderr, "Couldn't attach QP\n"); return FAILURE; } #ifdef HAVE_RAW_ETH_EXP if (user_param.use_promiscuous) { struct ibv_exp_flow_attr attr = { .type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT, .num_of_specs = 0, .port = user_param.ib_port, .flags = 0 }; if ((flow_promisc = ibv_exp_create_flow(ctx.qp[0], &attr)) == NULL) { perror("error"); fprintf(stderr, "Couldn't attach promiscous rule QP\n"); } } #endif //build ONE Raw Ethernet packets on ctx buffer create_raw_eth_pkt(&user_param,&ctx, &my_dest_info , &rem_dest_info); if (user_param.output == FULL_VERBOSITY) { printf(RESULT_LINE); printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR); printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT)); } // Prepare IB resources for rtr(ready to read)/rts(ready to send) if (user_param.work_rdma_cm == OFF) { if (ctx_connect(&ctx, NULL, &user_param, NULL)) { fprintf(stderr," Unable to Connect the HCA's through the link\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } } //Post Send send_wqes for current message size ctx_set_send_wqes(&ctx,&user_param,NULL); // Post receive recv_wqes for current message size if (ctx_set_recv_wqes(&ctx,&user_param)) { fprintf(stderr," Failed to post receive recv_wqes\n"); return 1; } //latency test function for SEND verb latency test. if (run_iter_lat_send(&ctx, &user_param)) { return 17; } //print report (like print_report_bw) in the correct format // (as set before: FMT_LAT or FMT_LAT_DUR) user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); //destory promisc flow #ifdef HAVE_RAW_ETH_EXP if (user_param.use_promiscuous) { if (ibv_exp_destroy_flow(flow_promisc)) { perror("error"); fprintf(stderr, "Couldn't Destory promisc flow\n"); return FAILURE; } } #endif //destroy flow #ifdef HAVE_RAW_ETH_EXP if (ibv_exp_destroy_flow(flow_create_result)) { #else if (ibv_destroy_flow(flow_create_result)) { #endif perror("error"); fprintf(stderr, "Couldn't Destory flow\n"); return FAILURE; } free(flow_rules); //Deallocate all perftest resources. if (destroy_ctx(&ctx, &user_param)) { fprintf(stderr,"Failed to destroy_ctx\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } if (user_param.output == FULL_VERBOSITY) printf(RESULT_LINE); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 0; }