int main(int argc, char *argv[]) {

	struct ibv_device	       *ib_dev;
	struct pingpong_context    ctx;
	struct pingpong_dest       *my_dest,*rem_dest;
	struct perftest_parameters user_param;
	struct perftest_comm	   user_comm;
	int                        i = 0;

	memset(&ctx,0,sizeof(struct pingpong_context));
	memset(&user_param, 0, sizeof(struct perftest_parameters));
	memset(&user_comm,0,sizeof(struct perftest_comm));

	user_param.verb    = WRITE;
	user_param.tst     = BW;
	user_param.spec    = PL;
	user_param.version = VERSION;

	// Configure the parameters values according to user arguments or defalut values.
	if (parser(&user_param,argv,argc)) {
		fprintf(stderr," Parser function exited with Error\n");
		return 1;
	}

	// Finding the IB device selected (or defalut if no selected).
	ib_dev = ctx_find_dev(user_param.ib_devname);
	if (!ib_dev) {
		fprintf(stderr," Unable to find the Infiniband/RoCE deivce\n");
		return 1;
	}

	// Getting the relevant context from the device
	ctx.context = ibv_open_device(ib_dev);
	if (!ctx.context) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return 1;
	}

	// See if MTU and link type are valid and supported.
	if (check_link_and_mtu(ctx.context,&user_param)) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return FAILURE;
	}

	// Print basic test information.
	ctx_print_test_info(&user_param);

	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);

	// copy the rellevant user parameters to the comm struct + creating rdma_cm resources.
	if (create_comm_struct(&user_comm,&user_param)) { 
		fprintf(stderr," Unable to create RDMA_CM resources\n");
		return 1;
	}

	// Create (if nessacery) the rdma_cm ids and channel.
	if (user_param.work_rdma_cm == ON) {

	    if (create_rdma_resources(&ctx,&user_param)) {
			fprintf(stderr," Unable to create the rdma_resources\n");
			return FAILURE;
	    }
		
  	    if (user_param.machine == CLIENT) {

			if (rdma_client_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}
		
		} else {

			if (rdma_server_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}
		}
					
	} else {
    
	    // create all the basic IB resources (data buffer, PD, MR, CQ and events channel)
	    if (ctx_init(&ctx,&user_param)) {
			fprintf(stderr, " Couldn't create IB resources\n");
			return FAILURE;
	    }
	}

	// Set up the Connection.
	if (set_up_connection(&ctx,&user_param,my_dest)) {
		fprintf(stderr," Unable to set up socket connection\n");
		return FAILURE;
	}

	// Print this machine QP information
	for (i=0; i < user_param.num_of_qps; i++) 
		ctx_print_pingpong_data(&my_dest[i],&user_comm);

	// Init the connection and print the local data.
	if (establish_connection(&user_comm)) {
		fprintf(stderr," Unable to init the socket connection\n");
		return FAILURE;
	}

	// shaking hands and gather the other side info.
	for (i=0; i < user_param.num_of_qps; i++) {

			if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
				fprintf(stderr," Failed to exchange date between server and clients\n");
				return 1;   
			}

			// Print remote machine QP information
			user_comm.rdma_params->side = REMOTE;
			ctx_print_pingpong_data(&rem_dest[i],&user_comm);

			if (user_param.work_rdma_cm == OFF) {

				if (pp_connect_ctx(&ctx,my_dest[i].psn,&rem_dest[i],&user_param,i)) {
					fprintf(stderr," Unable to Connect the HCA's through the link\n");
					return FAILURE;
				}
			}

			// An additional handshake is required after moving qp to RTR.
			if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
				fprintf(stderr," Failed to exchange date between server and clients\n");
				return FAILURE; 
			}
	}	

	printf(RESULT_LINE);
	printf(RESULT_FMT);

	// For half duplex tests, server just waits for client to exit 
	if (user_param.machine == SERVER && !user_param.duplex) {
		
		if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
			fprintf(stderr,"Failed to close connection between server and client\n");
			return 1;
		}
		printf(RESULT_LINE);
		return 0;
	}

	ALLOCATE(tposted,cycles_t,user_param.iters*user_param.num_of_qps);
	ALLOCATE(tcompleted,cycles_t,user_param.iters*user_param.num_of_qps);

	if (user_param.all == ON) {

		for (i = 1; i < 24 ; ++i) {
			user_param.size = 1 << i;
			if(run_iter(&ctx,&user_param,rem_dest))
				return 17;
			print_report(&user_param);
		}

	} else {

		if(run_iter(&ctx,&user_param,rem_dest))
			return 18;
		print_report(&user_param);
	}

	free(tposted);
	free(tcompleted);

	// Closing connection.
	if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
	 	fprintf(stderr,"Failed to close connection between server and client\n");
		return 1;
	}

	free(my_dest);
	free(rem_dest);
	printf(RESULT_LINE);
	return 0;
}
/*---------------------------------------------------------------------------*/
int run_client_test(struct perf_parameters *user_param)
{
	struct session_data	sess_data;
	struct perf_comm	*comm;
	struct thread_data	*tdata;
	char			url[256];
	int			i = 0;
	int			max_cpus;
	pthread_t		statistics_thread_id;
	struct perf_command	command;
	int			size_log2;
	int			max_size_log2 = 24;


	/* client session attributes */
	struct xio_session_attr attr = {
		&ses_ops,
		NULL,
		0
	};
	xio_init();

	g_mhz		= get_cpu_mhz(0);
	max_cpus	= sysconf(_SC_NPROCESSORS_ONLN);
	threads_iter	= 1;
	size_log2	= 0;

	tdata = calloc(user_param->threads_num, sizeof(*tdata));
	if (tdata == NULL) {
		fprintf(fd, "malloc failed\n");
		return -1;
	}

	comm = create_comm_struct(user_param);
	if (establish_connection(comm)) {
		fprintf(stderr, "failed to establish connection\n");
		free(tdata);
		destroy_comm_struct(comm);
		return -1;
	}

	if (user_param->output_file) {
		fd = fopen(user_param->output_file, "w");
		if (fd == NULL) {
			fprintf(fd, "file open failed. %s\n",
				user_param->output_file);
			free(sess_data.tdata);
			destroy_comm_struct(comm);
			return -1;
		}
		fprintf(fd, "size, threads, tps, bw[Mbps], lat[usec]\n");
		fflush(fd);
	}


	printf("%s", RESULT_FMT);
	printf("%s", RESULT_LINE);


	while (threads_iter <= user_param->threads_num)  {
		data_len	= (uint64_t)1 << size_log2;

		memset(&sess_data, 0, sizeof(sess_data));
		memset(tdata, 0, user_param->threads_num*sizeof(*tdata));
		sess_data.tdata = tdata;

		command.test_param.machine_type	= user_param->machine_type;
		command.test_param.test_type	= user_param->test_type;
		command.test_param.verb		= user_param->verb;
		command.test_param.data_len	= data_len;
		command.command			= GetTestParams;

		ctx_write_data(comm, &command, sizeof(command));

		sprintf(url, "rdma://%s:%d", user_param->server_addr,
			user_param->server_port);
		sess_data.session = xio_session_create(XIO_SESSION_CLIENT,
				&attr, url, 0, 0, &sess_data);
		if (sess_data.session == NULL) {
			int error = xio_errno();
			fprintf(stderr,
				"session creation failed. reason %d - (%s)\n",
				error, xio_strerror(error));
			goto cleanup;
		}

		pthread_create(&statistics_thread_id, NULL,
			       statistics_thread_cb, &sess_data);

		/* spawn threads to handle connection */
		for (i = 0; i < threads_iter; i++) {
			sess_data.tdata[i].affinity		=
				((user_param->cpu + i) % max_cpus);
			sess_data.tdata[i].cid			= i;
			sess_data.tdata[i].sdata		= &sess_data;
			sess_data.tdata[i].user_param		= user_param;
			sess_data.tdata[i].data_len		= data_len;

			/* all threads are working on the same session */
			sess_data.tdata[i].session	= sess_data.session;
			pthread_create(&sess_data.tdata[i].thread_id, NULL,
				       worker_thread, &sess_data.tdata[i]);
		}

		pthread_join(statistics_thread_id, NULL);

		/* join the threads */
		for (i = 0; i < threads_iter; i++)
			pthread_join(sess_data.tdata[i].thread_id, NULL);

		/* close the session */
		xio_session_destroy(sess_data.session);

		if (sess_data.abort) {
			fprintf(stderr, "program aborted\n");
			goto cleanup;
		}

		/* send result to server */
		command.results.bytes		= data_len;
		command.results.threads		= threads_iter;
		command.results.tps		= sess_data.tps;
		command.results.avg_bw		= sess_data.avg_bw;
		command.results.avg_lat		= sess_data.avg_lat_us;
		command.results.min_lat		= sess_data.min_lat_us;
		command.results.max_lat		= sess_data.max_lat_us;
		command.command			= GetTestResults;

		/* sync point */
		ctx_write_data(comm, &command, sizeof(command));

		printf(REPORT_FMT,
		       data_len,
		       threads_iter,
		       sess_data.tps,
		       sess_data.avg_bw,
		       sess_data.avg_lat_us,
		       sess_data.min_lat_us,
		       sess_data.max_lat_us);
		if (fd)
			fprintf(fd, "%lu, %d, %lu, %.2lf, %.2lf\n",
				data_len,
				threads_iter,
				sess_data.tps,
				sess_data.avg_bw,
				sess_data.avg_lat_us);
		fflush(fd);

		/* sync point */
		ctx_read_data(comm, NULL, 0, NULL);

		if (++size_log2 < max_size_log2)
			continue;

		threads_iter++;
		size_log2 = 0;
	}

	printf("%s", RESULT_LINE);

cleanup:
	if (fd)
		fclose(fd);

	ctx_hand_shake(comm);

	ctx_close_connection(comm);

	destroy_comm_struct(comm);

	free(tdata);

	xio_shutdown();

	return 0;
}
Exemple #3
0
int main(int argc, char *argv[]) {

	int                        i = 0;
	int                        size_max_pow = 24;
	int						   ret_val;
	struct report_options      report;
	struct pingpong_context    ctx;
	struct pingpong_dest	   *my_dest  = NULL;
	struct pingpong_dest	   *rem_dest = NULL;
	struct mcast_parameters	   mcg_params;
	struct ibv_device          *ib_dev = NULL;
	struct perftest_parameters user_param;
	struct perftest_comm	   user_comm;

	/* init default values to user's parameters */
	memset(&ctx,		0, sizeof(struct pingpong_context));
	memset(&user_param, 0, sizeof(struct perftest_parameters));
	memset(&user_comm , 0, sizeof(struct perftest_comm));
	memset(&mcg_params, 0, sizeof(struct mcast_parameters));

	user_param.verb    = SEND;
	user_param.tst     = LAT;
	strncpy(user_param.version, VERSION, sizeof(user_param.version));
	user_param.r_flag  = &report;

	// Configure the parameters values according to user arguments or defalut values.
	ret_val = parser(&user_param,argv,argc);
	if (ret_val) {
		if (ret_val != VERSION_EXIT && ret_val != HELP_EXIT)
			fprintf(stderr," Parser function exited with Error\n");
		return 1;
	}

	if(user_param.use_xrc || user_param.connection_type == DC) {
		user_param.num_of_qps *= 2;
	}

	//Checking that the user did not run with RawEth. for this we have raw_etherent_bw test.
	if (user_param.connection_type == RawEth) {
        fprintf(stderr," This test cannot run Raw Ethernet QPs (you have chosen RawEth as connection type\n");
        return FAILURE;
    }

	// Finding the IB device selected (or defalut if no selected).
	ib_dev = ctx_find_dev(user_param.ib_devname);
	if (!ib_dev) {
		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
		return 1;
	}

	if (user_param.use_mcg)
		GET_STRING(mcg_params.ib_devname,ibv_get_device_name(ib_dev));

	// Getting the relevant context from the device
	ctx.context = ibv_open_device(ib_dev);
	if (!ctx.context) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return 1;
	}

	// See if MTU and link type are valid and supported.
	if (check_link(ctx.context,&user_param)) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return FAILURE;
	}

	// copy the relevant user parameters to the comm struct + creating rdma_cm resources.
	if (create_comm_struct(&user_comm,&user_param)) {
		fprintf(stderr," Unable to create RDMA_CM resources\n");
		return 1;
	}

	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
		printf("\n************************************\n");
		printf("* Waiting for client to connect... *\n");
		printf("************************************\n");
	}

	// Initialize the connection and print the local data.
	if (establish_connection(&user_comm)) {
		fprintf(stderr," Unable to init the socket connection\n");
		return FAILURE;
	}

	exchange_versions(&user_comm, &user_param);

	check_sys_data(&user_comm, &user_param);

	// See if MTU and link type are valid and supported.
	if (check_mtu(ctx.context,&user_param, &user_comm)) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return FAILURE;
	}

	// Print basic test information.
	ctx_print_test_info(&user_param);

	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);

	// Allocating arrays needed for the test.
	alloc_ctx(&ctx,&user_param);

	// Create (if nessacery) the rdma_cm ids and channel.
	if (user_param.work_rdma_cm == ON) {

	    if (user_param.machine == CLIENT) {
			if (retry_rdma_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}

		} else {
    		if (create_rdma_resources(&ctx,&user_param)) {
				fprintf(stderr," Unable to create the rdma_resources\n");
				return FAILURE;
    		}
			if (rdma_server_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}
		}

	} else {

		 // create all the basic IB resources (data buffer, PD, MR, CQ and events channel)
	    if (ctx_init(&ctx,&user_param)) {
			fprintf(stderr, " Couldn't create IB resources\n");
			return FAILURE;
	    }
	}

	// Set up the Connection.
	if (send_set_up_connection(&ctx,&user_param,my_dest,&mcg_params,&user_comm)) {
		fprintf(stderr," Unable to set up socket connection\n");
		return 1;
	}

	for (i=0; i < user_param.num_of_qps; i++)
		ctx_print_pingpong_data(&my_dest[i],&user_comm);

	user_comm.rdma_params->side = REMOTE;
	for (i=0; i < user_param.num_of_qps; i++) {

		// shaking hands and gather the other side info.
		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
			fprintf(stderr,"Failed to exchange data between server and clients\n");
			return 1;
		}

		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
	}

        if (user_param.work_rdma_cm == OFF)
        {
                if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0]))
                {
                        fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
                        fprintf(stderr," Please Try to use a different IP version.\n\n");
                        return 1;
                }
        }

	if (user_param.use_mcg) {

		memcpy(mcg_params.base_mgid.raw,mcg_params.mgid.raw,16);
		memcpy(mcg_params.mgid.raw,rem_dest[0].gid.raw,16);
		mcg_params.base_mlid = mcg_params.mlid;
		mcg_params.is_2nd_mgid_used = ON;
		if (!strcmp(link_layer_str(user_param.link_type),"IB")) {
			// Request for Mcast group create registery in SM.
			if (join_multicast_group(SUBN_ADM_METHOD_SET,&mcg_params)) {
				fprintf(stderr," Failed to Join Mcast request\n");
				return 1;
			}
		}

		/*
		 * The next stall in code (50 ms sleep) is a work around for fixing the
		 * the bug this test had in Multicast for the past 1 year.
		 * It appears, that when a switch involved, it takes ~ 10 ms for the join
		 * request to propogate on the IB fabric, thus we need to wait for it.
		 * what happened before this fix was  reaching the post_send
		 * code segment in about 350 ns from here, and the switch(es) dropped
		 * the packet because join request wasn't finished.
		 */
		usleep(50000);
	}

	if (user_param.work_rdma_cm == OFF) {

		// Prepare IB resources for rtr/rts.
		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
			fprintf(stderr," Unable to Connect the HCA's through the link\n");
			return 1;
		}
	}

	// shaking hands and gather the other side info.
	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
		fprintf(stderr,"Failed to exchange data between server and clients\n");
		return 1;
	}

    if (user_param.use_event) {

		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
			fprintf(stderr, "Couldn't request RCQ notification\n");
			return 1;
		}

		if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
			fprintf(stderr, "Couldn't request RCQ notification\n");
			return 1;
		}
    }
	if (user_param.output == FULL_VERBOSITY) {
		printf(RESULT_LINE);
		printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
	}

	ctx_set_send_wqes(&ctx,&user_param,rem_dest);

	if (user_param.test_method == RUN_ALL) {

		if (user_param.connection_type == UD)
			size_max_pow =  (int)UD_MSG_2_EXP(MTU_SIZE(user_param.curr_mtu)) + 1;

		for (i = 1; i < size_max_pow ; ++i) {

			user_param.size = (uint64_t)1 << i;

			// Post recevie recv_wqes fo current message size
			if (ctx_set_recv_wqes(&ctx,&user_param)) {
				fprintf(stderr," Failed to post receive recv_wqes\n");
				return 1;
			}

			// Sync between the client and server so the client won't send packets
			// Before the server has posted his receive wqes (in UC/UD it will result in a deadlock).
			if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
				fprintf(stderr,"Failed to exchange data between server and clients\n");
				return 1;
			}

			if(run_iter_lat_send(&ctx, &user_param))
				return 17;

			user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
		}

	} else {

		// Post recevie recv_wqes fo current message size
		if (ctx_set_recv_wqes(&ctx,&user_param)) {
			fprintf(stderr," Failed to post receive recv_wqes\n");
			return 1;
		}

		// Sync between the client and server so the client won't send packets
		// Before the server has posted his receive wqes (in UC/UD it will result in a deadlock).
		if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
			fprintf(stderr,"Failed to exchange data between server and clients\n");
			return 1;
		}

		if(run_iter_lat_send(&ctx, &user_param))
			return 17;

		user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
	}

	if (user_param.output == FULL_VERBOSITY) {
		printf(RESULT_LINE);
	}

	if (ctx_close_connection(&user_comm,my_dest,rem_dest)) {
		fprintf(stderr,"Failed to close connection between server and client\n");
		fprintf(stderr," Trying to close this side resources\n");
	}

	return send_destroy_ctx(&ctx,&user_param,&mcg_params);
}
int main(int argc, char *argv[]) {

	int                         i = 0;
	struct report_options       report = {};
	struct pingpong_context     ctx;
	struct ibv_device           *ib_dev;
	struct perftest_parameters  user_param;
	struct pingpong_dest	    my_dest,rem_dest;
	struct perftest_comm		user_comm;
	
	/* init default values to user's parameters */
	memset(&ctx,0,sizeof(struct pingpong_context));
	memset(&user_param,0,sizeof(struct perftest_parameters));
	memset(&user_comm,0,sizeof(struct perftest_comm));
	memset(&my_dest,0,sizeof(struct pingpong_dest));
	memset(&rem_dest,0,sizeof(struct pingpong_dest));

	user_param.verb    = READ;
	user_param.tst     = LAT;
	user_param.r_flag  = &report;
	user_param.version = VERSION;

	// Configure the parameters values according to user arguments or defalut values.
	if (parser(&user_param,argv,argc)) {
		fprintf(stderr," Parser function exited with Error\n");
		return FAILURE;
	}

	// Finding the IB device selected (or defalut if no selected).
	ib_dev = ctx_find_dev(user_param.ib_devname);
	if (!ib_dev) {
		fprintf(stderr," Unable to find the Infiniband/RoCE deivce\n");
		return FAILURE;
	}

	// Getting the relevant context from the device
	ctx.context = ibv_open_device(ib_dev);
	if (!ctx.context) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return 1;
	}

	// See if MTU and link type are valid and supported.
	if (check_link_and_mtu(ctx.context,&user_param)) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return FAILURE;
	}

	// Print basic test information.
	ctx_print_test_info(&user_param);

	// copy the rellevant user parameters to the comm struct + creating rdma_cm resources.
	if (create_comm_struct(&user_comm,&user_param)) { 
		fprintf(stderr," Unable to create RDMA_CM resources\n");
		return 1;
	}

	// Create (if nessacery) the rdma_cm ids and channel.
	if (user_param.work_rdma_cm == ON) {

		if (create_rdma_resources(&ctx,&user_param)) {
			fprintf(stderr," Unable to create the rdma_resources\n");
			return FAILURE;
		}

		if (user_param.machine == CLIENT) {

			if (rdma_client_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}
		
		} else {

			if (rdma_server_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}
		}

	} else {

		// create all the basic IB resources (data buffer, PD, MR, CQ and events channel)
	    if (ctx_init(&ctx,&user_param)) {
			fprintf(stderr, " Couldn't create IB resources\n");
			return FAILURE;
	    }
	}

	// Set up the Connection.
	if (set_up_connection(&ctx,&user_param,&my_dest)) {
		fprintf(stderr," Unable to set up socket connection\n");
		return 1;
	} 

	ctx_print_pingpong_data(&my_dest,&user_comm);

	// Init the connection and print the local data.
	if (establish_connection(&user_comm)) {
		fprintf(stderr," Unable to init the socket connection\n");
		return 1;
	}	

	//  shaking hands and gather the other side info.
	if (ctx_hand_shake(&user_comm,&my_dest,&rem_dest)) {
		fprintf(stderr,"Failed to exchange date between server and clients\n");
		return 1;
	}

	user_comm.rdma_params->side = REMOTE;
	ctx_print_pingpong_data(&rem_dest,&user_comm);

	if (user_param.work_rdma_cm == OFF) {

		if (pp_connect_ctx(&ctx,my_dest.psn,&rem_dest,my_dest.out_reads,&user_param)) {
			fprintf(stderr," Unable to Connect the HCA's through the link\n");
			return 1;
		}
	}

	// An additional handshake is required after moving qp to RTR.
	if (ctx_hand_shake(&user_comm,&my_dest,&rem_dest)) {
       fprintf(stderr,"Failed to exchange date between server and clients\n");
       return 1;
    }

	ALLOCATE(tstamp,cycles_t,user_param.iters);

	// Only Client post read request. 
	if (user_param.machine == SERVER) {

		if (ctx_close_connection(&user_comm,&my_dest,&rem_dest)) {
		 	fprintf(stderr,"Failed to close connection between server and client\n");
		 	return 1;
		}
		printf(RESULT_LINE);
		return 0; // destroy_ctx(&ctx,&user_param);

	} 

	if (user_param.use_event) {
		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
			fprintf(stderr, "Couldn't request CQ notification\n");
			return 1;
		} 
	}

	printf(RESULT_LINE);
	printf(RESULT_FMT_LAT);

	if (user_param.all == ON) {
		for (i = 1; i < 24 ; ++i) {
			user_param.size = 1 << i;
			if(run_iter(&ctx,&user_param,&rem_dest))
				return 17;
	    	
			print_report(&user_param);
		}
	} else {
		if(run_iter(&ctx,&user_param,&rem_dest))
			return 18;
		
		print_report(&user_param);
	}

	if (ctx_close_connection(&user_comm,&my_dest,&rem_dest)) {
	 	fprintf(stderr,"Failed to close connection between server and client\n");
	 	return 1;
	}

	printf(RESULT_LINE);

	return 0; // destroy_ctx(&ctx,&user_param);
}
Exemple #5
0
int main(int argc, char *argv[])
{
	struct ibv_device		*ib_dev = NULL;
	struct pingpong_context  	ctx;
	struct pingpong_dest	 	*my_dest  = NULL;
	struct pingpong_dest		*rem_dest = NULL;
	struct perftest_parameters  	user_param;
	struct perftest_comm		user_comm;
	struct mcast_parameters     	mcg_params;
	struct bw_report_data		my_bw_rep, rem_bw_rep;
	int                      	ret_parser,i = 0;
	int                      	size_max_pow = 24;

	/* init default values to user's parameters */
	memset(&ctx, 0,sizeof(struct pingpong_context));
	memset(&user_param, 0 , sizeof(struct perftest_parameters));
	memset(&mcg_params, 0 , sizeof(struct mcast_parameters));
	memset(&user_comm, 0,sizeof(struct perftest_comm));

	user_param.verb    = SEND;
	user_param.tst     = BW;
	strncpy(user_param.version, VERSION, sizeof(user_param.version));

	/* Configure the parameters values according to user arguments or defalut values. */
	ret_parser = parser(&user_param,argv,argc);
	if (ret_parser) {
		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
			fprintf(stderr," Parser function exited with Error\n");
		return 1;
	}
	if((user_param.connection_type == DC || user_param.use_xrc) && user_param.duplex) {
		user_param.num_of_qps *= 2;
	}
	/* Checking that the user did not run with RawEth. for this we have raw_etherent_bw test. */
	if (user_param.connection_type == RawEth) {
		fprintf(stderr," This test cannot run Raw Ethernet QPs (you have chosen RawEth as connection type\n");
		fprintf(stderr," For this we have raw_ethernet_bw test in this package.\n");
		return FAILURE;
	}

	/* Finding the IB device selected (or defalut if no selected). */
	ib_dev = ctx_find_dev(user_param.ib_devname);
	if (!ib_dev) {
		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
		return 1;
	}

	if (user_param.use_mcg)
		GET_STRING(mcg_params.ib_devname,ibv_get_device_name(ib_dev));

	/* Getting the relevant context from the device */
	ctx.context = ibv_open_device(ib_dev);
	if (!ctx.context) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return 1;
	}


	/* See if MTU and link type are valid and supported. */
	if (check_link(ctx.context,&user_param)) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return FAILURE;
	}

	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
	if (create_comm_struct(&user_comm,&user_param)) {
		fprintf(stderr," Unable to create RDMA_CM resources\n");
		return 1;
	}

	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
		printf("\n************************************\n");
		printf("* Waiting for client to connect... *\n");
		printf("************************************\n");
	}

	/* Initialize the connection and print the local data. */
	if (establish_connection(&user_comm)) {
		fprintf(stderr," Unable to init the socket connection\n");
		return FAILURE;
	}

	exchange_versions(&user_comm, &user_param);

	check_sys_data(&user_comm, &user_param);

	/* See if MTU and link type are valid and supported. */
	if (check_mtu(ctx.context,&user_param, &user_comm)) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return FAILURE;
	}

	/* Print basic test information. */
	ctx_print_test_info(&user_param);

	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);

	if (user_param.transport_type == IBV_TRANSPORT_IWARP)
		ctx.send_rcredit = 1;

	/* Allocating arrays needed for the test. */
	alloc_ctx(&ctx,&user_param);

	/* Create (if nessacery) the rdma_cm ids and channel. */
	if (user_param.work_rdma_cm == ON) {

		if (user_param.machine == CLIENT) {
			if (retry_rdma_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}

		} else {
			if (create_rdma_resources(&ctx,&user_param)) {
				fprintf(stderr," Unable to create the rdma_resources\n");
				return FAILURE;
			}
			if (rdma_server_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}
		}

	} else {

		/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
		if (ctx_init(&ctx,&user_param)) {
			fprintf(stderr, " Couldn't create IB resources\n");
			return FAILURE;
		}
	}

	/* Set up the Connection. */
	if (send_set_up_connection(&ctx,&user_param,my_dest,&mcg_params,&user_comm)) {
		fprintf(stderr," Unable to set up socket connection\n");
		return 1;
	}

	if (ctx.send_rcredit)
		ctx_alloc_credit(&ctx,&user_param,my_dest);

	for (i=0; i < user_param.num_of_qps; i++)
		ctx_print_pingpong_data(&my_dest[i],&user_comm);

	user_comm.rdma_params->side = REMOTE;
	for (i=0; i < user_param.num_of_qps; i++) {

		/* shaking hands and gather the other side info. */
		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
			fprintf(stderr,"Failed to exchange data between server and clients\n");
			return 1;
		}

		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
	}

	if (user_param.work_rdma_cm == OFF) {
		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
			fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
			fprintf(stderr," Please Try to use a different IP version.\n\n");
			return 1;
		}
	}

	/* If credit for available recieve buffers is necessary,
	 * the credit sending is done via RDMA WRITE ops and the ctx_hand_shake above
	 * is used to exchange the rkeys and buf addresses for the RDMA WRITEs
	 */
	if (ctx.send_rcredit)
		ctx_set_credit_wqes(&ctx,&user_param,rem_dest);

	/* Joining the Send side port the Mcast gid */
	if (user_param.use_mcg && (user_param.machine == CLIENT || user_param.duplex)) {

		memcpy(mcg_params.mgid.raw, rem_dest[0].gid.raw, 16);
		if (set_mcast_group(&ctx,&user_param,&mcg_params)) {
			fprintf(stderr," Unable to Join Sender to Mcast gid\n");
			return 1;
		}
		/*
		 * The next stall in code (50 ms sleep) is a work around for fixing the
		 * the bug this test had in Multicast for the past 1 year.
		 * It appears, that when a switch involved, it takes ~ 10 ms for the join
		 * request to propogate on the IB fabric, thus we need to wait for it.
		 * what happened before this fix was client reaching the post_send
		 * code segment in about 350 ns from here, and the switch(es) dropped
		 * the packet because join request wasn't finished.
		 */
		usleep(50000);
	}

	if (user_param.work_rdma_cm == OFF) {

		/* Prepare IB resources for rtr/rts. */
		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
			fprintf(stderr," Unable to Connect the HCA's through the link\n");
			return 1;
		}
	}

	/* shaking hands and gather the other side info. */
	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
		fprintf(stderr,"Failed to exchange data between server and clients\n");
		return 1;
	}

	if (user_param.use_event) {

		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
			fprintf(stderr, " Couldn't request CQ notification\n");
			return 1;
		}

		if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
			fprintf(stderr, " Couldn't request CQ notification\n");
			return 1;
		}
	}

	if (user_param.output == FULL_VERBOSITY) {
		if (user_param.report_per_port) {
			printf(RESULT_LINE_PER_PORT);
			printf((user_param.report_fmt == MBS ? RESULT_FMT_PER_PORT : RESULT_FMT_G_PER_PORT));
		}
		else {
			printf(RESULT_LINE);
			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
		}
		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
	}

	if (user_param.test_method == RUN_ALL) {

		if (user_param.connection_type == UD)
			size_max_pow =  (int)UD_MSG_2_EXP(MTU_SIZE(user_param.curr_mtu)) + 1;

		for (i = 1; i < size_max_pow ; ++i) {

			user_param.size = (uint64_t)1 << i;

			if (user_param.machine == CLIENT || user_param.duplex)
				ctx_set_send_wqes(&ctx,&user_param,rem_dest);

			if (user_param.machine == SERVER || user_param.duplex) {
				if (ctx_set_recv_wqes(&ctx,&user_param)) {
					fprintf(stderr," Failed to post receive recv_wqes\n");
					return 1;
				}
			}

			if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
				fprintf(stderr,"Failed to exchange data between server and clients\n");
				return 1;
			}

			if (ctx.send_rcredit) {
				int j;
				for (j = 0; j < user_param.num_of_qps; j++)
					ctx.credit_buf[j] = 0;
			}

			if (user_param.duplex) {
				if(run_iter_bi(&ctx,&user_param))
					return 17;

			} else if (user_param.machine == CLIENT) {

				if(run_iter_bw(&ctx,&user_param)) {
					return 17;
				}

			} else	{

				if(run_iter_bw_server(&ctx,&user_param)) {
					return 17;
				}
			}

			print_report_bw(&user_param,&my_bw_rep);

			if (user_param.duplex && user_param.test_type != DURATION) {
				xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
				print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
			}
			if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
				fprintf(stderr,"Failed to exchange data between server and clients\n");
				return 1;
			}

			/* Check if last iteration ended well in UC/UD */
			if (user_param.check_alive_exited) {
				break;
			}
		}

	} else if (user_param.test_method == RUN_REGULAR) {

		if (user_param.machine == CLIENT || user_param.duplex)
			ctx_set_send_wqes(&ctx,&user_param,rem_dest);

		if (user_param.machine == SERVER || user_param.duplex) {
			if (ctx_set_recv_wqes(&ctx,&user_param)) {
				fprintf(stderr," Failed to post receive recv_wqes\n");
				return 1;
			}
		}

		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
			fprintf(stderr,"Failed to exchange data between server and clients\n");
			return 1;
		}

		if (user_param.duplex) {

			if(run_iter_bi(&ctx,&user_param))
				return 17;

		} else if (user_param.machine == CLIENT) {

			if(run_iter_bw(&ctx,&user_param)) {
				return 17;
			}

		} else if(run_iter_bw_server(&ctx,&user_param)) {

			return 17;
		}

		print_report_bw(&user_param,&my_bw_rep);

		if (user_param.duplex && user_param.test_type != DURATION) {
			xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
			print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
		}

		if (user_param.report_both && user_param.duplex) {
			printf(RESULT_LINE);
			printf("\n Local results: \n");
			printf(RESULT_LINE);
			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
			print_full_bw_report(&user_param, &my_bw_rep, NULL);
			printf(RESULT_LINE);

			printf("\n Remote results: \n");
			printf(RESULT_LINE);
			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
			print_full_bw_report(&user_param, &rem_bw_rep, NULL);
		}
	} else if (user_param.test_method == RUN_INFINITELY) {

		if (user_param.machine == CLIENT)
			ctx_set_send_wqes(&ctx,&user_param,rem_dest);

		else if (user_param.machine == SERVER) {

			if (ctx_set_recv_wqes(&ctx,&user_param)) {
				fprintf(stderr," Failed to post receive recv_wqes\n");
				return 1;
			}
		}

		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
			fprintf(stderr,"Failed to exchange data between server and clients\n");
			return 1;
		}

		if (user_param.machine == CLIENT) {

			if(run_iter_bw_infinitely(&ctx,&user_param)) {
				fprintf(stderr," Error occured while running infinitely! aborting ...\n");
				return 1;
			}

		} else if (user_param.machine == SERVER) {

			if(run_iter_bw_infinitely_server(&ctx,&user_param)) {
				fprintf(stderr," Error occured while running infinitely on server! aborting ...\n");
				return 1;
			}
		}
	}

	if (user_param.output == FULL_VERBOSITY) {
		if (user_param.report_per_port)
			printf(RESULT_LINE_PER_PORT);
		else
			printf(RESULT_LINE);
	}

	if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
		fprintf(stderr," Failed to close connection between server and client\n");
		fprintf(stderr," Trying to close this side resources\n");
	}

	/* Destory all test resources, including Mcast if exists */
	if (send_destroy_ctx(&ctx,&user_param,&mcg_params)) {
		fprintf(stderr,"Couldn't Destory all SEND resources\n");
		return FAILURE;
	}
	if (user_param.work_rdma_cm == ON) {
		user_comm.rdma_params->work_rdma_cm = ON;
		if (destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params)) {
			fprintf(stderr,"Failed to destroy resources\n");
			return 1;
		}
	}

	if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
		fprintf(stderr,"Error: BW result is below bw limit\n");
		return 1;
	}

	if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) {
		fprintf(stderr,"Error: Msg rate  is below msg_rate limit\n");
		return 1;
	}

	return 0;
}
Exemple #6
0
int main(int argc, char *argv[])
{
	int                        ret_parser,i = 0;
	struct ibv_device	   *ib_dev = NULL;
	struct pingpong_context    ctx;
	struct pingpong_dest       *my_dest = NULL;
	struct pingpong_dest       *rem_dest = NULL;
	struct perftest_parameters user_param;
	struct perftest_comm	   user_comm;
	struct bw_report_data      my_bw_rep, rem_bw_rep;

	/* init default values to user's parameters */
	memset(&ctx,0,sizeof(struct pingpong_context));
	memset(&user_param , 0 , sizeof(struct perftest_parameters));
	memset(&user_comm,0,sizeof(struct perftest_comm));

	user_param.verb    = READ;
	user_param.tst     = BW;
	strncpy(user_param.version, VERSION, sizeof(user_param.version));

	ret_parser = parser(&user_param,argv,argc);
	if (ret_parser) {
		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
			fprintf(stderr," Parser function exited with Error\n");
		return 1;
	}

	if((user_param.connection_type == DC || user_param.use_xrc) && user_param.duplex) {
		user_param.num_of_qps *= 2;
	}

	ib_dev =ctx_find_dev(user_param.ib_devname);
	if (!ib_dev)
		return 7;

	/* Getting the relevant context from the device */
	ctx.context = ibv_open_device(ib_dev);
	if (!ctx.context) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return 1;
	}

	/* See if MTU and link type are valid and supported. */
	if (check_link(ctx.context,&user_param)) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return FAILURE;
	}

	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
	if (create_comm_struct(&user_comm,&user_param)) {
		fprintf(stderr," Unable to create RDMA_CM resources\n");
		return 1;
	}

	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
		printf("\n************************************\n");
		printf("* Waiting for client to connect... *\n");
		printf("************************************\n");
	}

	/* Initialize the connection and print the local data. */
	if (establish_connection(&user_comm)) {
		fprintf(stderr," Unable to init the socket connection\n");
		return FAILURE;
	}

	exchange_versions(&user_comm, &user_param);

	check_sys_data(&user_comm, &user_param);

	/* See if MTU and link type are valid and supported. */
	if (check_mtu(ctx.context,&user_param, &user_comm)) {
		fprintf(stderr, " Couldn't get context for the device\n");
		return FAILURE;
	}

	/* Print basic test information. */
	ctx_print_test_info(&user_param);

	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);

	/* Allocating arrays needed for the test. */
	alloc_ctx(&ctx,&user_param);

	/* Create (if nessacery) the rdma_cm ids and channel. */
	if (user_param.work_rdma_cm == ON) {

		if (user_param.machine == CLIENT) {
			if (retry_rdma_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}

		} else {
			if (create_rdma_resources(&ctx,&user_param)) {
				fprintf(stderr," Unable to create the rdma_resources\n");
				return FAILURE;
			}
			if (rdma_server_connect(&ctx,&user_param)) {
				fprintf(stderr,"Unable to perform rdma_client function\n");
				return FAILURE;
			}
		}

	} else {
		/* create all the basic IB resources. */
		if (ctx_init(&ctx,&user_param)) {
			fprintf(stderr, " Couldn't create IB resources\n");
			return FAILURE;
		}
	}

	/* Set up the Connection. */
	if (set_up_connection(&ctx,&user_param,my_dest)) {
		fprintf(stderr," Unable to set up socket connection\n");
		return FAILURE;
	}

	/* Print this machine QP information */
	for (i=0; i < user_param.num_of_qps; i++)
		ctx_print_pingpong_data(&my_dest[i],&user_comm);

	user_comm.rdma_params->side = REMOTE;

	for (i=0; i < user_param.num_of_qps; i++) {

		/* shaking hands and gather the other side info. */
		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
			fprintf(stderr,"Failed to exchange data between server and clients\n");
			return 1;
		}
		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
	}

	if (user_param.work_rdma_cm == OFF) {
		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
			fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
			fprintf(stderr," Please Try to use a different IP version.\n\n");
			return 1;
		}
	}

	if (user_param.work_rdma_cm == OFF) {

		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
			fprintf(stderr," Unable to Connect the HCA's through the link\n");
			return 1;
		}
	}

	/* An additional handshake is required after moving qp to RTR. */
	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
		fprintf(stderr,"Failed to exchange data between server and clients\n");
		return 1;
	}

	if (user_param.output == FULL_VERBOSITY) {
		if (user_param.report_per_port) {
			printf(RESULT_LINE_PER_PORT);
			printf((user_param.report_fmt == MBS ? RESULT_FMT_PER_PORT : RESULT_FMT_G_PER_PORT));
		}
		else {
			printf(RESULT_LINE);
			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
		}
		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
	}

	/* For half duplex tests, server just waits for client to exit */
	if (user_param.machine == SERVER && !user_param.duplex) {

		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
			fprintf(stderr," Failed to exchange data between server and clients\n");
			return FAILURE;
		}

		xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
		print_full_bw_report(&user_param, &rem_bw_rep, NULL);

		if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
			fprintf(stderr,"Failed to close connection between server and client\n");
			return 1;
		}
		if (user_param.output == FULL_VERBOSITY) {
			if (user_param.report_per_port)
				printf(RESULT_LINE_PER_PORT);
			else
				printf(RESULT_LINE);
		}

		if (user_param.work_rdma_cm == ON) {
			if (destroy_ctx(&ctx,&user_param)) {
				fprintf(stderr, "Failed to destroy resources\n");
				return 1;
			}
			user_comm.rdma_params->work_rdma_cm = ON;
			return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
		}

		return destroy_ctx(&ctx,&user_param);

	}

	if (user_param.use_event) {
		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
			fprintf(stderr, "Couldn't request CQ notification\n");
			return 1;
		}
	}

	if (user_param.test_method == RUN_ALL) {

		for (i = 1; i < 24 ; ++i) {

			user_param.size = (uint64_t)1 << i;
			ctx_set_send_wqes(&ctx,&user_param,rem_dest);

			if(perform_warm_up(&ctx,&user_param)) {
				fprintf(stderr,"Problems with warm up\n");
				return 1;
			}

			if(user_param.duplex) {
				if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
					fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
					return 1;
				}
			}

			if(run_iter_bw(&ctx,&user_param))
				return 17;

			if (user_param.duplex && (atof(user_param.version) >= 4.6)) {
				if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
					fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
					return 1;
				}
			}

			print_report_bw(&user_param,&my_bw_rep);

			if (user_param.duplex) {
				xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
				print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
			}
		}

	} else if (user_param.test_method == RUN_REGULAR) {

		ctx_set_send_wqes(&ctx,&user_param,rem_dest);

		if(perform_warm_up(&ctx,&user_param)) {
			fprintf(stderr,"Problems with warm up\n");
			return 1;
		}

		if(user_param.duplex) {
			if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
				fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
				return 1;
			}
		}

		if(run_iter_bw(&ctx,&user_param)) {
			fprintf(stderr," Failed to complete run_iter_bw function successfully\n");
			return 1;
		}

		print_report_bw(&user_param,&my_bw_rep);

		if (user_param.duplex) {
			xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
			print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
		}

		if (user_param.report_both && user_param.duplex) {
			printf(RESULT_LINE);
			printf("\n Local results: \n");
			printf(RESULT_LINE);
			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
			print_full_bw_report(&user_param, &my_bw_rep, NULL);
			printf(RESULT_LINE);

			printf("\n Remote results: \n");
			printf(RESULT_LINE);
			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
			print_full_bw_report(&user_param, &rem_bw_rep, NULL);
		}
	} else if (user_param.test_method == RUN_INFINITELY) {

		ctx_set_send_wqes(&ctx,&user_param,rem_dest);

		if(run_iter_bw_infinitely(&ctx,&user_param)) {
			fprintf(stderr," Error occured while running! aborting ...\n");
			return 1;
		}
	}

	if (user_param.output == FULL_VERBOSITY) {
		if (user_param.report_per_port)
			printf(RESULT_LINE_PER_PORT);
		else
			printf(RESULT_LINE);
	}

	/* For half duplex tests, server just waits for client to exit */
	if (user_param.machine == CLIENT && !user_param.duplex) {

		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
			fprintf(stderr," Failed to exchange data between server and clients\n");
			return FAILURE;
		}

		xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
	}

	if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
		fprintf(stderr,"Failed to close connection between server and client\n");
		return 1;
	}

	if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
		fprintf(stderr,"Error: BW result is below bw limit\n");
		return 1;
	}

	if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) {
		fprintf(stderr,"Error: Msg rate  is below msg_rate limit\n");
		return 1;
	}

	if (user_param.work_rdma_cm == ON) {
		if (destroy_ctx(&ctx,&user_param)) {
			fprintf(stderr, "Failed to destroy resources\n");
			return 1;
		}
		user_comm.rdma_params->work_rdma_cm = ON;
		return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
	}

	return destroy_ctx(&ctx,&user_param);
}