void ibv_attach_device (struct netif *netif) { struct ibvif *ibvif; uint8_t mac[6] = {0x00, 0x02, 0xc9, 0xa4, 0x59, 0x41}; struct ibv_qp_attr qp_attr, query_attr; struct ibv_qp_init_attr init_attr; int qp_flags, attr_mask; uint8_t port_num = 1; fr_attr fr; int start, end, index, i; ibvif = (struct ibvif *)netif->state; /* Attaching the qp to the spec */ memset(&fr.attr_info, 0 , sizeof(struct ibv_exp_flow_attr)); fr.attr_info.type = IBV_EXP_FLOW_ATTR_NORMAL; fr.attr_info.size = sizeof(struct flow_rules); fr.attr_info.priority = 0; fr.attr_info.num_of_specs = 3; fr.attr_info.port = port_num; fr.attr_info.flags = 0; memset(&fr.spec_info, 0 , sizeof(struct ibv_exp_flow_spec_eth)); fr.spec_info.type = IBV_EXP_FLOW_SPEC_ETH; fr.spec_info.size = sizeof(struct ibv_exp_flow_spec_eth); fr.spec_info.val.ether_type = IP_ETHER_TYPE; fr.spec_info.mask.ether_type = 0xffff; memcpy(fr.spec_info.val.dst_mac, mac, sizeof(fr.spec_info.mask.dst_mac)); memset(fr.spec_info.mask.dst_mac, 0xff, sizeof(fr.spec_info.mask.dst_mac)); memset(&fr.ip_spec_info, 0 , sizeof(struct ibv_exp_flow_spec_ipv4)); fr.ip_spec_info.type = IBV_EXP_FLOW_SPEC_IPV4; fr.ip_spec_info.size = sizeof(struct ibv_exp_flow_spec_ipv4); fr.ip_spec_info.val.dst_ip = inet_addr("10.0.0.1"); fr.ip_spec_info.mask.dst_ip = 0xffffffff; if (netif->prot_thread->cpu == 6) { fr.ip_spec_info.val.src_ip = inet_addr("10.0.0.3"); } else if (netif->prot_thread->cpu == 7) { fr.ip_spec_info.val.src_ip = inet_addr("10.0.0.5"); } else if (netif->prot_thread->cpu == 8) { fr.ip_spec_info.val.src_ip = inet_addr("10.0.0.6"); } else if (netif->prot_thread->cpu == 9) { fr.ip_spec_info.val.src_ip = inet_addr("10.0.0.7"); } else if (netif->prot_thread->cpu == 10) { fr.ip_spec_info.val.src_ip = inet_addr("10.0.0.8"); } else if (netif->prot_thread->cpu == 11) { fr.ip_spec_info.val.src_ip = inet_addr("10.0.0.9"); } fr.ip_spec_info.mask.src_ip = 0xffffffff; memset(&fr.tcp_spec_info, 0 , sizeof(struct ibv_exp_flow_spec_tcp_udp)); fr.tcp_spec_info.type = IBV_EXP_FLOW_SPEC_TCP; fr.tcp_spec_info.size = sizeof(struct ibv_exp_flow_spec_tcp_udp); fr.tcp_spec_info.val.dst_port = bswap_16(90); fr.tcp_spec_info.mask.dst_port = 0xffff; ibvif->flow = ibv_exp_create_flow(ibvif->qp, &fr.attr_info); if (!ibvif->flow) { perror("IBV can't create flow\n"); exit(1); } /*start = netif->prot_thread->cpu * 100; end = start + 100; index = 0; for (i=start; i<end; i++) { fr.tcp_spec_info.val.src_port = htons(i); fr.tcp_spec_info.mask.src_port = 0xffff; ibvif->flow[index] = ibv_exp_create_flow(ibvif->qp, &fr.attr_info); if (!ibvif->flow[index]) { perror("IBV can't create flow\n"); exit(1); } index++; }*/ /* modify QP to send and receive */ qp_flags = IBV_QP_STATE; memset(&qp_attr, 0, sizeof(struct ibv_qp_attr)); qp_attr.qp_state = IBV_QPS_RTR; qp_attr.ah_attr.src_path_bits = 0; qp_attr.ah_attr.port_num = 1; //qp_attr.ah_attr.is_global = 0; //qp_attr.ah_attr.sl = 1; if (ibv_modify_qp(ibvif->qp, &qp_attr, qp_flags)) { perror("IBV can't set state to RTR\n"); exit(1); } qp_flags = IBV_QP_STATE; memset(&qp_attr, 0, sizeof(struct ibv_qp_attr)); qp_attr.qp_state = IBV_QPS_RTS; qp_attr.ah_attr.src_path_bits = 0; qp_attr.ah_attr.port_num = 1; //qp_attr.ah_attr.is_global = 0; //qp_attr.ah_attr.sl = 1; if (ibv_modify_qp(ibvif->qp, &qp_attr, qp_flags)) { perror("IBV can't set state to RTS\n"); exit(1); } ibv_query_qp(ibvif->qp, &query_attr, attr_mask, &init_attr); infini_post_recv(ibvif); netif->flags = NETIF_FLAG_BROADCAST | NETIF_FLAG_ETHARP | NETIF_FLAG_LINK_UP | NETIF_FLAG_UP; }
/* * Main function. implements raw_ethernet_send_lat */ int main(int argc, char *argv[]) { struct ibv_device *ib_dev = NULL; struct pingpong_context ctx; struct raw_ethernet_info my_dest_info,rem_dest_info; int ret_parser; struct perftest_parameters user_param; #ifdef HAVE_RAW_ETH_EXP struct ibv_exp_flow *flow_create_result = NULL; struct ibv_exp_flow_attr *flow_rules = NULL; struct ibv_exp_flow *flow_promisc = NULL; #else struct ibv_flow *flow_create_result = NULL; struct ibv_flow_attr *flow_rules = NULL; #endif struct report_options report; //allocate memory space for user parameters memset(&ctx, 0, sizeof(struct pingpong_context)); memset(&user_param, 0, sizeof(struct perftest_parameters)); memset(&my_dest_info, 0 , sizeof(struct raw_ethernet_info)); memset(&rem_dest_info, 0 , sizeof(struct raw_ethernet_info)); /* init default values to user's parameters that's relvant for this test: * Raw Ethernet Send Latency Test */ user_param.verb = SEND; user_param.tst = LAT; strncpy(user_param.version, VERSION, sizeof(user_param.version)); user_param.connection_type = RawEth; user_param.r_flag = &report; if (check_flow_steering_support()) { return 1; } /* Configure the parameters values according to user arguments or default values. */ ret_parser = parser(&user_param, argv,argc); //check for parsing errors if (ret_parser) { if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT) fprintf(stderr," Parser function exited with Error\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } //this is a bidirectional test, so we need to let the init functions //to think we are in duplex mode //TODO: ask Ido if that's ok, or should I add another field in user_param user_param.duplex = 1; // Find the selected IB device (or default if the user didn't select one). ib_dev = ctx_find_dev(user_param.ib_devname); if (!ib_dev) { fprintf(stderr," Unable to find the Infiniband/RoCE device\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } // Getting the relevant context from the device ctx.context = ibv_open_device(ib_dev); if (!ctx.context) { fprintf(stderr, " Couldn't get context for the device\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } // See if MTU and link type are valid and supported. if (check_link_and_mtu(ctx.context, &user_param)) { fprintf(stderr, " Couldn't get context for the device\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return FAILURE; } // Allocating arrays needed for the test. alloc_ctx(&ctx, &user_param); // Print basic test information. ctx_print_test_info(&user_param); //set up the connection, return the required flow rules (notice that user_param->duplex == TRUE) //so the function will setup like it's a bidirectional test if (send_set_up_connection(&flow_rules, &ctx, &user_param, &my_dest_info, &rem_dest_info)) { fprintf(stderr," Unable to set up socket connection\n"); return 1; } //print specifications of the test print_spec(flow_rules,&user_param); // Create (if necessary) the rdma_cm ids and channel. if (user_param.work_rdma_cm == ON) { //create resources if (create_rdma_resources(&ctx, &user_param)) { fprintf(stderr," Unable to create the rdma_resources\n"); return FAILURE; } if (user_param.machine == CLIENT) { //Connects the client to a QP on the other machine with rdma_cm if (rdma_client_connect(&ctx, &user_param)) { fprintf(stderr,"Unable to perform rdma_client function\n"); return FAILURE; } } else if (rdma_server_connect(&ctx, &user_param)) { //Assigning a server to listen on rdma_cm port and connect to it. fprintf(stderr,"Unable to perform rdma_server function\n"); return FAILURE; } } else { // initalize IB resources (data buffer, PD, MR, CQ and events channel) if (ctx_init(&ctx, &user_param)) { fprintf(stderr, " Couldn't create IB resources\n"); return FAILURE; } } //attaching the qp to the spec #ifdef HAVE_RAW_ETH_EXP flow_create_result = ibv_exp_create_flow(ctx.qp[0], flow_rules); #else flow_create_result = ibv_create_flow(ctx.qp[0], flow_rules); #endif if (!flow_create_result){ perror("error"); fprintf(stderr, "Couldn't attach QP\n"); return FAILURE; } #ifdef HAVE_RAW_ETH_EXP if (user_param.use_promiscuous) { struct ibv_exp_flow_attr attr = { .type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT, .num_of_specs = 0, .port = user_param.ib_port, .flags = 0 }; if ((flow_promisc = ibv_exp_create_flow(ctx.qp[0], &attr)) == NULL) { perror("error"); fprintf(stderr, "Couldn't attach promiscous rule QP\n"); } } #endif //build ONE Raw Ethernet packets on ctx buffer create_raw_eth_pkt(&user_param,&ctx, &my_dest_info , &rem_dest_info); if (user_param.output == FULL_VERBOSITY) { printf(RESULT_LINE); printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR); printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT)); } // Prepare IB resources for rtr(ready to read)/rts(ready to send) if (user_param.work_rdma_cm == OFF) { if (ctx_connect(&ctx, NULL, &user_param, NULL)) { fprintf(stderr," Unable to Connect the HCA's through the link\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } } //Post Send send_wqes for current message size ctx_set_send_wqes(&ctx,&user_param,NULL); // Post receive recv_wqes for current message size if (ctx_set_recv_wqes(&ctx,&user_param)) { fprintf(stderr," Failed to post receive recv_wqes\n"); return 1; } //latency test function for SEND verb latency test. if (run_iter_lat_send(&ctx, &user_param)) { return 17; } //print report (like print_report_bw) in the correct format // (as set before: FMT_LAT or FMT_LAT_DUR) user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); //destory promisc flow #ifdef HAVE_RAW_ETH_EXP if (user_param.use_promiscuous) { if (ibv_exp_destroy_flow(flow_promisc)) { perror("error"); fprintf(stderr, "Couldn't Destory promisc flow\n"); return FAILURE; } } #endif //destroy flow #ifdef HAVE_RAW_ETH_EXP if (ibv_exp_destroy_flow(flow_create_result)) { #else if (ibv_destroy_flow(flow_create_result)) { #endif perror("error"); fprintf(stderr, "Couldn't Destory flow\n"); return FAILURE; } free(flow_rules); //Deallocate all perftest resources. if (destroy_ctx(&ctx, &user_param)) { fprintf(stderr,"Failed to destroy_ctx\n"); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 1; } if (user_param.output == FULL_VERBOSITY) printf(RESULT_LINE); DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__); return 0; }