/* init the transport by its name */ static ucs_status_t init_iface(char *dev_name, char *tl_name, struct iface_info *iface_p) { ucs_status_t status; uct_iface_config_t *config; /* Defines interface configuration options */ /* Read transport-specific interface configuration */ status = uct_iface_config_read(tl_name, NULL, NULL, &config); CHKERR_JUMP(UCS_OK != status, "setup iface_config", error_ret); /* Open communication interface */ status = uct_iface_open(iface_p->pd, iface_p->worker, tl_name, dev_name, 0, config, &iface_p->iface); uct_config_release(config); CHKERR_JUMP(UCS_OK != status, "open temporary interface", error_ret); /* Get interface attributes */ status = uct_iface_query(iface_p->iface, &iface_p->attr); CHKERR_JUMP(UCS_OK != status, "query iface", error_iface); /* Check if current device and transport support short active messages */ if (iface_p->attr.cap.flags & UCT_IFACE_FLAG_AM_SHORT) { return UCS_OK; } error_iface: uct_iface_close(iface_p->iface); error_ret: return UCS_ERR_UNSUPPORTED; }
/* Checks if the device and transports are supported by UCX */ static ucs_status_t resource_supported(char *dev_name, char *tl_name, int kill_iface) { ucs_status_t status; /* Read transport-specific interface configuration */ status = uct_iface_config_read(tl_name, NULL, NULL, &iface_config); if (UCS_OK != status) { fprintf(stderr, "Failed to setup iface_config.\n");fflush(stderr); goto error0; } /* Open communication interface */ status = uct_iface_open(pd, worker, tl_name, dev_name, 0, iface_config, &iface); uct_iface_config_release(iface_config); if (UCS_OK != status) { fprintf(stderr, "Failed to open temporary interface.\n");fflush(stderr); goto error0; } /* Get interface attributes */ status = uct_iface_query(iface, &iface_attr); if (UCS_OK != status) { fprintf(stderr, "Failed to query iface.\n");fflush(stderr); goto error_iface0; } /* Check if current device and transport support short active messages */ if (iface_attr.cap.flags & UCT_IFACE_FLAG_AM_SHORT) { if (kill_iface) { uct_iface_close(iface); } return UCS_OK; } return UCS_ERR_UNSUPPORTED; error_iface0: uct_iface_close(iface); error0: return status; }
static ucs_status_t uct_perf_setup(ucx_perf_context_t *perf, ucx_perf_params_t *params) { uct_iface_config_t *iface_config; ucs_status_t status; uct_iface_params_t iface_params = { .tl_name = params->uct.tl_name, .dev_name = params->uct.dev_name, .rx_headroom = 0 }; status = ucs_async_context_init(&perf->uct.async, params->async_mode); if (status != UCS_OK) { goto out; } status = uct_worker_create(&perf->uct.async, params->thread_mode, &perf->uct.worker); if (status != UCS_OK) { goto out_cleanup_async; } status = uct_perf_create_md(perf); if (status != UCS_OK) { goto out_destroy_worker; } status = uct_iface_config_read(params->uct.tl_name, NULL, NULL, &iface_config); if (status != UCS_OK) { goto out_destroy_md; } status = uct_iface_open(perf->uct.md, perf->uct.worker, &iface_params, iface_config, &perf->uct.iface); uct_config_release(iface_config); if (status != UCS_OK) { ucs_error("Failed to open iface: %s", ucs_status_string(status)); goto out_destroy_md; } status = uct_perf_test_check_capabilities(params, perf->uct.iface); if (status != UCS_OK) { goto out_iface_close; } status = uct_perf_test_alloc_mem(perf, params); if (status != UCS_OK) { goto out_iface_close; } status = uct_perf_test_setup_endpoints(perf); if (status != UCS_OK) { ucs_error("Failed to setup endpoints: %s", ucs_status_string(status)); goto out_free_mem; } return UCS_OK; out_free_mem: uct_perf_test_free_mem(perf); out_iface_close: uct_iface_close(perf->uct.iface); out_destroy_md: uct_md_close(perf->uct.md); out_destroy_worker: uct_worker_destroy(perf->uct.worker); out_cleanup_async: ucs_async_context_cleanup(&perf->uct.async); out: return status; } static void uct_perf_cleanup(ucx_perf_context_t *perf) { uct_perf_test_cleanup_endpoints(perf); uct_perf_test_free_mem(perf); uct_iface_close(perf->uct.iface); uct_md_close(perf->uct.md); uct_worker_destroy(perf->uct.worker); ucs_async_context_cleanup(&perf->uct.async); }
void print_uct_config(ucs_config_print_flags_t print_flags, const char *tl_name) { uct_md_resource_desc_t *md_resources; unsigned md_rsc_index, num_md_resources; uct_tl_resource_desc_t *tl_resources; unsigned tl_rsc_index, num_tl_resources; uct_iface_config_t *config; char tl_names[UINT8_MAX][UCT_TL_NAME_MAX]; char cfg_title[UCT_TL_NAME_MAX + 128]; unsigned i, num_tls; ucs_status_t status; uct_md_h md; uct_md_config_t *md_config; status = uct_query_md_resources(&md_resources, &num_md_resources); if (status != UCS_OK) { return; } uct_md_component_config_print(print_flags); num_tls = 0; for (md_rsc_index = 0; md_rsc_index < num_md_resources; ++md_rsc_index) { status = uct_md_config_read(md_resources[md_rsc_index].md_name, NULL, NULL, &md_config); if (status != UCS_OK) { continue; } status = uct_md_open(md_resources[md_rsc_index].md_name, md_config, &md); uct_config_release(md_config); if (status != UCS_OK) { continue; } status = uct_md_query_tl_resources(md, &tl_resources, &num_tl_resources); if (status != UCS_OK) { uct_md_close(md); continue; } for (tl_rsc_index = 0; tl_rsc_index < num_tl_resources; ++tl_rsc_index) { i = 0; while (i < num_tls) { if (!strcmp(tl_names[i], tl_resources[tl_rsc_index].tl_name)) { break; } ++i; } /* Make sure this transport is not inserted to the array before, and * if user selects a specific transport - also make sure this is it. */ if ((i == num_tls) && ((tl_name == NULL) || !strcmp(tl_name, tl_resources[tl_rsc_index].tl_name))) { strncpy(tl_names[num_tls], tl_resources[tl_rsc_index].tl_name, UCT_TL_NAME_MAX); ++num_tls; } } uct_release_tl_resource_list(tl_resources); uct_md_close(md); } uct_release_md_resource_list(md_resources); for (i = 0; i < num_tls; ++i) { snprintf(cfg_title, sizeof(cfg_title), "%s transport configuration", tl_names[i]); status = uct_iface_config_read(tl_names[i], NULL, NULL, &config); if (status != UCS_OK) { printf("# < Failed to read configuration >\n"); continue; } uct_config_print(config, stdout, cfg_title, print_flags); uct_config_release(config); } }
static void print_iface_info(uct_worker_h worker, uct_md_h md, uct_tl_resource_desc_t *resource) { uct_iface_config_t *iface_config; uct_iface_attr_t iface_attr; ucs_status_t status; uct_iface_h iface; char buf[200] = {0}; status = uct_iface_config_read(resource->tl_name, NULL, NULL, &iface_config); if (status != UCS_OK) { return; } printf("# Device: %s\n", resource->dev_name); status = uct_iface_open(md, worker, resource->tl_name, resource->dev_name, 0, iface_config, &iface); uct_config_release(iface_config); if (status != UCS_OK) { printf("# < failed to open interface >\n"); return; } printf("#\n"); printf("# capabilities:\n"); status = uct_iface_query(iface, &iface_attr); if (status != UCS_OK) { printf("# < failed to query interface >\n"); } else { printf("# bandwidth: %.2f MB/sec\n", iface_attr.bandwidth / (1024 * 1024)); printf("# latency: %.0f nsec\n", iface_attr.latency * 1e9); printf("# overhead: %.0f nsec\n", iface_attr.overhead * 1e9); PRINT_CAP(PUT_SHORT, iface_attr.cap.flags, iface_attr.cap.put.max_short); PRINT_CAP(PUT_BCOPY, iface_attr.cap.flags, iface_attr.cap.put.max_bcopy); PRINT_CAP(PUT_ZCOPY, iface_attr.cap.flags, iface_attr.cap.put.max_zcopy); PRINT_CAP(GET_BCOPY, iface_attr.cap.flags, iface_attr.cap.get.max_bcopy); PRINT_CAP(GET_ZCOPY, iface_attr.cap.flags, iface_attr.cap.get.max_zcopy); PRINT_CAP(AM_SHORT, iface_attr.cap.flags, iface_attr.cap.am.max_short); PRINT_CAP(AM_BCOPY, iface_attr.cap.flags, iface_attr.cap.am.max_bcopy); PRINT_CAP(AM_ZCOPY, iface_attr.cap.flags, iface_attr.cap.am.max_zcopy); if (iface_attr.cap.flags & (UCT_IFACE_FLAG_AM_BCOPY|UCT_IFACE_FLAG_AM_ZCOPY)) { printf("# am header: %s\n", size_limit_to_str(iface_attr.cap.am.max_hdr)); } PRINT_ATOMIC_CAP(ATOMIC_ADD, iface_attr.cap.flags); PRINT_ATOMIC_CAP(ATOMIC_FADD, iface_attr.cap.flags); PRINT_ATOMIC_CAP(ATOMIC_SWAP, iface_attr.cap.flags); PRINT_ATOMIC_CAP(ATOMIC_CSWAP, iface_attr.cap.flags); buf[0] = '\0'; if (iface_attr.cap.flags & (UCT_IFACE_FLAG_CONNECT_TO_EP | UCT_IFACE_FLAG_CONNECT_TO_IFACE)) { if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) { strncat(buf, " to ep,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) { strncat(buf, " to iface,", sizeof(buf) - 1); } buf[strlen(buf) - 1] = '\0'; } else { strncat(buf, " none", sizeof(buf) - 1); } printf("# connection:%s\n", buf); printf("# device address: %zu bytes\n", iface_attr.device_addr_len); if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) { printf("# iface address: %zu bytes\n", iface_attr.iface_addr_len); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) { printf("# ep address: %zu bytes\n", iface_attr.ep_addr_len); } buf[0] = '\0'; if (iface_attr.cap.flags & (UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF | UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF | UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF | UCT_IFACE_FLAG_ERRHANDLE_AM_ID | UCT_IFACE_FLAG_ERRHANDLE_REMOTE_MEM | UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE)) { if (iface_attr.cap.flags & (UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF | UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF | UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF)) { strncat(buf, " buffer (", sizeof(buf) - 1); if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF) { strncat(buf, "short,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF) { strncat(buf, "bcopy,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF) { strncat(buf, "zcopy,", sizeof(buf) - 1); } buf[strlen(buf) - 1] = '\0'; strncat(buf, "),", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_AM_ID) { strncat(buf, " active-message id,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_REMOTE_MEM) { strncat(buf, " remote access,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE) { strncat(buf, " peer failure,", sizeof(buf) - 1); } buf[strlen(buf) - 1] = '\0'; } else { strncat(buf, " none", sizeof(buf) - 1); } printf("# error handling:%s\n", buf); } uct_iface_close(iface); printf("#\n"); }
static void print_iface_info(uct_worker_h worker, uct_md_h md, uct_tl_resource_desc_t *resource) { uct_iface_config_t *iface_config; uct_iface_attr_t iface_attr; ucs_status_t status; uct_iface_h iface; char buf[200] = {0}; uct_iface_params_t iface_params = { .tl_name = resource->tl_name, .dev_name = resource->dev_name, .rx_headroom = 0 }; status = uct_iface_config_read(resource->tl_name, NULL, NULL, &iface_config); if (status != UCS_OK) { return; } printf("# Device: %s\n", resource->dev_name); status = uct_iface_open(md, worker, &iface_params, iface_config, &iface); uct_config_release(iface_config); if (status != UCS_OK) { printf("# < failed to open interface >\n"); return; } printf("#\n"); printf("# capabilities:\n"); status = uct_iface_query(iface, &iface_attr); if (status != UCS_OK) { printf("# < failed to query interface >\n"); } else { printf("# bandwidth: %-.2f MB/sec\n", iface_attr.bandwidth / (1024 * 1024)); printf("# latency: %-.0f nsec\n", iface_attr.latency * 1e9); printf("# overhead: %-.0f nsec\n", iface_attr.overhead * 1e9); PRINT_CAP(PUT_SHORT, iface_attr.cap.flags, iface_attr.cap.put.max_short); PRINT_CAP(PUT_BCOPY, iface_attr.cap.flags, iface_attr.cap.put.max_bcopy); PRINT_CAP(PUT_ZCOPY, iface_attr.cap.flags, iface_attr.cap.put.max_zcopy); PRINT_CAP(GET_BCOPY, iface_attr.cap.flags, iface_attr.cap.get.max_bcopy); PRINT_CAP(GET_ZCOPY, iface_attr.cap.flags, iface_attr.cap.get.max_zcopy); PRINT_CAP(AM_SHORT, iface_attr.cap.flags, iface_attr.cap.am.max_short); PRINT_CAP(AM_BCOPY, iface_attr.cap.flags, iface_attr.cap.am.max_bcopy); PRINT_CAP(AM_ZCOPY, iface_attr.cap.flags, iface_attr.cap.am.max_zcopy); if (iface_attr.cap.flags & (UCT_IFACE_FLAG_AM_BCOPY|UCT_IFACE_FLAG_AM_ZCOPY)) { printf("# am header: %s\n", size_limit_to_str(iface_attr.cap.am.max_hdr)); } PRINT_ATOMIC_CAP(ATOMIC_ADD, iface_attr.cap.flags); PRINT_ATOMIC_CAP(ATOMIC_FADD, iface_attr.cap.flags); PRINT_ATOMIC_CAP(ATOMIC_SWAP, iface_attr.cap.flags); PRINT_ATOMIC_CAP(ATOMIC_CSWAP, iface_attr.cap.flags); buf[0] = '\0'; if (iface_attr.cap.flags & (UCT_IFACE_FLAG_CONNECT_TO_EP | UCT_IFACE_FLAG_CONNECT_TO_IFACE)) { if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) { strncat(buf, " to ep,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) { strncat(buf, " to iface,", sizeof(buf) - 1); } buf[strlen(buf) - 1] = '\0'; } else { strncat(buf, " none", sizeof(buf) - 1); } printf("# connection:%s\n", buf); printf("# device address: %zu bytes\n", iface_attr.device_addr_len); if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) { printf("# iface address: %zu bytes\n", iface_attr.iface_addr_len); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) { printf("# ep address: %zu bytes\n", iface_attr.ep_addr_len); } buf[0] = '\0'; if (iface_attr.cap.flags & (UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF | UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF | UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF | UCT_IFACE_FLAG_ERRHANDLE_AM_ID | UCT_IFACE_FLAG_ERRHANDLE_REMOTE_MEM | UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE)) { if (iface_attr.cap.flags & (UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF | UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF | UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF)) { strncat(buf, " buffer (", sizeof(buf) - 1); if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF) { strncat(buf, "short,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF) { strncat(buf, "bcopy,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF) { strncat(buf, "zcopy,", sizeof(buf) - 1); } buf[strlen(buf) - 1] = '\0'; strncat(buf, "),", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_AM_ID) { strncat(buf, " active-message id,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_REMOTE_MEM) { strncat(buf, " remote access,", sizeof(buf) - 1); } if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE) { strncat(buf, " peer failure,", sizeof(buf) - 1); } buf[strlen(buf) - 1] = '\0'; } else { strncat(buf, " none", sizeof(buf) - 1); } printf("# error handling:%s\n", buf); } uct_iface_close(iface); printf("#\n"); } static ucs_status_t print_tl_info(uct_md_h md, const char *tl_name, uct_tl_resource_desc_t *resources, unsigned num_resources, int print_opts, ucs_config_print_flags_t print_flags) { ucs_async_context_t async; uct_worker_h worker; ucs_status_t status; unsigned i; status = ucs_async_context_init(&async, UCS_ASYNC_MODE_THREAD); if (status != UCS_OK) { return status; } /* coverity[alloc_arg] */ status = uct_worker_create(&async, UCS_THREAD_MODE_MULTI, &worker); if (status != UCS_OK) { goto out; } printf("#\n"); printf("# Transport: %s\n", tl_name); printf("#\n"); if (num_resources == 0) { printf("# (No supported devices found)\n"); } for (i = 0; i < num_resources; ++i) { ucs_assert(!strcmp(tl_name, resources[i].tl_name)); print_iface_info(worker, md, &resources[i]); } uct_worker_destroy(worker); out: ucs_async_context_cleanup(&async); return status; }