/* init the transport by its name */
static ucs_status_t init_iface(char *dev_name, char *tl_name, struct iface_info *iface_p)
{
ucs_status_t status;
uct_iface_config_t *config; /* Defines interface configuration options */
/* Read transport-specific interface configuration */
status = uct_iface_config_read(tl_name, NULL, NULL, &config);
CHKERR_JUMP(UCS_OK != status, "setup iface_config", error_ret);
/* Open communication interface */
status = uct_iface_open(iface_p->pd, iface_p->worker, tl_name, dev_name, 0, config,
&iface_p->iface);
uct_config_release(config);
CHKERR_JUMP(UCS_OK != status, "open temporary interface", error_ret);
/* Get interface attributes */
status = uct_iface_query(iface_p->iface, &iface_p->attr);
CHKERR_JUMP(UCS_OK != status, "query iface", error_iface);
/* Check if current device and transport support short active messages */
if (iface_p->attr.cap.flags & UCT_IFACE_FLAG_AM_SHORT) {
return UCS_OK;
}
error_iface:
uct_iface_close(iface_p->iface);
error_ret:
return UCS_ERR_UNSUPPORTED;
}
/* Checks if the device and transports are supported by UCX */
static ucs_status_t resource_supported(char *dev_name, char *tl_name, int kill_iface)
{
ucs_status_t status;
/* Read transport-specific interface configuration */
status = uct_iface_config_read(tl_name, NULL, NULL, &iface_config);
if (UCS_OK != status) {
fprintf(stderr, "Failed to setup iface_config.\n");fflush(stderr);
goto error0;
}
/* Open communication interface */
status = uct_iface_open(pd, worker, tl_name, dev_name, 0, iface_config, &iface);
uct_iface_config_release(iface_config);
if (UCS_OK != status) {
fprintf(stderr, "Failed to open temporary interface.\n");fflush(stderr);
goto error0;
}
/* Get interface attributes */
status = uct_iface_query(iface, &iface_attr);
if (UCS_OK != status) {
fprintf(stderr, "Failed to query iface.\n");fflush(stderr);
goto error_iface0;
}
/* Check if current device and transport support short active messages */
if (iface_attr.cap.flags & UCT_IFACE_FLAG_AM_SHORT) {
if (kill_iface) {
uct_iface_close(iface);
}
return UCS_OK;
}
return UCS_ERR_UNSUPPORTED;
error_iface0:
uct_iface_close(iface);
error0:
return status;
}
static ucs_status_t uct_perf_setup(ucx_perf_context_t *perf, ucx_perf_params_t *params)
{
uct_iface_config_t *iface_config;
ucs_status_t status;
uct_iface_params_t iface_params = {
.tl_name = params->uct.tl_name,
.dev_name = params->uct.dev_name,
.rx_headroom = 0
};
status = ucs_async_context_init(&perf->uct.async, params->async_mode);
if (status != UCS_OK) {
goto out;
}
status = uct_worker_create(&perf->uct.async, params->thread_mode,
&perf->uct.worker);
if (status != UCS_OK) {
goto out_cleanup_async;
}
status = uct_perf_create_md(perf);
if (status != UCS_OK) {
goto out_destroy_worker;
}
status = uct_iface_config_read(params->uct.tl_name, NULL, NULL, &iface_config);
if (status != UCS_OK) {
goto out_destroy_md;
}
status = uct_iface_open(perf->uct.md, perf->uct.worker, &iface_params,
iface_config, &perf->uct.iface);
uct_config_release(iface_config);
if (status != UCS_OK) {
ucs_error("Failed to open iface: %s", ucs_status_string(status));
goto out_destroy_md;
}
status = uct_perf_test_check_capabilities(params, perf->uct.iface);
if (status != UCS_OK) {
goto out_iface_close;
}
status = uct_perf_test_alloc_mem(perf, params);
if (status != UCS_OK) {
goto out_iface_close;
}
status = uct_perf_test_setup_endpoints(perf);
if (status != UCS_OK) {
ucs_error("Failed to setup endpoints: %s", ucs_status_string(status));
goto out_free_mem;
}
return UCS_OK;
out_free_mem:
uct_perf_test_free_mem(perf);
out_iface_close:
uct_iface_close(perf->uct.iface);
out_destroy_md:
uct_md_close(perf->uct.md);
out_destroy_worker:
uct_worker_destroy(perf->uct.worker);
out_cleanup_async:
ucs_async_context_cleanup(&perf->uct.async);
out:
return status;
}
static void uct_perf_cleanup(ucx_perf_context_t *perf)
{
uct_perf_test_cleanup_endpoints(perf);
uct_perf_test_free_mem(perf);
uct_iface_close(perf->uct.iface);
uct_md_close(perf->uct.md);
uct_worker_destroy(perf->uct.worker);
ucs_async_context_cleanup(&perf->uct.async);
}
void print_uct_config(ucs_config_print_flags_t print_flags, const char *tl_name)
{
uct_md_resource_desc_t *md_resources;
unsigned md_rsc_index, num_md_resources;
uct_tl_resource_desc_t *tl_resources;
unsigned tl_rsc_index, num_tl_resources;
uct_iface_config_t *config;
char tl_names[UINT8_MAX][UCT_TL_NAME_MAX];
char cfg_title[UCT_TL_NAME_MAX + 128];
unsigned i, num_tls;
ucs_status_t status;
uct_md_h md;
uct_md_config_t *md_config;
status = uct_query_md_resources(&md_resources, &num_md_resources);
if (status != UCS_OK) {
return;
}
uct_md_component_config_print(print_flags);
num_tls = 0;
for (md_rsc_index = 0; md_rsc_index < num_md_resources; ++md_rsc_index) {
status = uct_md_config_read(md_resources[md_rsc_index].md_name, NULL,
NULL, &md_config);
if (status != UCS_OK) {
continue;
}
status = uct_md_open(md_resources[md_rsc_index].md_name, md_config, &md);
uct_config_release(md_config);
if (status != UCS_OK) {
continue;
}
status = uct_md_query_tl_resources(md, &tl_resources, &num_tl_resources);
if (status != UCS_OK) {
uct_md_close(md);
continue;
}
for (tl_rsc_index = 0; tl_rsc_index < num_tl_resources; ++tl_rsc_index) {
i = 0;
while (i < num_tls) {
if (!strcmp(tl_names[i], tl_resources[tl_rsc_index].tl_name)) {
break;
}
++i;
}
/* Make sure this transport is not inserted to the array before, and
* if user selects a specific transport - also make sure this is it.
*/
if ((i == num_tls) &&
((tl_name == NULL) || !strcmp(tl_name, tl_resources[tl_rsc_index].tl_name)))
{
strncpy(tl_names[num_tls], tl_resources[tl_rsc_index].tl_name,
UCT_TL_NAME_MAX);
++num_tls;
}
}
uct_release_tl_resource_list(tl_resources);
uct_md_close(md);
}
uct_release_md_resource_list(md_resources);
for (i = 0; i < num_tls; ++i) {
snprintf(cfg_title, sizeof(cfg_title), "%s transport configuration",
tl_names[i]);
status = uct_iface_config_read(tl_names[i], NULL, NULL, &config);
if (status != UCS_OK) {
printf("# < Failed to read configuration >\n");
continue;
}
uct_config_print(config, stdout, cfg_title, print_flags);
uct_config_release(config);
}
}
static void print_iface_info(uct_worker_h worker, uct_md_h md,
uct_tl_resource_desc_t *resource)
{
uct_iface_config_t *iface_config;
uct_iface_attr_t iface_attr;
ucs_status_t status;
uct_iface_h iface;
char buf[200] = {0};
status = uct_iface_config_read(resource->tl_name, NULL, NULL, &iface_config);
if (status != UCS_OK) {
return;
}
printf("# Device: %s\n", resource->dev_name);
status = uct_iface_open(md, worker, resource->tl_name, resource->dev_name,
0, iface_config, &iface);
uct_config_release(iface_config);
if (status != UCS_OK) {
printf("# < failed to open interface >\n");
return;
}
printf("#\n");
printf("# capabilities:\n");
status = uct_iface_query(iface, &iface_attr);
if (status != UCS_OK) {
printf("# < failed to query interface >\n");
} else {
printf("# bandwidth: %.2f MB/sec\n", iface_attr.bandwidth / (1024 * 1024));
printf("# latency: %.0f nsec\n", iface_attr.latency * 1e9);
printf("# overhead: %.0f nsec\n", iface_attr.overhead * 1e9);
PRINT_CAP(PUT_SHORT, iface_attr.cap.flags, iface_attr.cap.put.max_short);
PRINT_CAP(PUT_BCOPY, iface_attr.cap.flags, iface_attr.cap.put.max_bcopy);
PRINT_CAP(PUT_ZCOPY, iface_attr.cap.flags, iface_attr.cap.put.max_zcopy);
PRINT_CAP(GET_BCOPY, iface_attr.cap.flags, iface_attr.cap.get.max_bcopy);
PRINT_CAP(GET_ZCOPY, iface_attr.cap.flags, iface_attr.cap.get.max_zcopy);
PRINT_CAP(AM_SHORT, iface_attr.cap.flags, iface_attr.cap.am.max_short);
PRINT_CAP(AM_BCOPY, iface_attr.cap.flags, iface_attr.cap.am.max_bcopy);
PRINT_CAP(AM_ZCOPY, iface_attr.cap.flags, iface_attr.cap.am.max_zcopy);
if (iface_attr.cap.flags & (UCT_IFACE_FLAG_AM_BCOPY|UCT_IFACE_FLAG_AM_ZCOPY)) {
printf("# am header: %s\n",
size_limit_to_str(iface_attr.cap.am.max_hdr));
}
PRINT_ATOMIC_CAP(ATOMIC_ADD, iface_attr.cap.flags);
PRINT_ATOMIC_CAP(ATOMIC_FADD, iface_attr.cap.flags);
PRINT_ATOMIC_CAP(ATOMIC_SWAP, iface_attr.cap.flags);
PRINT_ATOMIC_CAP(ATOMIC_CSWAP, iface_attr.cap.flags);
buf[0] = '\0';
if (iface_attr.cap.flags & (UCT_IFACE_FLAG_CONNECT_TO_EP |
UCT_IFACE_FLAG_CONNECT_TO_IFACE))
{
if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) {
strncat(buf, " to ep,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) {
strncat(buf, " to iface,", sizeof(buf) - 1);
}
buf[strlen(buf) - 1] = '\0';
} else {
strncat(buf, " none", sizeof(buf) - 1);
}
printf("# connection:%s\n", buf);
printf("# device address: %zu bytes\n", iface_attr.device_addr_len);
if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) {
printf("# iface address: %zu bytes\n", iface_attr.iface_addr_len);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) {
printf("# ep address: %zu bytes\n", iface_attr.ep_addr_len);
}
buf[0] = '\0';
if (iface_attr.cap.flags & (UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF |
UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF |
UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF |
UCT_IFACE_FLAG_ERRHANDLE_AM_ID |
UCT_IFACE_FLAG_ERRHANDLE_REMOTE_MEM |
UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE))
{
if (iface_attr.cap.flags & (UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF |
UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF |
UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF))
{
strncat(buf, " buffer (", sizeof(buf) - 1);
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF) {
strncat(buf, "short,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF) {
strncat(buf, "bcopy,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF) {
strncat(buf, "zcopy,", sizeof(buf) - 1);
}
buf[strlen(buf) - 1] = '\0';
strncat(buf, "),", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_AM_ID) {
strncat(buf, " active-message id,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_REMOTE_MEM) {
strncat(buf, " remote access,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE) {
strncat(buf, " peer failure,", sizeof(buf) - 1);
}
buf[strlen(buf) - 1] = '\0';
} else {
strncat(buf, " none", sizeof(buf) - 1);
}
printf("# error handling:%s\n", buf);
}
uct_iface_close(iface);
printf("#\n");
}
static void print_iface_info(uct_worker_h worker, uct_md_h md,
uct_tl_resource_desc_t *resource)
{
uct_iface_config_t *iface_config;
uct_iface_attr_t iface_attr;
ucs_status_t status;
uct_iface_h iface;
char buf[200] = {0};
uct_iface_params_t iface_params = {
.tl_name = resource->tl_name,
.dev_name = resource->dev_name,
.rx_headroom = 0
};
status = uct_iface_config_read(resource->tl_name, NULL, NULL, &iface_config);
if (status != UCS_OK) {
return;
}
printf("# Device: %s\n", resource->dev_name);
status = uct_iface_open(md, worker, &iface_params, iface_config, &iface);
uct_config_release(iface_config);
if (status != UCS_OK) {
printf("# < failed to open interface >\n");
return;
}
printf("#\n");
printf("# capabilities:\n");
status = uct_iface_query(iface, &iface_attr);
if (status != UCS_OK) {
printf("# < failed to query interface >\n");
} else {
printf("# bandwidth: %-.2f MB/sec\n", iface_attr.bandwidth / (1024 * 1024));
printf("# latency: %-.0f nsec\n", iface_attr.latency * 1e9);
printf("# overhead: %-.0f nsec\n", iface_attr.overhead * 1e9);
PRINT_CAP(PUT_SHORT, iface_attr.cap.flags, iface_attr.cap.put.max_short);
PRINT_CAP(PUT_BCOPY, iface_attr.cap.flags, iface_attr.cap.put.max_bcopy);
PRINT_CAP(PUT_ZCOPY, iface_attr.cap.flags, iface_attr.cap.put.max_zcopy);
PRINT_CAP(GET_BCOPY, iface_attr.cap.flags, iface_attr.cap.get.max_bcopy);
PRINT_CAP(GET_ZCOPY, iface_attr.cap.flags, iface_attr.cap.get.max_zcopy);
PRINT_CAP(AM_SHORT, iface_attr.cap.flags, iface_attr.cap.am.max_short);
PRINT_CAP(AM_BCOPY, iface_attr.cap.flags, iface_attr.cap.am.max_bcopy);
PRINT_CAP(AM_ZCOPY, iface_attr.cap.flags, iface_attr.cap.am.max_zcopy);
if (iface_attr.cap.flags & (UCT_IFACE_FLAG_AM_BCOPY|UCT_IFACE_FLAG_AM_ZCOPY)) {
printf("# am header: %s\n",
size_limit_to_str(iface_attr.cap.am.max_hdr));
}
PRINT_ATOMIC_CAP(ATOMIC_ADD, iface_attr.cap.flags);
PRINT_ATOMIC_CAP(ATOMIC_FADD, iface_attr.cap.flags);
PRINT_ATOMIC_CAP(ATOMIC_SWAP, iface_attr.cap.flags);
PRINT_ATOMIC_CAP(ATOMIC_CSWAP, iface_attr.cap.flags);
buf[0] = '\0';
if (iface_attr.cap.flags & (UCT_IFACE_FLAG_CONNECT_TO_EP |
UCT_IFACE_FLAG_CONNECT_TO_IFACE))
{
if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) {
strncat(buf, " to ep,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) {
strncat(buf, " to iface,", sizeof(buf) - 1);
}
buf[strlen(buf) - 1] = '\0';
} else {
strncat(buf, " none", sizeof(buf) - 1);
}
printf("# connection:%s\n", buf);
printf("# device address: %zu bytes\n", iface_attr.device_addr_len);
if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) {
printf("# iface address: %zu bytes\n", iface_attr.iface_addr_len);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) {
printf("# ep address: %zu bytes\n", iface_attr.ep_addr_len);
}
buf[0] = '\0';
if (iface_attr.cap.flags & (UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF |
UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF |
UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF |
UCT_IFACE_FLAG_ERRHANDLE_AM_ID |
UCT_IFACE_FLAG_ERRHANDLE_REMOTE_MEM |
UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE))
{
if (iface_attr.cap.flags & (UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF |
UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF |
UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF))
{
strncat(buf, " buffer (", sizeof(buf) - 1);
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF) {
strncat(buf, "short,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF) {
strncat(buf, "bcopy,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF) {
strncat(buf, "zcopy,", sizeof(buf) - 1);
}
buf[strlen(buf) - 1] = '\0';
strncat(buf, "),", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_AM_ID) {
strncat(buf, " active-message id,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_REMOTE_MEM) {
strncat(buf, " remote access,", sizeof(buf) - 1);
}
if (iface_attr.cap.flags & UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE) {
strncat(buf, " peer failure,", sizeof(buf) - 1);
}
buf[strlen(buf) - 1] = '\0';
} else {
strncat(buf, " none", sizeof(buf) - 1);
}
printf("# error handling:%s\n", buf);
}
uct_iface_close(iface);
printf("#\n");
}
static ucs_status_t print_tl_info(uct_md_h md, const char *tl_name,
uct_tl_resource_desc_t *resources,
unsigned num_resources,
int print_opts,
ucs_config_print_flags_t print_flags)
{
ucs_async_context_t async;
uct_worker_h worker;
ucs_status_t status;
unsigned i;
status = ucs_async_context_init(&async, UCS_ASYNC_MODE_THREAD);
if (status != UCS_OK) {
return status;
}
/* coverity[alloc_arg] */
status = uct_worker_create(&async, UCS_THREAD_MODE_MULTI, &worker);
if (status != UCS_OK) {
goto out;
}
printf("#\n");
printf("# Transport: %s\n", tl_name);
printf("#\n");
if (num_resources == 0) {
printf("# (No supported devices found)\n");
}
for (i = 0; i < num_resources; ++i) {
ucs_assert(!strcmp(tl_name, resources[i].tl_name));
print_iface_info(worker, md, &resources[i]);
}
uct_worker_destroy(worker);
out:
ucs_async_context_cleanup(&async);
return status;
}
