int mca_pml_ucx_open(void) { ucp_params_t params; ucp_config_t *config; ucs_status_t status; PML_UCX_VERBOSE(1, "mca_pml_ucx_open"); /* Read options */ status = ucp_config_read("MPI", NULL, &config); if (UCS_OK != status) { return OMPI_ERROR; } params.features = UCP_FEATURE_TAG; params.request_size = sizeof(ompi_request_t); params.request_init = mca_pml_ucx_request_init; params.request_cleanup = mca_pml_ucx_request_cleanup; status = ucp_init(¶ms, config, &ompi_pml_ucx.ucp_context); ucp_config_release(config); if (UCS_OK != status) { return OMPI_ERROR; } return OMPI_SUCCESS; }
static ucs_status_t ucp_perf_setup(ucx_perf_context_t *perf, ucx_perf_params_t *params) { ucp_params_t ucp_params; ucp_config_t *config; ucs_status_t status; uint64_t features; status = ucp_perf_test_check_params(params, &features); if (status != UCS_OK) { goto err; } status = ucp_config_read(NULL, NULL, &config); if (status != UCS_OK) { goto err; } ucp_params.features = features; ucp_params.request_size = 0; ucp_params.request_init = NULL; ucp_params.request_cleanup = NULL; status = ucp_init(&ucp_params, config, &perf->ucp.context); ucp_config_release(config); if (status != UCS_OK) { goto err; } status = ucp_worker_create(perf->ucp.context, params->thread_mode, &perf->ucp.worker); if (status != UCS_OK) { goto err_cleanup; } status = ucp_perf_test_alloc_mem(perf, params); if (status != UCS_OK) { ucs_warn("ucp test failed to alocate memory"); goto err_destroy_worker; } status = ucp_perf_test_setup_endpoints(perf, features); if (status != UCS_OK) { if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) { ucs_error("Failed to setup endpoints: %s", ucs_status_string(status)); } goto err_free_mem; } return UCS_OK; err_free_mem: ucp_perf_test_free_mem(perf); err_destroy_worker: ucp_worker_destroy(perf->ucp.worker); err_cleanup: ucp_cleanup(perf->ucp.context); err: return status; }
void print_ucp_config(ucs_config_print_flags_t print_flags) { ucp_config_t *config; ucs_status_t status; status = ucp_config_read(NULL, NULL, &config); if (status != UCS_OK) { printf("<Failed to read UCP configuraton>\n"); return; } ucp_config_print(config, stdout, "protocol layer configuration", print_flags); ucp_config_release(config); }
static int mca_spml_ucx_component_open(void) { ucs_status_t err; ucp_config_t *ucp_config; ucp_params_t params; err = ucp_config_read("OSHMEM", NULL, &ucp_config); if (UCS_OK != err) { return OSHMEM_ERROR; } memset(¶ms, 0, sizeof(params)); params.field_mask = UCP_PARAM_FIELD_FEATURES|UCP_PARAM_FIELD_ESTIMATED_NUM_EPS; params.features = UCP_FEATURE_RMA|UCP_FEATURE_AMO32|UCP_FEATURE_AMO64; params.estimated_num_eps = ompi_proc_world_size(); err = ucp_init(¶ms, ucp_config, &mca_spml_ucx.ucp_context); ucp_config_release(ucp_config); if (UCS_OK != err) { return OSHMEM_ERROR; } return OSHMEM_SUCCESS; }
static int mca_spml_ucx_component_open(void) { ucs_status_t err; ucp_config_t *ucp_config; ucp_params_t params; err = ucp_config_read("OSHMEM", NULL, &ucp_config); if (UCS_OK != err) { return OSHMEM_ERROR; } memset(¶ms, 0, sizeof(params)); params.field_mask = UCP_PARAM_FIELD_FEATURES; params.features = UCP_FEATURE_RMA|UCP_FEATURE_AMO32|UCP_FEATURE_AMO64|UCP_PARAM_FIELD_MT_WORKERS_SHARED; params.mt_workers_shared = 0; err = ucp_init(¶ms, ucp_config, &mca_spml_ucx.ucp_context); ucp_config_release(ucp_config); if (UCS_OK != err) { return OSHMEM_ERROR; } return OSHMEM_SUCCESS; }
static ucs_status_t ucp_perf_setup(ucx_perf_context_t *perf, ucx_perf_params_t *params) { ucp_params_t ucp_params; ucp_worker_params_t worker_params; ucp_config_t *config; ucs_status_t status; ucp_params.field_mask = UCP_PARAM_FIELD_FEATURES; ucp_params.features = 0; status = ucp_perf_test_fill_params(params, &ucp_params); if (status != UCS_OK) { goto err; } status = ucp_config_read(NULL, NULL, &config); if (status != UCS_OK) { goto err; } status = ucp_init(&ucp_params, config, &perf->ucp.context); ucp_config_release(config); if (status != UCS_OK) { goto err; } worker_params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; worker_params.thread_mode = params->thread_mode; status = ucp_worker_create(perf->ucp.context, &worker_params, &perf->ucp.worker); if (status != UCS_OK) { goto err_cleanup; } status = ucp_perf_test_alloc_mem(perf, params); if (status != UCS_OK) { ucs_warn("ucp test failed to alocate memory"); goto err_destroy_worker; } status = ucp_perf_test_setup_endpoints(perf, ucp_params.features); if (status != UCS_OK) { if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) { ucs_error("Failed to setup endpoints: %s", ucs_status_string(status)); } goto err_free_mem; } return UCS_OK; err_free_mem: ucp_perf_test_free_mem(perf); err_destroy_worker: ucp_worker_destroy(perf->ucp.worker); err_cleanup: ucp_cleanup(perf->ucp.context); err: return status; }
void print_ucp_info(int print_opts, ucs_config_print_flags_t print_flags, uint64_t features) { ucp_config_t *config; ucs_status_t status; ucp_context_h context; ucp_worker_h worker; ucp_params_t params; ucp_worker_params_t worker_params; ucp_ep_params_t ep_params; ucp_address_t *address; size_t address_length; ucp_ep_h ep; status = ucp_config_read(NULL, NULL, &config); if (status != UCS_OK) { return; } memset(¶ms, 0, sizeof(params)); params.features = features; status = ucp_init(¶ms, config, &context); if (status != UCS_OK) { printf("<Failed to create UCP context>\n"); goto out_release_config; } if (print_opts & PRINT_UCP_CONTEXT) { ucp_context_print_info(context, stdout); } if (!(print_opts & (PRINT_UCP_WORKER|PRINT_UCP_EP))) { goto out_cleanup_context; } worker_params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; worker_params.thread_mode = UCS_THREAD_MODE_MULTI; status = ucp_worker_create(context, &worker_params, &worker); if (status != UCS_OK) { printf("<Failed to create UCP worker>\n"); goto out_cleanup_context; } if (print_opts & PRINT_UCP_WORKER) { ucp_worker_print_info(worker, stdout); } if (print_opts & PRINT_UCP_EP) { status = ucp_worker_get_address(worker, &address, &address_length); if (status != UCS_OK) { printf("<Failed to get UCP worker address>\n"); goto out_destroy_worker; } ep_params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS; ep_params.address = address; status = ucp_ep_create(worker, &ep_params, &ep); ucp_worker_release_address(worker, address); if (status != UCS_OK) { printf("<Failed to get create UCP endpoint>\n"); goto out_destroy_worker; } ucp_ep_print_info(ep, stdout); ucp_ep_destroy(ep); } out_destroy_worker: ucp_worker_destroy(worker); out_cleanup_context: ucp_cleanup(context); out_release_config: ucp_config_release(config); }
int main(int argc, char **argv) { /* UCP temporary vars */ ucp_params_t ucp_params; ucp_config_t *config; ucs_status_t status; /* UCP handler objects */ ucp_context_h ucp_context; ucp_worker_h ucp_worker; /* OOB connection vars */ uint64_t addr_len = 0; char *server = NULL; int oob_sock = -1; int ret = -1; /* Parse the command line */ if (parse_cmd(argc, argv, &server) != UCS_OK) { goto err; } /* UCP initialization */ status = ucp_config_read(NULL, NULL, &config); if (status != UCS_OK) { goto err; } ucp_params.features = UCP_FEATURE_TAG; if (ucp_test_mode == TEST_MODE_WAIT || ucp_test_mode == TEST_MODE_EVENTFD) { ucp_params.features |= UCP_FEATURE_WAKEUP; } ucp_params.request_size = sizeof(struct ucx_context); ucp_params.request_init = request_init; ucp_params.request_cleanup = NULL; status = ucp_init(&ucp_params, config, &ucp_context); ucp_config_print(config, stdout, NULL, UCS_CONFIG_PRINT_CONFIG); ucp_config_release(config); if (status != UCS_OK) { goto err; } status = ucp_worker_create(ucp_context, UCS_THREAD_MODE_SINGLE, &ucp_worker); if (status != UCS_OK) { goto err_cleanup; } status = ucp_worker_get_address(ucp_worker, &local_addr, &local_addr_len); if (status != UCS_OK) { goto err_worker; } printf("[0x%x] local address length: %zu\n", (unsigned int)pthread_self(), local_addr_len); /* OOB connection establishment */ if (server) { peer_addr_len = local_addr_len; oob_sock = run_client(server); if (oob_sock < 0) { goto err_addr; } ret = recv(oob_sock, &addr_len, sizeof(addr_len), 0); if (ret < 0) { fprintf(stderr, "failed to receive address length\n"); goto err_addr; } peer_addr_len = addr_len; peer_addr = malloc(peer_addr_len); if (!peer_addr) { fprintf(stderr, "unable to allocate memory\n"); goto err_addr; } ret = recv(oob_sock, peer_addr, peer_addr_len, 0); if (ret < 0) { fprintf(stderr, "failed to receive address\n"); goto err_peer_addr; } } else { oob_sock = run_server(); if (oob_sock < 0) { goto err_peer_addr; } addr_len = local_addr_len; ret = send(oob_sock, &addr_len, sizeof(addr_len), 0); if (ret < 0 || ret != sizeof(addr_len)) { fprintf(stderr, "failed to send address length\n"); goto err_peer_addr; } ret = send(oob_sock, local_addr, local_addr_len, 0); if (ret < 0 || ret != local_addr_len) { fprintf(stderr, "failed to send address\n"); goto err_peer_addr; } } close(oob_sock); ret = run_test(server, ucp_worker); err_peer_addr: free(peer_addr); err_addr: ucp_worker_release_address(ucp_worker, local_addr); err_worker: ucp_worker_destroy(ucp_worker); err_cleanup: ucp_cleanup(ucp_context); err: return ret; }
void print_ucp_info(int print_opts, ucs_config_print_flags_t print_flags, uint64_t ctx_features, const ucp_ep_params_t *base_ep_params, size_t estimated_num_eps, unsigned dev_type_bitmap, const char *mem_size) { ucp_config_t *config; ucs_status_t status; ucs_status_ptr_t status_ptr; ucp_context_h context; ucp_worker_h worker; ucp_params_t params; ucp_worker_params_t worker_params; ucp_ep_params_t ep_params; ucp_address_t *address; size_t address_length; resource_usage_t usage; ucp_ep_h ep; status = ucp_config_read(NULL, NULL, &config); if (status != UCS_OK) { return; } memset(¶ms, 0, sizeof(params)); params.field_mask = UCP_PARAM_FIELD_FEATURES | UCP_PARAM_FIELD_ESTIMATED_NUM_EPS; params.features = ctx_features; params.estimated_num_eps = estimated_num_eps; get_resource_usage(&usage); if (!(dev_type_bitmap & UCS_BIT(UCT_DEVICE_TYPE_SELF))) { ucp_config_modify(config, "SELF_DEVICES", ""); } if (!(dev_type_bitmap & UCS_BIT(UCT_DEVICE_TYPE_SHM))) { ucp_config_modify(config, "SHM_DEVICES", ""); } if (!(dev_type_bitmap & UCS_BIT(UCT_DEVICE_TYPE_NET))) { ucp_config_modify(config, "NET_DEVICES", ""); } status = ucp_init(¶ms, config, &context); if (status != UCS_OK) { printf("<Failed to create UCP context>\n"); goto out_release_config; } if ((print_opts & PRINT_MEM_MAP) && (mem_size != NULL)) { ucp_mem_print_info(mem_size, context, stdout); } if (print_opts & PRINT_UCP_CONTEXT) { ucp_context_print_info(context, stdout); print_resource_usage(&usage, "UCP context"); } if (!(print_opts & (PRINT_UCP_WORKER|PRINT_UCP_EP))) { goto out_cleanup_context; } worker_params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; worker_params.thread_mode = UCS_THREAD_MODE_MULTI; get_resource_usage(&usage); status = ucp_worker_create(context, &worker_params, &worker); if (status != UCS_OK) { printf("<Failed to create UCP worker>\n"); goto out_cleanup_context; } if (print_opts & PRINT_UCP_WORKER) { ucp_worker_print_info(worker, stdout); print_resource_usage(&usage, "UCP worker"); } if (print_opts & PRINT_UCP_EP) { status = ucp_worker_get_address(worker, &address, &address_length); if (status != UCS_OK) { printf("<Failed to get UCP worker address>\n"); goto out_destroy_worker; } ep_params = *base_ep_params; ep_params.field_mask |= UCP_EP_PARAM_FIELD_REMOTE_ADDRESS; ep_params.address = address; status = ucp_ep_create(worker, &ep_params, &ep); ucp_worker_release_address(worker, address); if (status != UCS_OK) { printf("<Failed to create UCP endpoint>\n"); goto out_destroy_worker; } ucp_ep_print_info(ep, stdout); status_ptr = ucp_disconnect_nb(ep); if (UCS_PTR_IS_PTR(status_ptr)) { do { ucp_worker_progress(worker); status = ucp_request_test(status_ptr, NULL); } while (status == UCS_INPROGRESS); ucp_request_release(status_ptr); } } out_destroy_worker: ucp_worker_destroy(worker); out_cleanup_context: ucp_cleanup(context); out_release_config: ucp_config_release(config); }
static int mlx_getinfo ( uint32_t version, const char *node, const char *service, uint64_t flags, const struct fi_info *hints, struct fi_info **info) { int status = -ENODATA; char *configfile_name = NULL; int inject_thresh = -1; mlx_descriptor.config = NULL; status = fi_param_get( &mlx_prov, "mlx_tinject_limit", &inject_thresh); if (!status) inject_thresh = FI_MLX_DEFAULT_INJECT_SIZE; FI_INFO( &mlx_prov, FI_LOG_CORE, "used inlect size = %d \n", inject_thresh); status = fi_param_get( &mlx_prov, "mlx_config", &configfile_name); if (!status) { configfile_name = NULL; } /* NS is disabled by default */ status = fi_param_get( &mlx_prov, "mlx_ns_enable", &mlx_descriptor.use_ns); if (!status) { mlx_descriptor.use_ns = 0; } status = fi_param_get( &mlx_prov, "mlx_ns_port", &mlx_descriptor.ns_port); if (!status) { mlx_descriptor.ns_port = FI_MLX_DEFAULT_NS_PORT; } status = ucp_config_read( NULL, status? NULL: configfile_name, &mlx_descriptor.config); if (status != UCS_OK) { FI_WARN( &mlx_prov, FI_LOG_CORE, "MLX error: invalid config file\n\t%d (%s)\n", status, ucs_status_string(status)); } /*Setup some presets*/ status = ucm_config_modify("MALLOC_HOOKS", "no"); if (status != UCS_OK) { FI_WARN( &mlx_prov, FI_LOG_CORE, "MLX error: failed to switch off UCM memory hooks:\t%d (%s)\n", status, ucs_status_string(status)); } FI_INFO( &mlx_prov, FI_LOG_CORE, "Loaded MLX version %s\n", ucp_get_version_string()); #if ENABLE_DEBUG if (mlx_descriptor.config && fi_log_enabled( &mlx_prov, FI_LOG_INFO, FI_LOG_CORE)) { ucp_config_print( mlx_descriptor.config, stderr, "Used MLX configuration", (1<<4)-1); } #endif *info = NULL; if (node || service) { FI_WARN(&mlx_prov, FI_LOG_CORE, "fi_getinfo with \"node != NULL \" or \"service != NULL \" is temporary not supported\n"); node = service = NULL; flags = 0; } /* Only Pure MLX address and IPv4 are supported */ if (hints->addr_format == FI_ADDR_MLX) { mlx_info.addr_format = FI_ADDR_MLX; } if (hints->addr_format <= FI_SOCKADDR_IN) { mlx_descriptor.use_ns = 1; mlx_info.addr_format = FI_SOCKADDR_IN; } status = util_getinfo( &mlx_util_prov, version, service, node, flags, hints, info); return status; }
static int component_init(bool enable_progress_threads, bool enable_mpi_threads) { ucp_config_t *config = NULL; ucp_params_t context_params; bool progress_registered = false, requests_created = false; int ret = OMPI_SUCCESS; ucs_status_t status; mca_osc_ucx_component.ucp_context = NULL; mca_osc_ucx_component.ucp_worker = NULL; mca_osc_ucx_component.enable_mpi_threads = enable_mpi_threads; status = ucp_config_read("MPI", NULL, &config); if (UCS_OK != status) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: ucp_config_read failed: %d\n", __FILE__, __LINE__, status); return OMPI_ERROR; } OBJ_CONSTRUCT(&mca_osc_ucx_component.requests, opal_free_list_t); requests_created = true; ret = opal_free_list_init (&mca_osc_ucx_component.requests, sizeof(ompi_osc_ucx_request_t), opal_cache_line_size, OBJ_CLASS(ompi_osc_ucx_request_t), 0, 0, 8, 0, 8, NULL, 0, NULL, NULL, NULL); if (OMPI_SUCCESS != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: opal_free_list_init failed: %d\n", __FILE__, __LINE__, ret); goto error; } mca_osc_ucx_component.num_incomplete_req_ops = 0; ret = opal_progress_register(progress_callback); progress_registered = true; if (OMPI_SUCCESS != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: opal_progress_register failed: %d\n", __FILE__, __LINE__, ret); goto error; } /* initialize UCP context */ memset(&context_params, 0, sizeof(ucp_context_h)); context_params.field_mask = UCP_PARAM_FIELD_FEATURES | UCP_PARAM_FIELD_MT_WORKERS_SHARED | UCP_PARAM_FIELD_ESTIMATED_NUM_EPS | UCP_PARAM_FIELD_REQUEST_INIT | UCP_PARAM_FIELD_REQUEST_SIZE; context_params.features = UCP_FEATURE_RMA | UCP_FEATURE_AMO32 | UCP_FEATURE_AMO64; context_params.mt_workers_shared = 0; context_params.estimated_num_eps = ompi_proc_world_size(); context_params.request_init = internal_req_init; context_params.request_size = sizeof(ompi_osc_ucx_internal_request_t); status = ucp_init(&context_params, config, &mca_osc_ucx_component.ucp_context); ucp_config_release(config); if (UCS_OK != status) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: ucp_init failed: %d\n", __FILE__, __LINE__, status); ret = OMPI_ERROR; goto error; } return ret; error: if (progress_registered) opal_progress_unregister(progress_callback); if (requests_created) OBJ_DESTRUCT(&mca_osc_ucx_component.requests); if (mca_osc_ucx_component.ucp_context) ucp_cleanup(mca_osc_ucx_component.ucp_context); return ret; }