ucs_status_t uct_iface_mem_alloc(uct_iface_h tl_iface, size_t length, const char *name, uct_allocated_memory_t *mem) { uct_base_iface_t *iface = ucs_derived_of(tl_iface, uct_base_iface_t); uct_pd_attr_t pd_attr; ucs_status_t status; status = uct_mem_alloc(length, iface->config.alloc_methods, iface->config.num_alloc_methods, &iface->pd, 1, name, mem); if (status != UCS_OK) { goto err; } /* If the memory was not allocated using PD, register it */ if (mem->method != UCT_ALLOC_METHOD_PD) { status = uct_pd_query(iface->pd, &pd_attr); if (status != UCS_OK) { goto err_free; } /* If PD does not support registration, allow only the PD method */ if (!(pd_attr.cap.flags & UCT_PD_FLAG_REG)) { ucs_error("%s pd does not supprt registration, so cannot use any allocation " "method except 'pd'", iface->pd->component->name); status = UCS_ERR_NO_MEMORY; goto err_free; } status = uct_pd_mem_reg(iface->pd, mem->address, mem->length, &mem->memh); if (status != UCS_OK) { goto err_free; } mem->pd = iface->pd; } return UCS_OK; err_free: uct_mem_free(mem); err: return status; }
ucs_status_t uct_mem_alloc(size_t min_length, uct_alloc_method_t *methods, unsigned num_methods, uct_pd_h *pds, unsigned num_pds, const char *alloc_name, uct_allocated_memory_t *mem) { uct_alloc_method_t *method; uct_pd_attr_t pd_attr; ucs_status_t status; size_t alloc_length; unsigned pd_index; uct_mem_h memh; uct_pd_h pd; void *address; int shmid; if (min_length == 0) { ucs_error("Allocation length cannot be 0"); return UCS_ERR_INVALID_PARAM; } if (num_methods == 0) { ucs_error("No allocation methods provided"); return UCS_ERR_INVALID_PARAM; } for (method = methods; method < methods + num_methods; ++method) { ucs_debug("trying allocation method %s", uct_alloc_method_names[*method]); switch (*method) { case UCT_ALLOC_METHOD_PD: /* Allocate with one of the specified protection domains */ for (pd_index = 0; pd_index < num_pds; ++pd_index) { pd = pds[pd_index]; status = uct_pd_query(pd, &pd_attr); if (status != UCS_OK) { ucs_error("Failed to query PD"); return status; } /* Check if PD supports allocation */ if (!(pd_attr.cap.flags & UCT_PD_FLAG_ALLOC)) { continue; } /* Allocate memory using the PD. * If the allocation fails, it's considered an error and we don't * fall-back, because this PD already exposed support for memory * allocation. */ alloc_length = min_length; status = uct_pd_mem_alloc(pd, &alloc_length, &address, alloc_name, &memh); if (status != UCS_OK) { ucs_error("failed to allocate %zu bytes using pd %s: %s", alloc_length, pd->component->name, ucs_status_string(status)); return status; } mem->pd = pd; mem->memh = memh; goto allocated; } break; case UCT_ALLOC_METHOD_HEAP: /* Allocate aligned memory using libc allocator */ alloc_length = min_length; address = ucs_memalign(UCS_SYS_CACHE_LINE_SIZE, alloc_length UCS_MEMTRACK_VAL); if (address != NULL) { goto allocated_without_pd; } ucs_debug("failed to allocate %zu bytes from the heap", alloc_length); break; case UCT_ALLOC_METHOD_MMAP: /* Request memory from operating system using mmap() */ alloc_length = ucs_align_up_pow2(min_length, ucs_get_page_size()); address = ucs_mmap(NULL, alloc_length, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0 UCS_MEMTRACK_VAL); if (address != MAP_FAILED) { goto allocated_without_pd; } ucs_debug("failed to mmap %zu bytes: %m", alloc_length); break; case UCT_ALLOC_METHOD_HUGE: /* Allocate huge pages */ alloc_length = min_length; status = ucs_sysv_alloc(&alloc_length, &address, SHM_HUGETLB, &shmid UCS_MEMTRACK_VAL); if (status == UCS_OK) { goto allocated_without_pd; } ucs_debug("failed to allocate %zu bytes from hugetlb: %s", min_length, ucs_status_string(status)); break; default: ucs_error("Invalid allocation method %d", *method); return UCS_ERR_INVALID_PARAM; } } ucs_debug("Could not allocate memory with any of the provided methods"); return UCS_ERR_NO_MEMORY; allocated_without_pd: mem->pd = NULL; mem->memh = UCT_INVALID_MEM_HANDLE; allocated: ucs_debug("allocated %zu bytes at %p using %s", alloc_length, address, (mem->pd == NULL) ? uct_alloc_method_names[*method] : mem->pd->component->name); mem->address = address; mem->length = alloc_length; mem->method = *method; return UCS_OK; }
static ucs_status_t ucp_fill_resources(ucp_context_h context, const ucp_config_t *config) { unsigned num_tl_resources; unsigned num_pd_resources; uct_pd_resource_desc_t *pd_rscs; ucs_status_t status; ucp_rsc_index_t i; unsigned pd_index; uct_pd_h pd; /* if we got here then num_resources > 0. * if the user's device list is empty, there is no match */ if (0 == config->devices.count) { ucs_error("The device list is empty. Please specify the devices you would like to use " "or omit the UCX_DEVICES so that the default will be used."); status = UCS_ERR_NO_ELEM; goto err; } /* if we got here then num_resources > 0. * if the user's tls list is empty, there is no match */ if (0 == config->tls.count) { ucs_error("The TLs list is empty. Please specify the transports you would like to use " "or omit the UCX_TLS so that the default will be used."); status = UCS_ERR_NO_ELEM; goto err; } /* List protection domain resources */ status = uct_query_pd_resources(&pd_rscs, &num_pd_resources); if (status != UCS_OK) { goto err; } /* Error check: Make sure there is at least one PD */ if (num_pd_resources == 0) { ucs_error("No pd resources found"); status = UCS_ERR_NO_DEVICE; goto err_release_pd_resources; } if (num_pd_resources >= UCP_MAX_PDS) { ucs_error("Only up to %ld PDs are supported", UCP_MAX_PDS); status = UCS_ERR_EXCEEDS_LIMIT; goto err_release_pd_resources; } context->num_pds = 0; context->pd_rscs = NULL; context->pds = NULL; context->pd_attrs = NULL; context->num_tls = 0; context->tl_rscs = NULL; /* Allocate array of PD resources we would actually use */ context->pd_rscs = ucs_calloc(num_pd_resources, sizeof(*context->pd_rscs), "ucp_pd_resources"); if (context->pd_rscs == NULL) { status = UCS_ERR_NO_MEMORY; goto err_free_context_resources; } /* Allocate array of protection domains */ context->pds = ucs_calloc(num_pd_resources, sizeof(*context->pds), "ucp_pds"); if (context->pds == NULL) { status = UCS_ERR_NO_MEMORY; goto err_free_context_resources; } /* Allocate array of protection domains attributes */ context->pd_attrs = ucs_calloc(num_pd_resources, sizeof(*context->pd_attrs), "ucp_pd_attrs"); if (context->pd_attrs == NULL) { status = UCS_ERR_NO_MEMORY; goto err_free_context_resources; } /* Open all protection domains, keep only those which have at least one TL * resources selected on them. */ pd_index = 0; for (i = 0; i < num_pd_resources; ++i) { status = uct_pd_open(pd_rscs[i].pd_name, &pd); if (status != UCS_OK) { goto err_free_context_resources; } context->pd_rscs[pd_index] = pd_rscs[i]; context->pds[pd_index] = pd; /* Save PD attributes */ status = uct_pd_query(pd, &context->pd_attrs[pd_index]); if (status != UCS_OK) { goto err_free_context_resources; } /* Add communication resources of each PD */ status = ucp_add_tl_resources(context, pd, pd_index, config, &num_tl_resources); if (status != UCS_OK) { goto err_free_context_resources; } /* If the PD does not have transport resources, don't use it */ if (num_tl_resources > 0) { ++pd_index; ++context->num_pds; } else { ucs_debug("closing pd %s because it has no selected transport resources", pd_rscs[i].pd_name); uct_pd_close(pd); } } /* Error check: Make sure there is at least one transport */ if (0 == context->num_tls) { ucs_error("There are no available resources matching the configured criteria"); status = UCS_ERR_NO_DEVICE; goto err_free_context_resources; } /* Error check: Make sure there are no too many transports */ if (context->num_tls >= UCP_MAX_TLS) { ucs_error("Exceeded resources limit (%u requested, up to %d are supported)", context->num_tls, UCP_MAX_TLS); status = UCS_ERR_EXCEEDS_LIMIT; goto err_free_context_resources; } uct_release_pd_resource_list(pd_rscs); return UCS_OK; err_free_context_resources: ucp_free_resources(context); err_release_pd_resources: uct_release_pd_resource_list(pd_rscs); err: return status; }
static ucs_status_t uct_perf_test_setup_endpoints(ucx_perf_context_t *perf) { unsigned group_size, i, group_index; uct_device_addr_t *dev_addr; uct_iface_addr_t *iface_addr; uct_ep_addr_t *ep_addr; uct_iface_attr_t iface_attr; uct_pd_attr_t pd_attr; unsigned long va; void *rkey_buffer; ucs_status_t status; struct iovec vec[5]; void *req; status = uct_iface_query(perf->uct.iface, &iface_attr); if (status != UCS_OK) { ucs_error("Failed to uct_iface_query: %s", ucs_status_string(status)); goto err; } status = uct_pd_query(perf->uct.pd, &pd_attr); if (status != UCS_OK) { ucs_error("Failed to uct_pd_query: %s", ucs_status_string(status)); goto err; } dev_addr = calloc(1, iface_attr.device_addr_len); iface_addr = calloc(1, iface_attr.iface_addr_len); ep_addr = calloc(1, iface_attr.ep_addr_len); rkey_buffer = calloc(1, pd_attr.rkey_packed_size); if ((iface_addr == NULL) || (ep_addr == NULL) || (rkey_buffer == NULL)) { goto err_free; } status = uct_iface_get_device_address(perf->uct.iface, dev_addr); if (status != UCS_OK) { ucs_error("Failed to uct_iface_get_device_address: %s", ucs_status_string(status)); goto err_free; } if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) { status = uct_iface_get_address(perf->uct.iface, iface_addr); if (status != UCS_OK) { ucs_error("Failed to uct_iface_get_address: %s", ucs_status_string(status)); goto err_free; } } status = uct_pd_mkey_pack(perf->uct.pd, perf->uct.recv_mem.memh, rkey_buffer); if (status != UCS_OK) { ucs_error("Failed to uct_rkey_pack: %s", ucs_status_string(status)); goto err_free; } group_size = rte_call(perf, group_size); group_index = rte_call(perf, group_index); perf->uct.peers = calloc(group_size, sizeof(*perf->uct.peers)); if (perf->uct.peers == NULL) { goto err_free; } if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) { for (i = 0; i < group_size; ++i) { if (i == group_index) { continue; } status = uct_ep_create(perf->uct.iface, &perf->uct.peers[i].ep); if (status != UCS_OK) { ucs_error("Failed to uct_ep_create: %s", ucs_status_string(status)); goto err_destroy_eps; } status = uct_ep_get_address(perf->uct.peers[i].ep, ep_addr); if (status != UCS_OK) { ucs_error("Failed to uct_ep_get_address: %s", ucs_status_string(status)); goto err_destroy_eps; } } } va = (uintptr_t)perf->recv_buffer; vec[0].iov_base = &va; vec[0].iov_len = sizeof(va); vec[1].iov_base = rkey_buffer; vec[1].iov_len = pd_attr.rkey_packed_size; vec[2].iov_base = dev_addr; vec[2].iov_len = iface_attr.device_addr_len; vec[3].iov_base = iface_addr; vec[3].iov_len = iface_attr.iface_addr_len; vec[4].iov_base = ep_addr; vec[4].iov_len = iface_attr.ep_addr_len; rte_call(perf, post_vec, vec, 5, &req); rte_call(perf, exchange_vec, req); for (i = 0; i < group_size; ++i) { if (i == group_index) { continue; } vec[0].iov_base = &va; vec[0].iov_len = sizeof(va); vec[1].iov_base = rkey_buffer; vec[1].iov_len = pd_attr.rkey_packed_size; vec[2].iov_base = dev_addr; vec[2].iov_len = iface_attr.device_addr_len; vec[3].iov_base = iface_addr; vec[3].iov_len = iface_attr.iface_addr_len; vec[4].iov_base = ep_addr; vec[4].iov_len = iface_attr.ep_addr_len; rte_call(perf, recv_vec, i, vec, 5, req); perf->uct.peers[i].remote_addr = va; status = uct_rkey_unpack(rkey_buffer, &perf->uct.peers[i].rkey); if (status != UCS_OK) { ucs_error("Failed to uct_rkey_unpack: %s", ucs_status_string(status)); return status; } if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) { status = uct_ep_connect_to_ep(perf->uct.peers[i].ep, dev_addr, ep_addr); } else if (iface_attr.cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) { status = uct_ep_create_connected(perf->uct.iface, dev_addr, iface_addr, &perf->uct.peers[i].ep); } else { status = UCS_ERR_UNSUPPORTED; } if (status != UCS_OK) { ucs_error("Failed to connect endpoint: %s", ucs_status_string(status)); goto err_destroy_eps; } } uct_perf_iface_flush_b(perf); rte_call(perf, barrier); free(ep_addr); free(iface_addr); free(dev_addr); free(rkey_buffer); return UCS_OK; err_destroy_eps: for (i = 0; i < group_size; ++i) { if (perf->uct.peers[i].rkey.type != NULL) { uct_rkey_release(&perf->uct.peers[i].rkey); } if (perf->uct.peers[i].ep != NULL) { uct_ep_destroy(perf->uct.peers[i].ep); } } free(perf->uct.peers); err_free: free(ep_addr); free(iface_addr); free(dev_addr); free(rkey_buffer); err: return status; }