/* Should be called with lock held */
static void ucs_module_loader_add_dl_dir()
{
char *dlpath_dup = NULL;
size_t max_length;
Dl_info dl_info;
char *p, *path;
int ret;
(void)dlerror();
ret = dladdr((void*)&ucs_module_loader_state, &dl_info);
if (ret == 0) {
ucs_error("dladdr failed: %s", dlerror());
return;
}
ucs_module_debug("ucs library path: %s", dl_info.dli_fname);
/* copy extension */
dlpath_dup = ucs_strdup(dl_info.dli_fname,
UCS_MODULE_PATH_MEMTRACK_NAME);
if (dlpath_dup == NULL) {
return;
}
p = basename(dlpath_dup);
p = strchr(p, '.');
if (p != NULL) {
strncpy(ucs_module_loader_state.module_ext, p,
sizeof(ucs_module_loader_state.module_ext) - 1);
}
ucs_free(dlpath_dup);
/* copy directory component */
dlpath_dup = ucs_strdup(dl_info.dli_fname,
UCS_MODULE_PATH_MEMTRACK_NAME);
if (dlpath_dup == NULL) {
return;
}
/* construct module directory path */
max_length = strlen(dlpath_dup) + /* directory */
1 + /* '/' */
strlen(UCX_MODULE_SUBDIR) + /* sub-directory */
1; /* '\0' */
path = ucs_malloc(max_length, UCS_MODULE_PATH_MEMTRACK_NAME);
if (path == NULL) {
goto out;
}
snprintf(path, max_length, "%s/%s", dirname(dlpath_dup), UCX_MODULE_SUBDIR);
ucs_module_loader_state.srch_path[ucs_module_loader_state.srchpath_cnt++] = path;
out:
ucs_free(dlpath_dup);
}
static void ucp_free_resources(ucp_context_t *context)
{
ucp_rsc_index_t i;
ucs_free(context->tl_rscs);
for (i = 0; i < context->num_mds; ++i) {
if (context->mds[i] != NULL) {
uct_md_close(context->mds[i]);
}
}
ucs_free(context->md_attrs);
ucs_free(context->mds);
ucs_free(context->md_rscs);
}
ucs_status_t uct_mem_free(const uct_allocated_memory_t *mem)
{
int ret;
switch (mem->method) {
case UCT_ALLOC_METHOD_MD:
return uct_md_mem_free(mem->md, mem->memh);
case UCT_ALLOC_METHOD_HEAP:
ucs_free(mem->address);
return UCS_OK;
case UCT_ALLOC_METHOD_MMAP:
ret = ucs_munmap(mem->address, mem->length);
if (ret != 0) {
ucs_warn("munmap(address=%p, length=%zu) failed: %m", mem->address,
mem->length);
return UCS_ERR_INVALID_PARAM;
}
return UCS_OK;
case UCT_ALLOC_METHOD_HUGE:
return ucs_sysv_free(mem->address);
default:
ucs_warn("Invalid memory allocation method: %d", mem->method);
return UCS_ERR_INVALID_PARAM;
}
}
void uct_dc_ep_release(uct_dc_ep_t *ep)
{
ucs_assert_always(ep->state == UCT_DC_EP_INVALID);
ucs_debug("release dc_ep %p", ep);
ucs_list_del(&ep->list);
ucs_free(ep);
}
static ucs_status_t uct_gdr_copy_rkey_release(uct_md_component_t *mdc, uct_rkey_t rkey,
void *handle)
{
ucs_assert(NULL == handle);
ucs_free((void *)rkey);
return UCS_OK;
}
ucs_status_t ucs_config_clone_array(void *src, void *dest, const void *arg)
{
ucs_config_array_field_t *dest_array = dest, *src_array = src;
const ucs_config_array_t *array = arg;
ucs_status_t status;
unsigned i;
dest_array->data = ucs_calloc(src_array->count, array->elem_size,
"config array");
if (dest_array->data == NULL) {
return UCS_ERR_NO_MEMORY;
}
dest_array->count = src_array->count;
for (i = 0; i < src_array->count; ++i) {
status = array->parser.clone((char*)src_array->data + i * array->elem_size,
(char*)dest_array->data + i * array->elem_size,
array->parser.arg);
if (status != UCS_OK) {
ucs_free(dest_array->data);
return status;
}
}
return UCS_OK;
}
void ucp_cleanup(ucp_context_h context)
{
ucp_free_resources(context);
ucp_free_config(context);
UCP_THREAD_LOCK_FINALIZE(&context->mt_lock);
ucs_free(context);
}
static UCS_CLASS_CLEANUP_FUNC(uct_rdmacm_ep_t)
{
uct_rdmacm_iface_t *iface = ucs_derived_of(self->super.super.iface, uct_rdmacm_iface_t);
ucs_debug("rdmacm_ep %p: destroying", self);
UCS_ASYNC_BLOCK(iface->super.worker->async);
if (self->is_on_pending) {
ucs_list_del(&self->list_elem);
self->is_on_pending = 0;
}
/* remove the slow progress function in case it was placed on the slow progress
* chain but wasn't invoked yet */
uct_worker_progress_unregister_safe(&iface->super.worker->super,
&self->slow_prog_id);
/* if the destroyed ep is the active one on the iface, mark it as destroyed
* so that arriving events on the iface won't try to access this ep */
if (iface->ep == self) {
iface->ep = UCT_RDMACM_IFACE_BLOCKED_NO_EP;
}
UCS_ASYNC_UNBLOCK(iface->super.worker->async);
ucs_free(self->priv_data);
}
void ucp_cleanup(ucp_context_h context)
{
ucp_tag_cleanup(context);
ucp_free_resources(context);
ucp_free_config(context);
ucs_free(context);
}
ucs_status_t ucp_config_read(const char *env_prefix, const char *filename,
ucp_config_t **config_p)
{
ucp_config_t *config;
ucs_status_t status;
config = ucs_malloc(sizeof(*config), "ucp config");
if (config == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err;
}
status = ucs_config_parser_fill_opts(config, ucp_config_table, env_prefix,
NULL, 0);
if (status != UCS_OK) {
goto err_free;
}
*config_p = config;
return UCS_OK;
err_free:
ucs_free(config);
err:
return status;
}
void uct_dc_mlx5_ep_release(uct_dc_mlx5_ep_t *ep)
{
ucs_assert_always(!(ep->flags & UCT_DC_MLX5_EP_FLAG_VALID));
ucs_debug("release dc_mlx5_ep %p", ep);
ucs_list_del(&ep->list);
ucs_free(ep);
}
static ucs_status_t uct_gdr_copy_mem_reg(uct_md_h uct_md, void *address, size_t length,
unsigned flags, uct_mem_h *memh_p)
{
uct_gdr_copy_mem_t *mem_hndl = NULL;
size_t reg_size;
void *ptr;
ucs_status_t status;
mem_hndl = ucs_malloc(sizeof(uct_gdr_copy_mem_t), "gdr_copy handle");
if (NULL == mem_hndl) {
ucs_error("failed to allocate memory for gdr_copy_mem_t");
return UCS_ERR_NO_MEMORY;
}
reg_size = (length + GPU_PAGE_SIZE - 1) & GPU_PAGE_MASK;
ptr = (void *) ((uintptr_t)address & GPU_PAGE_MASK);
status = uct_gdr_copy_mem_reg_internal(uct_md, ptr, reg_size, 0, mem_hndl);
if (status != UCS_OK) {
ucs_free(mem_hndl);
return status;
}
*memh_p = mem_hndl;
return UCS_OK;
}
ucs_status_t ucp_mem_unmap(ucp_context_h context, ucp_mem_h memh)
{
uct_allocated_memory_t mem;
uct_mem_h alloc_pd_memh;
ucs_status_t status;
ucs_debug("unmapping buffer %p memh %p", memh->address, memh);
if (memh == &ucp_mem_dummy_handle) {
return UCS_OK;
}
/* Unregister from all protection domains */
status = ucp_memh_dereg_pds(context, memh, &alloc_pd_memh);
if (status != UCS_OK) {
return status;
}
/* If the memory was also allocated, release it */
if (memh->alloc_method != UCT_ALLOC_METHOD_LAST) {
mem.address = memh->address;
mem.length = memh->length;
mem.method = memh->alloc_method;
mem.pd = memh->alloc_pd; /* May be NULL if method is not PD */
mem.memh = alloc_pd_memh; /* May be INVALID if method is not PD */
status = uct_mem_free(&mem);
if (status != UCS_OK) {
return status;
}
}
ucs_free(memh);
return UCS_OK;
}
static UCS_CLASS_CLEANUP_FUNC(uct_mm_ep_t)
{
uct_mm_iface_t *iface = ucs_derived_of(self->super.super.iface, uct_mm_iface_t);
ucs_status_t status;
uct_mm_remote_seg_t *remote_seg;
struct sglib_hashed_uct_mm_remote_seg_t_iterator iter;
uct_mm_ep_signal_remote(self, UCT_MM_IFACE_SIGNAL_DISCONNECT);
uct_worker_progress_unregister(iface->super.worker, uct_mm_iface_progress,
iface);
for (remote_seg = sglib_hashed_uct_mm_remote_seg_t_it_init(&iter, self->remote_segments_hash);
remote_seg != NULL; remote_seg = sglib_hashed_uct_mm_remote_seg_t_it_next(&iter)) {
sglib_hashed_uct_mm_remote_seg_t_delete(self->remote_segments_hash, remote_seg);
/* detach the remote proceess's descriptors segment */
status = uct_mm_md_mapper_ops(iface->super.md)->detach(remote_seg);
if (status != UCS_OK) {
ucs_warn("Unable to detach shared memory segment of descriptors: %s",
ucs_status_string(status));
}
ucs_free(remote_seg);
}
/* detach the remote proceess's shared memory segment (remote recv FIFO) */
status = uct_mm_md_mapper_ops(iface->super.md)->detach(&self->mapped_desc);
if (status != UCS_OK) {
ucs_error("error detaching from remote FIFO");
}
uct_mm_ep_pending_purge(&self->super.super, NULL, NULL);
}
ucs_status_t ucp_address_pack(ucp_worker_h worker, ucp_ep_h ep, uint64_t tl_bitmap,
unsigned *order, size_t *size_p, void **buffer_p)
{
ucp_address_packed_device_t *devices;
ucp_rsc_index_t num_devices;
ucs_status_t status;
void *buffer;
size_t size;
/* Collect all devices we want to pack */
status = ucp_address_gather_devices(worker, tl_bitmap, ep != NULL,
&devices, &num_devices);
if (status != UCS_OK) {
goto out;
}
/* Calculate packed size */
size = ucp_address_packed_size(worker, devices, num_devices);
/* Allocate address */
buffer = ucs_malloc(size, "ucp_address");
if (buffer == NULL) {
status = UCS_ERR_NO_MEMORY;
goto out_free_devices;
}
memset(buffer, 0, size);
/* Pack the address */
status = ucp_address_do_pack(worker, ep, buffer, size, tl_bitmap, order,
devices, num_devices);
if (status != UCS_OK) {
ucs_free(buffer);
goto out_free_devices;
}
VALGRIND_CHECK_MEM_IS_DEFINED(buffer, size);
*size_p = size;
*buffer_p = buffer;
status = UCS_OK;
out_free_devices:
ucs_free(devices);
out:
return status;
}
static ucs_status_t ucp_ep_send_wireup_am(ucp_ep_h ep, uct_ep_h uct_ep,
uint8_t am_id, ucp_rsc_index_t dst_rsc_index)
{
ucp_rsc_index_t rsc_index = ep->uct.rsc_index;
ucp_worker_h worker = ep->worker;
ucp_context_h context = worker->context;
uct_iface_attr_t *iface_attr = &worker->iface_attrs[rsc_index];
ucp_wireup_msg_t *msg;
ucs_status_t status;
size_t msg_len;
msg_len = sizeof(*msg) + iface_attr->ep_addr_len;
/* TODO use custom pack callback to avoid this allocation and memcopy */
msg = ucs_malloc(msg_len, "conn_req");
if (msg == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err;
}
msg->src_uuid = worker->uuid;
msg->src_pd_index = context->tl_rscs[rsc_index].pd_index;
msg->src_rsc_index = rsc_index;
msg->dst_rsc_index = dst_rsc_index;
status = uct_ep_get_address(ep->uct.next_ep, (struct sockaddr *)(msg + 1));
if (status != UCS_OK) {
goto err_free;
}
status = uct_ep_am_bcopy(uct_ep, am_id, (uct_pack_callback_t)memcpy,
msg, msg_len);
if (status != UCS_OK) {
ucs_debug("failed to send conn msg: %s%s", ucs_status_string(status),
(status == UCS_ERR_NO_RESOURCE) ? ", will retry" : "");
goto err_free;
}
ucp_wireup_log(worker, am_id, msg, 1);
ucs_free(msg);
return UCS_OK;
err_free:
ucs_free(msg);
err:
return status;
}
ucs_status_t ucp_ep_create(ucp_worker_h worker, const ucp_address_t *address,
ucp_ep_h *ep_p)
{
char peer_name[UCP_WORKER_NAME_MAX];
uint8_t addr_indices[UCP_MAX_LANES];
ucp_address_entry_t *address_list;
unsigned address_count;
ucs_status_t status;
uint64_t dest_uuid;
ucp_ep_h ep;
UCS_ASYNC_BLOCK(&worker->async);
status = ucp_address_unpack(address, &dest_uuid, peer_name, sizeof(peer_name),
&address_count, &address_list);
if (status != UCS_OK) {
ucs_error("failed to unpack remote address: %s", ucs_status_string(status));
goto out;
}
ep = ucp_worker_ep_find(worker, dest_uuid);
if (ep != NULL) {
/* TODO handle a case where the existing endpoint is incomplete */
*ep_p = ep;
status = UCS_OK;
goto out_free_address;
}
/* allocate endpoint */
status = ucp_ep_new(worker, dest_uuid, peer_name, "from api call", &ep);
if (status != UCS_OK) {
goto out_free_address;
}
/* initialize transport endpoints */
status = ucp_wireup_init_lanes(ep, address_count, address_list, addr_indices);
if (status != UCS_OK) {
goto err_destroy_ep;
}
/* send initial wireup message */
if (!(ep->flags & UCP_EP_FLAG_LOCAL_CONNECTED)) {
status = ucp_wireup_send_request(ep);
if (status != UCS_OK) {
goto err_destroy_ep;
}
}
*ep_p = ep;
goto out_free_address;
err_destroy_ep:
ucp_ep_destroy(ep);
out_free_address:
ucs_free(address_list);
out:
UCS_ASYNC_UNBLOCK(&worker->async);
return status;
}
void ucp_rkey_buffer_release(void *rkey_buffer)
{
if (rkey_buffer == &ucp_mem_dummy_buffer) {
/* Dummy key, just return */
return;
}
ucs_free(rkey_buffer);
}
ucs_status_t ucp_ep_new(ucp_worker_h worker, uint64_t dest_uuid,
const char *peer_name, const char *message,
ucp_ep_h *ep_p)
{
ucs_status_t status;
ucp_ep_config_key_t key;
ucp_ep_h ep;
khiter_t hash_it;
int hash_extra_status = 0;
ep = ucs_calloc(1, sizeof(*ep), "ucp ep");
if (ep == NULL) {
ucs_error("Failed to allocate ep");
status = UCS_ERR_NO_MEMORY;
goto err;
}
/* EP configuration without any lanes */
memset(&key, 0, sizeof(key));
key.rma_lane_map = 0;
key.amo_lane_map = 0;
key.reachable_md_map = 0;
key.am_lane = UCP_NULL_RESOURCE;
key.rndv_lane = UCP_NULL_RESOURCE;
key.wireup_msg_lane = UCP_NULL_LANE;
key.num_lanes = 0;
memset(key.amo_lanes, UCP_NULL_LANE, sizeof(key.amo_lanes));
ep->worker = worker;
ep->dest_uuid = dest_uuid;
ep->cfg_index = ucp_worker_get_ep_config(worker, &key);
ep->am_lane = UCP_NULL_LANE;
ep->flags = 0;
#if ENABLE_DEBUG_DATA
ucs_snprintf_zero(ep->peer_name, UCP_WORKER_NAME_MAX, "%s", peer_name);
#endif
hash_it = kh_put(ucp_worker_ep_hash, &worker->ep_hash, dest_uuid,
&hash_extra_status);
if (ucs_unlikely(hash_it == kh_end(&worker->ep_hash))) {
ucs_error("Hash failed with ep %p to %s 0x%"PRIx64"->0x%"PRIx64" %s "
"with status %d", ep, peer_name, worker->uuid, ep->dest_uuid,
message, hash_extra_status);
status = UCS_ERR_NO_RESOURCE;
goto err_free_ep;
}
kh_value(&worker->ep_hash, hash_it) = ep;
*ep_p = ep;
ucs_debug("created ep %p to %s 0x%"PRIx64"->0x%"PRIx64" %s", ep, peer_name,
worker->uuid, ep->dest_uuid, message);
return UCS_OK;
err_free_ep:
ucs_free(ep);
err:
return status;
}
void ucs_timerq_cleanup(ucs_timer_queue_t *timerq)
{
ucs_trace_func("timerq=%p", timerq);
if (timerq->num_timers > 0) {
ucs_warn("timer queue with %d timers being destroyed", timerq->num_timers);
}
ucs_free(timerq->timers);
}
/* decrement reference count and release the handler if reached 0 */
static void ucs_async_handler_put(ucs_async_handler_t *handler)
{
if (ucs_atomic_fadd32(&handler->refcount, -1) > 1) {
return;
}
ucs_debug("release async handler " UCS_ASYNC_HANDLER_FMT,
UCS_ASYNC_HANDLER_ARG(handler));
ucs_free(handler);
}
void ucp_rkey_destroy(ucp_rkey_h rkey)
{
unsigned num_rkeys = ucs_count_one_bits(rkey->pd_map);
unsigned i;
for (i = 0; i < num_rkeys; ++i) {
uct_rkey_release(&rkey->uct[i]);
}
ucs_free(rkey);
}
static ucs_status_t ucp_mem_alloc(ucp_context_h context, size_t length,
const char *name, ucp_mem_h memh)
{
uct_allocated_memory_t mem;
uct_alloc_method_t method;
unsigned method_index, pd_index, num_pds;
ucs_status_t status;
uct_pd_h *pds;
pds = ucs_calloc(context->num_pds, sizeof(*pds), "temp pds");
if (pds == NULL) {
return UCS_ERR_NO_MEMORY;
}
for (method_index = 0; method_index < context->config.num_alloc_methods;
++method_index)
{
method = context->config.alloc_methods[method_index].method;
/* If we are trying PD method, gather all PDs which match the component
* name specified in the configuration.
*/
num_pds = 0;
if (method == UCT_ALLOC_METHOD_PD) {
for (pd_index = 0; pd_index < context->num_pds; ++pd_index) {
if (ucp_is_pd_selected_by_config(context, method_index, pd_index)) {
pds[num_pds++] = context->pds[pd_index];
}
}
}
status = uct_mem_alloc(length, &method, 1, pds, num_pds, name, &mem);
if (status == UCS_OK) {
goto allocated;
}
}
status = UCS_ERR_NO_MEMORY;
goto out;
allocated:
ucs_debug("allocated memory at %p with method %s, now registering it",
mem.address, uct_alloc_method_names[mem.method]);
memh->address = mem.address;
memh->length = mem.length;
memh->alloc_method = mem.method;
memh->alloc_pd = mem.pd;
status = ucp_memh_reg_pds(context, memh, mem.memh);
if (status != UCS_OK) {
uct_mem_free(&mem);
}
out:
ucs_free(pds);
return status;
}
void ucp_ep_destroy(ucp_ep_h ep)
{
ucs_debug("destroy ep %p", ep);
ucp_wireup_stop(ep);
if (ep->state & UCP_EP_STATE_READY_TO_SEND) {
ucp_ep_destroy_uct_ep_safe(ep, ep->uct_ep);
} else {
ucs_assert(ep->uct_ep == NULL);
}
ucs_free(ep);
}
void ucs_mpool_hugetlb_free(ucs_mpool_t *mp, void *chunk)
{
ucs_hugetlb_mpool_chunk_hdr_t *hdr;
hdr = (ucs_hugetlb_mpool_chunk_hdr_t*)chunk - 1;
if (hdr->hugetlb) {
ucs_sysv_free(hdr);
} else {
ucs_free(hdr);
}
}
ucs_status_t ucp_ep_create(ucp_worker_h worker, const ucp_address_t *address,
ucp_ep_h *ep_p)
{
char peer_name[UCP_WORKER_NAME_MAX];
ucs_status_t status;
uint64_t dest_uuid;
unsigned address_count;
ucp_address_entry_t *address_list;
ucp_ep_h ep;
UCS_ASYNC_BLOCK(&worker->async);
status = ucp_address_unpack(address, &dest_uuid, peer_name, sizeof(peer_name),
&address_count, &address_list);
if (status != UCS_OK) {
ucs_error("failed to unpack remote address: %s", ucs_status_string(status));
goto out;
}
ep = ucp_worker_ep_find(worker, dest_uuid);
if (ep != NULL) {
/* TODO handle a case where the existing endpoint is incomplete */
ucs_debug("returning existing ep %p which is already connected to %"PRIx64,
ep, ep->dest_uuid);
*ep_p = ep;
status = UCS_OK;
goto out_free_address;
}
status = ucp_ep_create_connected(worker, dest_uuid, peer_name, address_count,
address_list, " from api call", &ep);
if (status != UCS_OK) {
goto out_free_address;
}
/* send initial wireup message */
if (!(ep->flags & UCP_EP_FLAG_LOCAL_CONNECTED)) {
status = ucp_wireup_send_request(ep);
if (status != UCS_OK) {
goto err_destroy_ep;
}
}
*ep_p = ep;
goto out_free_address;
err_destroy_ep:
ucp_ep_destroy(ep);
out_free_address:
ucs_free(address_list);
out:
UCS_ASYNC_UNBLOCK(&worker->async);
return status;
}
void ucs_config_release_array(void *ptr, const void *arg)
{
ucs_config_array_field_t *array_field = ptr;
const ucs_config_array_t *array = arg;
unsigned i;
for (i = 0; i < array_field->count; ++i) {
array->parser.release((char*)array_field->data + i * array->elem_size,
array->parser.arg);
}
ucs_free(array_field->data);
}
ucs_status_t ucp_init_version(unsigned api_major_version, unsigned api_minor_version,
const ucp_params_t *params, const ucp_config_t *config,
ucp_context_h *context_p)
{
unsigned major_version, minor_version, release_number;
ucp_context_t *context;
ucs_status_t status;
ucp_get_version(&major_version, &minor_version, &release_number);
if ((api_major_version != major_version) || (api_minor_version != minor_version)) {
ucs_error("UCP version is incompatible, required: %d.%d, actual: %d.%d (release %d)",
api_major_version, api_minor_version,
major_version, minor_version, release_number);
status = UCS_ERR_NOT_IMPLEMENTED;
goto err;
}
/* allocate a ucp context */
context = ucs_calloc(1, sizeof(*context), "ucp context");
if (context == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err;
}
status = ucp_fill_config(context, params, config);
if (status != UCS_OK) {
goto err_free_ctx;
}
/* fill resources we should use */
status = ucp_fill_resources(context, config);
if (status != UCS_OK) {
goto err_free_config;
}
/* initialize tag matching */
ucs_queue_head_init(&context->tag.expected);
ucs_queue_head_init(&context->tag.unexpected);
ucs_debug("created ucp context %p [%d mds %d tls] features 0x%lx", context,
context->num_mds, context->num_tls, context->config.features);
*context_p = context;
return UCS_OK;
err_free_config:
ucp_free_config(context);
err_free_ctx:
ucs_free(context);
err:
return status;
}
void ucp_ep_destroy(ucp_ep_h ep)
{
ucp_worker_h worker = ep->worker;
ucs_debug("destroy ep %p", ep);
UCS_ASYNC_BLOCK(&worker->async);
sglib_hashed_ucp_ep_t_delete(worker->ep_hash, ep);
ucp_ep_destory_uct_eps(ep);
UCS_ASYNC_UNBLOCK(&worker->async);
ucs_free(ep);
}
static ucs_status_t uct_gdr_copy_mem_dereg(uct_md_h uct_md, uct_mem_h memh)
{
uct_gdr_copy_mem_t *mem_hndl = memh;
ucs_status_t status;
status = uct_gdr_copy_mem_dereg_internal(uct_md, mem_hndl);
if (status != UCS_OK) {
ucs_warn("failed to deregister memory handle");
}
ucs_free(mem_hndl);
return status;
}
