ucs_status_t ucp_ep_new(ucp_worker_h worker, uint64_t dest_uuid,
const char *peer_name, const char *message, ucp_ep_h *ep_p)
{
ucp_ep_h ep;
ep = ucs_calloc(1, sizeof(*ep), "ucp ep");
if (ep == NULL) {
ucs_error("Failed to allocate ep");
return UCS_ERR_NO_MEMORY;
}
ep->worker = worker;
ep->uct_ep = NULL;
ep->config.max_short_egr = SIZE_MAX;
ep->config.max_bcopy_egr = SIZE_MAX;
ep->config.max_short_put = SIZE_MAX;
ep->config.max_bcopy_put = SIZE_MAX;
ep->config.max_bcopy_get = SIZE_MAX;
ep->dest_uuid = dest_uuid;
ep->rsc_index = -1;
ep->dst_pd_index = -1;
ep->state = 0;
sglib_hashed_ucp_ep_t_add(worker->ep_hash, ep);
#if ENABLE_DEBUG_DATA
ucs_snprintf_zero(ep->peer_name, UCP_PEER_NAME_MAX, "%s", peer_name);
#endif
ucs_debug("created ep %p to %s 0x%"PRIx64"->0x%"PRIx64" %s", ep,
ucp_ep_peer_name(ep), worker->uuid, ep->dest_uuid, message);
*ep_p = ep;
return UCS_OK;
}
ucs_status_t ucs_config_clone_array(void *src, void *dest, const void *arg)
{
ucs_config_array_field_t *dest_array = dest, *src_array = src;
const ucs_config_array_t *array = arg;
ucs_status_t status;
unsigned i;
dest_array->data = ucs_calloc(src_array->count, array->elem_size,
"config array");
if (dest_array->data == NULL) {
return UCS_ERR_NO_MEMORY;
}
dest_array->count = src_array->count;
for (i = 0; i < src_array->count; ++i) {
status = array->parser.clone((char*)src_array->data + i * array->elem_size,
(char*)dest_array->data + i * array->elem_size,
array->parser.arg);
if (status != UCS_OK) {
ucs_free(dest_array->data);
return status;
}
}
return UCS_OK;
}
static ucs_status_t ucp_ep_new(ucp_worker_h worker, uint64_t dest_uuid,
const char *peer_name, const char *message,
ucp_ep_h *ep_p)
{
ucp_ep_h ep;
ep = ucs_calloc(1, sizeof(*ep), "ucp ep");
if (ep == NULL) {
ucs_error("Failed to allocate ep");
return UCS_ERR_NO_MEMORY;
}
ep->worker = worker;
ep->dest_uuid = dest_uuid;
ep->rma_dst_pdi = UCP_NULL_RESOURCE;
ep->amo_dst_pdi = UCP_NULL_RESOURCE;
ep->cfg_index = 0;
ep->flags = 0;
#if ENABLE_DEBUG_DATA
ucs_snprintf_zero(ep->peer_name, UCP_WORKER_NAME_MAX, "%s", peer_name);
#endif
sglib_hashed_ucp_ep_t_add(worker->ep_hash, ep);
*ep_p = ep;
ucs_debug("created ep %p to %s 0x%"PRIx64"->0x%"PRIx64" %s", ep, peer_name,
worker->uuid, ep->dest_uuid, message);
return UCS_OK;
}
ucs_status_t uct_ugni_query_tl_resources(uct_pd_h pd, const char *tl_name,
uct_tl_resource_desc_t **resource_p,
unsigned *num_resources_p)
{
uct_tl_resource_desc_t *resources;
int num_devices = job_info.num_devices;
uct_ugni_device_t *devs = job_info.devices;
int i;
ucs_status_t status = UCS_OK;
pthread_mutex_lock(&uct_ugni_global_lock);
resources = ucs_calloc(job_info.num_devices, sizeof(uct_tl_resource_desc_t),
"resource desc");
if (NULL == resources) {
ucs_error("Failed to allocate memory");
num_devices = 0;
resources = NULL;
status = UCS_ERR_NO_MEMORY;
goto error;
}
for (i = 0; i < job_info.num_devices; i++) {
uct_ugni_device_get_resource(tl_name, &devs[i], &resources[i]);
}
error:
*num_resources_p = num_devices;
*resource_p = resources;
pthread_mutex_unlock(&uct_ugni_global_lock);
return status;
}
ucs_status_t ucp_ep_new(ucp_worker_h worker, uint64_t dest_uuid,
const char *peer_name, const char *message,
ucp_ep_h *ep_p)
{
ucs_status_t status;
ucp_ep_config_key_t key;
ucp_ep_h ep;
khiter_t hash_it;
int hash_extra_status = 0;
ep = ucs_calloc(1, sizeof(*ep), "ucp ep");
if (ep == NULL) {
ucs_error("Failed to allocate ep");
status = UCS_ERR_NO_MEMORY;
goto err;
}
/* EP configuration without any lanes */
memset(&key, 0, sizeof(key));
key.rma_lane_map = 0;
key.amo_lane_map = 0;
key.reachable_md_map = 0;
key.am_lane = UCP_NULL_RESOURCE;
key.rndv_lane = UCP_NULL_RESOURCE;
key.wireup_msg_lane = UCP_NULL_LANE;
key.num_lanes = 0;
memset(key.amo_lanes, UCP_NULL_LANE, sizeof(key.amo_lanes));
ep->worker = worker;
ep->dest_uuid = dest_uuid;
ep->cfg_index = ucp_worker_get_ep_config(worker, &key);
ep->am_lane = UCP_NULL_LANE;
ep->flags = 0;
#if ENABLE_DEBUG_DATA
ucs_snprintf_zero(ep->peer_name, UCP_WORKER_NAME_MAX, "%s", peer_name);
#endif
hash_it = kh_put(ucp_worker_ep_hash, &worker->ep_hash, dest_uuid,
&hash_extra_status);
if (ucs_unlikely(hash_it == kh_end(&worker->ep_hash))) {
ucs_error("Hash failed with ep %p to %s 0x%"PRIx64"->0x%"PRIx64" %s "
"with status %d", ep, peer_name, worker->uuid, ep->dest_uuid,
message, hash_extra_status);
status = UCS_ERR_NO_RESOURCE;
goto err_free_ep;
}
kh_value(&worker->ep_hash, hash_it) = ep;
*ep_p = ep;
ucs_debug("created ep %p to %s 0x%"PRIx64"->0x%"PRIx64" %s", ep, peer_name,
worker->uuid, ep->dest_uuid, message);
return UCS_OK;
err_free_ep:
ucs_free(ep);
err:
return status;
}
static ucs_status_t ucp_mem_alloc(ucp_context_h context, size_t length,
const char *name, ucp_mem_h memh)
{
uct_allocated_memory_t mem;
uct_alloc_method_t method;
unsigned method_index, pd_index, num_pds;
ucs_status_t status;
uct_pd_h *pds;
pds = ucs_calloc(context->num_pds, sizeof(*pds), "temp pds");
if (pds == NULL) {
return UCS_ERR_NO_MEMORY;
}
for (method_index = 0; method_index < context->config.num_alloc_methods;
++method_index)
{
method = context->config.alloc_methods[method_index].method;
/* If we are trying PD method, gather all PDs which match the component
* name specified in the configuration.
*/
num_pds = 0;
if (method == UCT_ALLOC_METHOD_PD) {
for (pd_index = 0; pd_index < context->num_pds; ++pd_index) {
if (ucp_is_pd_selected_by_config(context, method_index, pd_index)) {
pds[num_pds++] = context->pds[pd_index];
}
}
}
status = uct_mem_alloc(length, &method, 1, pds, num_pds, name, &mem);
if (status == UCS_OK) {
goto allocated;
}
}
status = UCS_ERR_NO_MEMORY;
goto out;
allocated:
ucs_debug("allocated memory at %p with method %s, now registering it",
mem.address, uct_alloc_method_names[mem.method]);
memh->address = mem.address;
memh->length = mem.length;
memh->alloc_method = mem.method;
memh->alloc_pd = mem.pd;
status = ucp_memh_reg_pds(context, memh, mem.memh);
if (status != UCS_OK) {
uct_mem_free(&mem);
}
out:
ucs_free(pds);
return status;
}
static ucs_status_t
ucp_address_gather_devices(ucp_worker_h worker, uint64_t tl_bitmap, int has_ep,
ucp_address_packed_device_t **devices_p,
ucp_rsc_index_t *num_devices_p)
{
ucp_context_h context = worker->context;
ucp_address_packed_device_t *dev, *devices;
uct_iface_attr_t *iface_attr;
ucp_rsc_index_t num_devices;
ucp_rsc_index_t i;
uint64_t mask;
devices = ucs_calloc(context->num_tls, sizeof(*devices), "packed_devices");
if (devices == NULL) {
return UCS_ERR_NO_MEMORY;
}
num_devices = 0;
for (i = 0; i < context->num_tls; ++i) {
mask = UCS_BIT(i);
if (!(mask & tl_bitmap)) {
continue;
}
iface_attr = &worker->ifaces[i].attr;
if (!(iface_attr->cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) &&
!(iface_attr->cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP)) {
continue;
}
dev = ucp_address_get_device(context->tl_rscs[i].tl_rsc.dev_name,
devices, &num_devices);
dev->tl_addrs_size += iface_attr->iface_addr_len;
if (!(iface_attr->cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) && has_ep) {
/* ep address and its length */
dev->tl_addrs_size += 1 + iface_attr->ep_addr_len;
}
dev->tl_addrs_size += sizeof(uint16_t); /* tl name checksum */
dev->tl_addrs_size += sizeof(ucp_address_packed_iface_attr_t); /* iface attr */
dev->tl_addrs_size += 1; /* iface address length */
dev->rsc_index = i;
dev->dev_addr_len = iface_attr->device_addr_len;
dev->tl_bitmap |= mask;
}
*devices_p = devices;
*num_devices_p = num_devices;
return UCS_OK;
}
static ucs_status_t
ucp_address_gather_devices(ucp_worker_h worker, uint64_t tl_bitmap, int has_ep,
ucp_address_packed_device_t **devices_p,
ucp_rsc_index_t *num_devices_p)
{
ucp_context_h context = worker->context;
ucp_address_packed_device_t *dev, *devices;
uct_iface_attr_t *iface_attr;
ucp_rsc_index_t num_devices;
ucp_rsc_index_t i;
uint64_t mask;
devices = ucs_calloc(context->num_tls, sizeof(*devices), "packed_devices");
if (devices == NULL) {
return UCS_ERR_NO_MEMORY;
}
num_devices = 0;
for (i = 0; i < context->num_tls; ++i) {
mask = UCS_BIT(i);
if (!(mask & tl_bitmap)) {
continue;
}
dev = ucp_address_get_device(context->tl_rscs[i].tl_rsc.dev_name,
devices, &num_devices);
iface_attr = &worker->iface_attrs[i];
if (iface_attr->cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE) {
dev->tl_addrs_size += iface_attr->iface_addr_len;
} else if (iface_attr->cap.flags & UCT_IFACE_FLAG_CONNECT_TO_EP) {
if (has_ep) {
dev->tl_addrs_size += iface_attr->ep_addr_len;
} else {
/* Empty address */
}
} else {
continue;
}
dev->rsc_index = i;
dev->dev_addr_len = iface_attr->device_addr_len;
dev->tl_bitmap |= mask;
dev->tl_addrs_size += 1; /* address length */
dev->tl_addrs_size += ucp_address_string_packed_size(context->tl_rscs[i].tl_rsc.tl_name);
}
*devices_p = devices;
*num_devices_p = num_devices;
return UCS_OK;
}
ucs_status_t ucp_init_version(unsigned api_major_version, unsigned api_minor_version,
const ucp_params_t *params, const ucp_config_t *config,
ucp_context_h *context_p)
{
unsigned major_version, minor_version, release_number;
ucp_context_t *context;
ucs_status_t status;
ucp_get_version(&major_version, &minor_version, &release_number);
if ((api_major_version != major_version) || (api_minor_version != minor_version)) {
ucs_error("UCP version is incompatible, required: %d.%d, actual: %d.%d (release %d)",
api_major_version, api_minor_version,
major_version, minor_version, release_number);
status = UCS_ERR_NOT_IMPLEMENTED;
goto err;
}
/* allocate a ucp context */
context = ucs_calloc(1, sizeof(*context), "ucp context");
if (context == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err;
}
status = ucp_fill_config(context, params, config);
if (status != UCS_OK) {
goto err_free_ctx;
}
/* fill resources we should use */
status = ucp_fill_resources(context, config);
if (status != UCS_OK) {
goto err_free_config;
}
/* initialize tag matching */
ucs_queue_head_init(&context->tag.expected);
ucs_queue_head_init(&context->tag.unexpected);
ucs_debug("created ucp context %p [%d mds %d tls] features 0x%lx", context,
context->num_mds, context->num_tls, context->config.features);
*context_p = context;
return UCS_OK;
err_free_config:
ucp_free_config(context);
err_free_ctx:
ucs_free(context);
err:
return status;
}
ucs_status_t ucp_init(uint64_t features, size_t request_headroom,
const ucp_config_t *config, ucp_context_h *context_p)
{
ucp_context_t *context;
ucs_status_t status;
/* allocate a ucp context */
context = ucs_calloc(1, sizeof(*context), "ucp context");
if (context == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err;
}
status = ucp_fill_config(context, features, config);
if (status != UCS_OK) {
goto err_free_ctx;
}
/* fill resources we should use */
status = ucp_fill_resources(context, config);
if (status != UCS_OK) {
goto err_free_resources;
}
/* initialize tag matching */
status = ucp_tag_init(context);
if (status != UCS_OK) {
goto err_free_config;
}
*context_p = context;
return UCS_OK;
err_free_resources:
ucp_free_resources(context);
err_free_config:
ucp_free_config(context);
err_free_ctx:
ucs_free(context);
err:
return status;
}
int ucs_config_sscanf_array(const char *buf, void *dest, const void *arg)
{
ucs_config_array_field_t *field = dest;
void *temp_field;
const ucs_config_array_t *array = arg;
char *dup, *token, *saveptr;
int ret;
unsigned i;
dup = strdup(buf);
if (dup == NULL) {
return 0;
}
saveptr = NULL;
token = strtok_r(dup, ",", &saveptr);
temp_field = ucs_calloc(UCS_CONFIG_ARRAY_MAX, array->elem_size, "config array");
i = 0;
while (token != NULL) {
ret = array->parser.read(token, (char*)temp_field + i * array->elem_size,
array->parser.arg);
if (!ret) {
ucs_free(temp_field);
free(dup);
return 0;
}
++i;
if (i >= UCS_CONFIG_ARRAY_MAX) {
break;
}
token = strtok_r(NULL, ",", &saveptr);
}
field->data = temp_field;
field->count = i;
free(dup);
return 1;
}
static ucs_status_t uct_gdr_copy_query_tl_resources(uct_md_h md,
uct_tl_resource_desc_t **resource_p,
unsigned *num_resources_p)
{
uct_tl_resource_desc_t *resource;
resource = ucs_calloc(1, sizeof(uct_tl_resource_desc_t), "resource desc");
if (NULL == resource) {
ucs_error("Failed to allocate memory");
return UCS_ERR_NO_MEMORY;
}
ucs_snprintf_zero(resource->tl_name, sizeof(resource->tl_name), "%s",
UCT_GDR_COPY_TL_NAME);
ucs_snprintf_zero(resource->dev_name, sizeof(resource->dev_name), "%s",
UCT_CUDA_DEV_NAME);
resource->dev_type = UCT_DEVICE_TYPE_ACC;
*num_resources_p = 1;
*resource_p = resource;
return UCS_OK;
}
static ucs_status_t uct_cma_query_tl_resources(uct_pd_h pd,
uct_tl_resource_desc_t **resource_p,
unsigned *num_resources_p)
{
uct_tl_resource_desc_t *resource;
resource = ucs_calloc(1, sizeof(uct_tl_resource_desc_t), "resource desc");
if (NULL == resource) {
ucs_error("Failed to allocate memory");
return UCS_ERR_NO_MEMORY;
}
ucs_snprintf_zero(resource->tl_name, sizeof(resource->tl_name), "%s",
UCT_CMA_TL_NAME);
ucs_snprintf_zero(resource->dev_name, sizeof(resource->dev_name), "%s",
pd->component->name);
resource->dev_type = UCT_DEVICE_TYPE_SHM;
*num_resources_p = 1;
*resource_p = resource;
return UCS_OK;
}
static ucs_status_t uct_self_query_tl_resources(uct_md_h md,
uct_tl_resource_desc_t **resource_p,
unsigned *num_resources_p)
{
uct_tl_resource_desc_t *resource = 0;
ucs_trace_func("md=%p", md);
resource = ucs_calloc(1, sizeof(*resource), "resource desc");
if (NULL == resource) {
ucs_error("Failed to allocate memory");
return UCS_ERR_NO_MEMORY;
}
ucs_snprintf_zero(resource->tl_name, sizeof(resource->tl_name), "%s",
UCT_SELF_NAME);
ucs_snprintf_zero(resource->dev_name, sizeof(resource->dev_name), "%s",
UCT_SELF_NAME);
resource->dev_type = UCT_DEVICE_TYPE_SELF;
*num_resources_p = 1;
*resource_p = resource;
return UCS_OK;
}
static ucs_status_t uct_cuda_query_tl_resources(uct_pd_h pd,
uct_tl_resource_desc_t **resource_p,
unsigned *num_resources_p)
{
uct_tl_resource_desc_t *resource;
resource = ucs_calloc(1, sizeof(uct_tl_resource_desc_t), "resource desc");
if (NULL == resource) {
ucs_error("Failed to allocate memory");
return UCS_ERR_NO_MEMORY;
}
ucs_snprintf_zero(resource->tl_name, sizeof(resource->tl_name), "%s",
UCT_CUDA_TL_NAME);
ucs_snprintf_zero(resource->dev_name, sizeof(resource->dev_name), "%s",
UCT_CUDA_DEV_NAME);
resource->latency = 1; /* FIXME temp value */
resource->bandwidth = (long) (6911 * pow(1024,2)); /* FIXME temp value */
*num_resources_p = 1;
*resource_p = resource;
return UCS_OK;
}
ucs_status_t uct_ugni_query_tl_resources(uct_pd_h pd,
uct_tl_resource_desc_t **resource_p,
unsigned *num_resources_p)
{
uct_tl_resource_desc_t *resources;
int num_devices = job_info.num_devices;
uct_ugni_device_t *devs = job_info.devices;
int i;
ucs_status_t rc = UCS_OK;
assert(!strncmp(pd->component->name,
UCT_UGNI_TL_NAME,
UCT_PD_NAME_MAX));
pthread_mutex_lock(&uct_ugni_global_lock);
resources = ucs_calloc(job_info.num_devices, sizeof(uct_tl_resource_desc_t),
"resource desc");
if (NULL == resources) {
ucs_error("Failed to allocate memory");
num_devices = 0;
resources = NULL;
rc = UCS_ERR_NO_MEMORY;
goto error;
}
for (i = 0; i < job_info.num_devices; i++) {
uct_ugni_device_get_resource(&devs[i], &resources[i]);
}
error:
*num_resources_p = num_devices;
*resource_p = resources;
pthread_mutex_unlock(&uct_ugni_global_lock);
return rc;
}
static ucs_status_t ucp_fill_config(ucp_context_h context,
const ucp_params_t *params,
const ucp_config_t *config)
{
unsigned i, num_alloc_methods, method;
const char *method_name;
ucs_status_t status;
if (0 == params->features) {
ucs_warn("empty features set passed to ucp context create");
}
context->config.features = params->features;
context->config.tag_sender_mask = params->tag_sender_mask;
context->config.request.size = params->request_size;
context->config.request.init = params->request_init;
context->config.request.cleanup = params->request_cleanup;
context->config.ext = config->ctx;
/* Get allocation alignment from configuration, make sure it's valid */
if (config->alloc_prio.count == 0) {
ucs_error("No allocation methods specified - aborting");
status = UCS_ERR_INVALID_PARAM;
goto err;
}
num_alloc_methods = config->alloc_prio.count;
context->config.num_alloc_methods = num_alloc_methods;
/* Allocate an array to hold the allocation methods configuration */
context->config.alloc_methods = ucs_calloc(num_alloc_methods,
sizeof(*context->config.alloc_methods),
"ucp_alloc_methods");
if (context->config.alloc_methods == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err;
}
/* Parse the allocation methods specified in the configuration */
for (i = 0; i < num_alloc_methods; ++i) {
method_name = config->alloc_prio.methods[i];
if (!strncasecmp(method_name, "md:", 3)) {
/* If the method name begins with 'md:', treat it as memory domain
* component name.
*/
context->config.alloc_methods[i].method = UCT_ALLOC_METHOD_MD;
strncpy(context->config.alloc_methods[i].mdc_name,
method_name + 3, UCT_MD_COMPONENT_NAME_MAX);
ucs_debug("allocation method[%d] is md '%s'", i, method_name + 3);
} else {
/* Otherwise, this is specific allocation method name.
*/
context->config.alloc_methods[i].method = UCT_ALLOC_METHOD_LAST;
for (method = 0; method < UCT_ALLOC_METHOD_LAST; ++method) {
if ((method != UCT_ALLOC_METHOD_MD) &&
!strcmp(method_name, uct_alloc_method_names[method]))
{
/* Found the allocation method in the internal name list */
context->config.alloc_methods[i].method = method;
strcpy(context->config.alloc_methods[i].mdc_name, "");
ucs_debug("allocation method[%d] is '%s'", i, method_name);
break;
}
}
if (context->config.alloc_methods[i].method == UCT_ALLOC_METHOD_LAST) {
ucs_error("Invalid allocation method: %s", method_name);
status = UCS_ERR_INVALID_PARAM;
goto err_free;
}
}
}
return UCS_OK;
err_free:
ucs_free(context->config.alloc_methods);
err:
return status;
}
static ucs_status_t ucp_fill_resources(ucp_context_h context,
const ucp_config_t *config)
{
unsigned num_tl_resources;
unsigned num_md_resources;
uct_md_resource_desc_t *md_rscs;
ucs_status_t status;
ucp_rsc_index_t i;
unsigned md_index;
uct_md_h md;
uct_md_config_t *md_config;
uint64_t masks[UCT_DEVICE_TYPE_LAST] = {0};
/* if we got here then num_resources > 0.
* if the user's device list is empty, there is no match */
if ((0 == config->devices[UCT_DEVICE_TYPE_NET].count) &&
(0 == config->devices[UCT_DEVICE_TYPE_SHM].count) &&
(0 == config->devices[UCT_DEVICE_TYPE_ACC].count) &&
(0 == config->devices[UCT_DEVICE_TYPE_SELF].count)) {
ucs_error("The device lists are empty. Please specify the devices you would like to use "
"or omit the UCX_*_DEVICES so that the default will be used.");
status = UCS_ERR_NO_ELEM;
goto err;
}
/* if we got here then num_resources > 0.
* if the user's tls list is empty, there is no match */
if (0 == config->tls.count) {
ucs_error("The TLs list is empty. Please specify the transports you would like to use "
"or omit the UCX_TLS so that the default will be used.");
status = UCS_ERR_NO_ELEM;
goto err;
}
/* List memory domain resources */
status = uct_query_md_resources(&md_rscs, &num_md_resources);
if (status != UCS_OK) {
goto err;
}
/* Sort md's by name, to increase the likelihood of reusing the same ep
* configuration (since remote md map is part of the key).
*/
qsort(md_rscs, num_md_resources, sizeof(*md_rscs), ucp_md_rsc_compare_name);
/* Error check: Make sure there is at least one MD */
if (num_md_resources == 0) {
ucs_error("No md resources found");
status = UCS_ERR_NO_DEVICE;
goto err_release_md_resources;
}
context->num_mds = 0;
context->md_rscs = NULL;
context->mds = NULL;
context->md_attrs = NULL;
context->num_tls = 0;
context->tl_rscs = NULL;
/* Allocate array of MD resources we would actually use */
context->md_rscs = ucs_calloc(num_md_resources, sizeof(*context->md_rscs),
"ucp_md_resources");
if (context->md_rscs == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err_free_context_resources;
}
/* Allocate array of memory domains */
context->mds = ucs_calloc(num_md_resources, sizeof(*context->mds), "ucp_mds");
if (context->mds == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err_free_context_resources;
}
/* Allocate array of memory domains attributes */
context->md_attrs = ucs_calloc(num_md_resources, sizeof(*context->md_attrs),
"ucp_md_attrs");
if (context->md_attrs == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err_free_context_resources;
}
/* Open all memory domains, keep only those which have at least one TL
* resources selected on them.
*/
md_index = 0;
for (i = 0; i < num_md_resources; ++i) {
status = uct_md_config_read(md_rscs[i].md_name, NULL, NULL, &md_config);
if (status != UCS_OK) {
goto err_free_context_resources;
}
status = uct_md_open(md_rscs[i].md_name, md_config, &md);
uct_config_release(md_config);
if (status != UCS_OK) {
goto err_free_context_resources;
}
context->md_rscs[md_index] = md_rscs[i];
context->mds[md_index] = md;
/* Save MD attributes */
status = uct_md_query(md, &context->md_attrs[md_index]);
if (status != UCS_OK) {
goto err_free_context_resources;
}
/* Add communication resources of each MD */
status = ucp_add_tl_resources(context, md, md_index, config,
&num_tl_resources, masks);
if (status != UCS_OK) {
goto err_free_context_resources;
}
/* If the MD does not have transport resources, don't use it */
if (num_tl_resources > 0) {
++md_index;
++context->num_mds;
} else {
ucs_debug("closing md %s because it has no selected transport resources",
md_rscs[i].md_name);
uct_md_close(md);
}
}
/* Error check: Make sure there is at least one transport */
if (0 == context->num_tls) {
ucs_error("There are no available resources matching the configured criteria");
status = UCS_ERR_NO_DEVICE;
goto err_free_context_resources;
}
if (context->num_mds > UCP_MD_INDEX_BITS) {
ucs_error("Only up to %d memory domains are supported (have: %d)",
UCP_MD_INDEX_BITS, context->num_mds);
status = UCS_ERR_EXCEEDS_LIMIT;
goto err_release_md_resources;
}
/* Notify the user if there are devices from the command line that are not available */
ucp_check_unavailable_devices(config->devices, masks);
/* Error check: Make sure there are not too many transports */
if (context->num_tls >= UCP_MAX_RESOURCES) {
ucs_error("Exceeded resources limit (%u requested, up to %d are supported)",
context->num_tls, UCP_MAX_RESOURCES);
status = UCS_ERR_EXCEEDS_LIMIT;
goto err_free_context_resources;
}
status = ucp_check_tl_names(context);
if (status != UCS_OK) {
goto err_free_context_resources;
}
uct_release_md_resource_list(md_rscs);
return UCS_OK;
err_free_context_resources:
ucp_free_resources(context);
err_release_md_resources:
uct_release_md_resource_list(md_rscs);
err:
return status;
}
ucs_status_t ucp_address_unpack(const void *buffer, uint64_t *remote_uuid_p,
char *remote_name, size_t max,
unsigned *address_count_p,
ucp_address_entry_t **address_list_p)
{
ucp_address_entry_t *address_list, *address;
const uct_device_addr_t *dev_addr;
ucp_rsc_index_t pd_index;
unsigned address_count;
int last_dev, last_tl;
int empty_dev;
size_t dev_addr_len;
size_t tl_addr_len;
const void *ptr;
const void *aptr;
ptr = buffer;
*remote_uuid_p = *(uint64_t*)ptr;
ptr += sizeof(uint64_t);
aptr = ucp_address_unpack_string(ptr, remote_name, max);
address_count = 0;
/* Count addresses */
ptr = aptr;
do {
if (*(uint8_t*)ptr == UCP_NULL_RESOURCE) {
break;
}
/* pd_index */
pd_index = (*(uint8_t*)ptr) & ~UCP_ADDRESS_FLAG_EMPTY;
empty_dev = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_EMPTY;
++ptr;
/* device address length */
dev_addr_len = (*(uint8_t*)ptr) & ~UCP_ADDRESS_FLAG_LAST;
last_dev = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LAST;
++ptr;
ptr += dev_addr_len;
last_tl = empty_dev;
while (!last_tl) {
ptr = ucp_address_skip_string(ptr); /* tl_name */
/* tl address length */
tl_addr_len = (*(uint8_t*)ptr) & ~UCP_ADDRESS_FLAG_LAST;
last_tl = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LAST;
++ptr;
++address_count;
ucs_assert(address_count <= UCP_MAX_RESOURCES);
ptr += tl_addr_len;
}
} while (!last_dev);
/* Allocate address list */
address_list = ucs_calloc(address_count, sizeof(*address_list),
"ucp_address_list");
if (address_list == NULL) {
return UCS_ERR_NO_MEMORY;
}
/* Unpack addresses */
address = address_list;
ptr = aptr;
do {
if (*(uint8_t*)ptr == UCP_NULL_RESOURCE) {
break;
}
/* pd_index */
pd_index = (*(uint8_t*)ptr) & ~UCP_ADDRESS_FLAG_EMPTY;
empty_dev = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_EMPTY;
++ptr;
/* device address length */
dev_addr_len = (*(uint8_t*)ptr) & ~UCP_ADDRESS_FLAG_LAST;
last_dev = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LAST;
++ptr;
dev_addr = ptr;
ptr += dev_addr_len;
last_tl = empty_dev;
while (!last_tl) {
/* tl name */
ptr = ucp_address_unpack_string(ptr, address->tl_name,
UCT_TL_NAME_MAX);
/* tl address length */
tl_addr_len = (*(uint8_t*)ptr) & ~UCP_ADDRESS_FLAG_LAST;
last_tl = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LAST;
++ptr;
address->dev_addr = dev_addr;
address->dev_addr_len = dev_addr_len;
address->pd_index = pd_index;
address->tl_addr = ptr;
address->tl_addr_len = tl_addr_len;
++address;
ptr += tl_addr_len;
}
} while (!last_dev);
*address_count_p = address_count;
*address_list_p = address_list;
return UCS_OK;
}
static ucs_status_t uct_ib_mlx5dv_md_open(struct ibv_device *ibv_device,
uct_ib_md_t **p_md)
{
uint32_t out[UCT_IB_MLX5DV_ST_SZ_DW(query_hca_cap_out)] = {};
uint32_t in[UCT_IB_MLX5DV_ST_SZ_DW(query_hca_cap_in)] = {};
struct mlx5dv_context_attr dv_attr = {};
ucs_status_t status = UCS_OK;
int atomic = 0, has_dc = 1;
struct ibv_context *ctx;
uct_ib_device_t *dev;
uct_ib_mlx5_md_t *md;
void *cap;
int ret;
#if HAVE_DECL_MLX5DV_IS_SUPPORTED
if (!mlx5dv_is_supported(ibv_device)) {
return UCS_ERR_UNSUPPORTED;
}
#endif
dv_attr.flags |= MLX5DV_CONTEXT_FLAGS_DEVX;
ctx = mlx5dv_open_device(ibv_device, &dv_attr);
if (ctx == NULL) {
ucs_debug("mlx5dv_open_device(%s) failed: %m", ibv_get_device_name(ibv_device));
status = UCS_ERR_UNSUPPORTED;
goto err;
}
md = ucs_calloc(1, sizeof(*md), "ib_mlx5_md");
if (md == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err_free_context;
}
md->super.ops = &uct_ib_mlx5dv_md_ops;
dev = &md->super.dev;
dev->ibv_context = ctx;
IBV_EXP_DEVICE_ATTR_SET_COMP_MASK(&dev->dev_attr);
ret = ibv_query_device_ex(dev->ibv_context, NULL, &dev->dev_attr);
if (ret != 0) {
ucs_error("ibv_query_device() returned %d: %m", ret);
status = UCS_ERR_IO_ERROR;
goto err_free;
}
cap = UCT_IB_MLX5DV_ADDR_OF(query_hca_cap_out, out, capability);
UCT_IB_MLX5DV_SET(query_hca_cap_in, in, opcode, UCT_IB_MLX5_CMD_OP_QUERY_HCA_CAP);
UCT_IB_MLX5DV_SET(query_hca_cap_in, in, op_mod, UCT_IB_MLX5_HCA_CAP_OPMOD_GET_MAX |
(UCT_IB_MLX5_CAP_GENERAL << 1));
ret = mlx5dv_devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
if (ret == 0) {
if (!UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dct)) {
has_dc = 0;
}
if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, compact_address_vector)) {
dev->flags |= UCT_IB_DEVICE_FLAG_AV;
}
if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, fixed_buffer_size)) {
md->flags |= UCT_IB_MLX5_MD_FLAG_KSM;
}
if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, atomic)) {
atomic = 1;
}
} else if ((errno != EPERM) &&
(errno != EPROTONOSUPPORT) &&
(errno != EOPNOTSUPP)) {
ucs_error("MLX5_CMD_OP_QUERY_HCA_CAP failed: %m");
status = UCS_ERR_IO_ERROR;
goto err_free;
} else {
status = UCS_ERR_UNSUPPORTED;
goto err_free;
}
if (atomic) {
int ops = UCT_IB_MLX5_ATOMIC_OPS_CMP_SWAP |
UCT_IB_MLX5_ATOMIC_OPS_FETCH_ADD;
uint8_t arg_size;
int cap_ops, mode8b;
UCT_IB_MLX5DV_SET(query_hca_cap_in, in, op_mod, UCT_IB_MLX5_HCA_CAP_OPMOD_GET_MAX |
(UCT_IB_MLX5_CAP_ATOMIC << 1));
ret = mlx5dv_devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
if (ret != 0) {
ucs_error("MLX5_CMD_OP_QUERY_HCA_CAP failed: %m");
return UCS_ERR_IO_ERROR;
}
arg_size = UCT_IB_MLX5DV_GET(atomic_caps, cap, atomic_size_qp);
cap_ops = UCT_IB_MLX5DV_GET(atomic_caps, cap, atomic_operations);
mode8b = UCT_IB_MLX5DV_GET(atomic_caps, cap, atomic_req_8B_endianness_mode);
if ((cap_ops & ops) == ops) {
dev->atomic_arg_sizes = sizeof(uint64_t);
if (!mode8b) {
dev->atomic_arg_sizes_be = sizeof(uint64_t);
}
}
ops |= UCT_IB_MLX5_ATOMIC_OPS_MASKED_CMP_SWAP |
UCT_IB_MLX5_ATOMIC_OPS_MASKED_FETCH_ADD;
if (has_dc) {
arg_size &= UCT_IB_MLX5DV_GET(query_hca_cap_out, out,
capability.atomic_caps.atomic_size_dc);
}
if ((cap_ops & ops) == ops) {
dev->ext_atomic_arg_sizes = arg_size;
if (mode8b) {
arg_size &= ~(sizeof(uint64_t));
}
dev->ext_atomic_arg_sizes_be = arg_size;
}
}
if (has_dc) {
status = uct_ib_mlx5_check_dc(dev);
}
dev->flags |= UCT_IB_DEVICE_FLAG_MLX5_PRM;
*p_md = &md->super;
return status;
err_free:
ucs_free(md);
err_free_context:
ibv_close_device(ctx);
err:
return status;
}
static ucs_status_t uct_ib_iface_init_lmc(uct_ib_iface_t *iface,
const uct_ib_iface_config_t *config)
{
unsigned i, j, num_path_bits;
unsigned first, last;
uint8_t lmc;
int step;
if (config->lid_path_bits.count == 0) {
ucs_error("List of path bits must not be empty");
return UCS_ERR_INVALID_PARAM;
}
/* count the number of lid_path_bits */
num_path_bits = 0;
for (i = 0; i < config->lid_path_bits.count; i++) {
num_path_bits += 1 + abs(config->lid_path_bits.ranges[i].first -
config->lid_path_bits.ranges[i].last);
}
iface->path_bits = ucs_calloc(1, num_path_bits * sizeof(*iface->path_bits),
"ib_path_bits");
if (iface->path_bits == NULL) {
return UCS_ERR_NO_MEMORY;
}
lmc = uct_ib_iface_port_attr(iface)->lmc;
/* go over the list of values (ranges) for the lid_path_bits and set them */
iface->path_bits_count = 0;
for (i = 0; i < config->lid_path_bits.count; ++i) {
first = config->lid_path_bits.ranges[i].first;
last = config->lid_path_bits.ranges[i].last;
/* range of values or one value */
if (first < last) {
step = 1;
} else {
step = -1;
}
/* fill the value/s */
for (j = first; j != (last + step); j += step) {
if (j >= UCS_BIT(lmc)) {
ucs_debug("Not using value %d for path_bits - must be < 2^lmc (lmc=%d)",
j, lmc);
if (step == 1) {
break;
} else {
continue;
}
}
ucs_assert(iface->path_bits_count <= num_path_bits);
iface->path_bits[iface->path_bits_count] = j;
iface->path_bits_count++;
}
}
return UCS_OK;
}
static int init_device_list(uct_ugni_job_info_t *inf)
{
ucs_status_t status = UCS_OK;
int i, num_active_devices;
int *dev_ids = NULL;
gni_return_t ugni_rc = GNI_RC_SUCCESS;
/* check if devices were already initilized */
if (-1 != inf->num_devices) {
ucs_debug("The device list is already initialized");
status = UCS_OK;
goto err_zero;
}
ugni_rc = GNI_GetNumLocalDevices(&inf->num_devices);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_GetNumLocalDevices failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
status = UCS_ERR_NO_DEVICE;
goto err_zero;
}
if (0 == inf->num_devices) {
ucs_debug("UGNI No device found");
status = UCS_OK;
goto err_zero;
}
if (inf->num_devices >= UCT_UGNI_MAX_DEVICES) {
ucs_error("UGNI, number of discovered devices (%d) " \
"is above the maximum supported devices (%d)",
inf->num_devices, UCT_UGNI_MAX_DEVICES);
status = UCS_ERR_UNSUPPORTED;
goto err_zero;
}
dev_ids = ucs_calloc(inf->num_devices, sizeof(int), "ugni device ids");
if (NULL == dev_ids) {
ucs_error("Failed to allocate memory");
status = UCS_ERR_NO_MEMORY;
goto err_zero;
}
ugni_rc = GNI_GetLocalDeviceIds(inf->num_devices, dev_ids);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_GetLocalDeviceIds failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
status = UCS_ERR_NO_DEVICE;
goto err_dev_id;
}
num_active_devices = 0;
for (i = 0; i < inf->num_devices; i++) {
status = uct_ugni_device_create(dev_ids[i], num_active_devices, &inf->devices[i]);
if (status != UCS_OK) {
ucs_warn("Failed to initialize ugni device %d (%s), ignoring it",
i, ucs_status_string(status));
} else {
++num_active_devices;
}
}
if (num_active_devices != inf->num_devices) {
ucs_warn("Error in detection devices");
status = UCS_ERR_NO_DEVICE;
goto err_dev_id;
}
ucs_debug("Initialized UGNI component with %d devices", inf->num_devices);
err_dev_id:
ucs_free(dev_ids);
err_zero:
return status;
}
static ucs_status_t uct_ib_mlx5dv_create_ksm(uct_ib_md_t *ibmd,
uct_ib_mem_t *ib_memh,
off_t offset)
{
uct_ib_mlx5_mem_t *memh = ucs_derived_of(ib_memh, uct_ib_mlx5_mem_t);
uct_ib_mlx5_md_t *md = ucs_derived_of(ibmd, uct_ib_mlx5_md_t);
uint32_t out[UCT_IB_MLX5DV_ST_SZ_DW(create_mkey_out)] = {};
struct ibv_mr *mr = memh->super.mr;
ucs_status_t status = UCS_OK;
struct mlx5dv_pd dvpd = {};
struct mlx5dv_obj dv = {};
size_t reg_length, length, inlen;
int list_size, i;
void *mkc, *klm;
uint32_t *in;
intptr_t addr;
if (!(md->flags & UCT_IB_MLX5_MD_FLAG_KSM)) {
return UCS_ERR_UNSUPPORTED;
}
reg_length = UCT_IB_MD_MAX_MR_SIZE;
addr = (intptr_t)mr->addr & ~(reg_length - 1);
length = mr->length + (intptr_t)mr->addr - addr;
list_size = ucs_div_round_up(length, reg_length);
inlen = UCT_IB_MLX5DV_ST_SZ_BYTES(create_mkey_in) +
UCT_IB_MLX5DV_ST_SZ_BYTES(klm) * list_size;
in = ucs_calloc(1, inlen, "mkey mailbox");
if (in == NULL) {
return UCS_ERR_NO_MEMORY;
}
dv.pd.in = md->super.pd;
dv.pd.out = &dvpd;
mlx5dv_init_obj(&dv, MLX5DV_OBJ_PD);
UCT_IB_MLX5DV_SET(create_mkey_in, in, opcode, UCT_IB_MLX5_CMD_OP_CREATE_MKEY);
mkc = UCT_IB_MLX5DV_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
UCT_IB_MLX5DV_SET(mkc, mkc, access_mode_1_0, UCT_IB_MLX5_MKC_ACCESS_MODE_KSM);
UCT_IB_MLX5DV_SET(mkc, mkc, a, 1);
UCT_IB_MLX5DV_SET(mkc, mkc, rw, 1);
UCT_IB_MLX5DV_SET(mkc, mkc, rr, 1);
UCT_IB_MLX5DV_SET(mkc, mkc, lw, 1);
UCT_IB_MLX5DV_SET(mkc, mkc, lr, 1);
UCT_IB_MLX5DV_SET(mkc, mkc, pd, dvpd.pdn);
UCT_IB_MLX5DV_SET(mkc, mkc, translations_octword_size, list_size);
UCT_IB_MLX5DV_SET(mkc, mkc, log_entity_size, ucs_ilog2(reg_length));
UCT_IB_MLX5DV_SET(mkc, mkc, qpn, 0xffffff);
UCT_IB_MLX5DV_SET(mkc, mkc, mkey_7_0, offset & 0xff);
UCT_IB_MLX5DV_SET64(mkc, mkc, start_addr, addr + offset);
UCT_IB_MLX5DV_SET64(mkc, mkc, len, length);
UCT_IB_MLX5DV_SET(create_mkey_in, in, translations_octword_actual_size, list_size);
klm = UCT_IB_MLX5DV_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
for (i = 0; i < list_size; i++) {
if (i == list_size - 1) {
UCT_IB_MLX5DV_SET(klm, klm, byte_count, length % reg_length);
} else {
UCT_IB_MLX5DV_SET(klm, klm, byte_count, reg_length);
}
UCT_IB_MLX5DV_SET(klm, klm, mkey, mr->lkey);
UCT_IB_MLX5DV_SET64(klm, klm, address, addr + (i * reg_length));
klm += UCT_IB_MLX5DV_ST_SZ_BYTES(klm);
}
memh->atomic_dvmr = mlx5dv_devx_obj_create(md->super.dev.ibv_context, in, inlen,
out, sizeof(out));
if (memh->atomic_dvmr == NULL) {
ucs_debug("CREATE_MKEY KSM failed: %m");
status = UCS_ERR_UNSUPPORTED;
md->flags &= ~UCT_IB_MLX5_MD_FLAG_KSM;
goto out;
}
memh->super.atomic_rkey =
(UCT_IB_MLX5DV_GET(create_mkey_out, out, mkey_index) << 8) |
(offset & 0xff);
ucs_debug("KSM registered memory %p..%p offset 0x%lx on %s rkey 0x%x",
mr->addr, mr->addr + mr->length, offset, uct_ib_device_name(&md->super.dev),
memh->super.atomic_rkey);
out:
ucs_free(in);
return status;
}
static UCS_CLASS_INIT_FUNC(uct_cm_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_cm_iface_config_t *config = ucs_derived_of(tl_config, uct_cm_iface_config_t);
ucs_status_t status;
int ret;
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ib_iface_t, &uct_cm_iface_ops, pd, worker,
dev_name, rx_headroom, 0 /* rx_priv_len */,
0 /* rx_hdr_len */, 1 /* tx_cq_len */,
IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE, /* mss */
&config->super);
if (worker->async == NULL) {
ucs_error("cm must have async!=NULL");
return UCS_ERR_INVALID_PARAM;
}
self->service_id = (uint32_t)(ucs_generate_uuid((uintptr_t)self) &
(~IB_CM_ASSIGN_SERVICE_ID_MASK));
self->num_outstanding = 0;
self->config.timeout_ms = (int)(config->timeout * 1e3 + 0.5);
self->config.max_outstanding = config->max_outstanding;
self->config.retry_count = ucs_min(config->retry_count, UINT8_MAX);
self->notify_q.head = NULL;
ucs_queue_head_init(&self->notify_q);
self->outstanding = ucs_calloc(self->config.max_outstanding,
sizeof(*self->outstanding),
"cm_outstanding");
if (self->outstanding == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err;
}
self->cmdev = ib_cm_open_device(uct_ib_iface_device(&self->super)->ibv_context);
if (self->cmdev == NULL) {
ucs_error("ib_cm_open_device() failed: %m. Check if ib_ucm.ko module is loaded.");
status = UCS_ERR_NO_DEVICE;
goto err_free_outstanding;
}
status = ucs_sys_fcntl_modfl(self->cmdev->fd, O_NONBLOCK, 0);
if (status != UCS_OK) {
goto err_close_device;
}
ret = ib_cm_create_id(self->cmdev, &self->listen_id, self);
if (ret) {
ucs_error("ib_cm_create_id() failed: %m");
status = UCS_ERR_NO_DEVICE;
goto err_close_device;
}
ret = ib_cm_listen(self->listen_id, self->service_id, 0);
if (ret) {
ucs_error("ib_cm_listen() failed: %m");
status = UCS_ERR_INVALID_ADDR;
goto err_destroy_id;
}
if (config->async_mode == UCS_ASYNC_MODE_SIGNAL) {
ucs_warn("ib_cm fd does not support SIGIO");
}
status = ucs_async_set_event_handler(config->async_mode, self->cmdev->fd,
POLLIN, uct_cm_iface_event_handler, self,
worker->async);
if (status != UCS_OK) {
ucs_error("failed to set event handler");
goto err_destroy_id;
}
ucs_debug("listening for SIDR service_id 0x%x on fd %d", self->service_id,
self->cmdev->fd);
return UCS_OK;
err_destroy_id:
ib_cm_destroy_id(self->listen_id);
err_close_device:
ib_cm_close_device(self->cmdev);
err_free_outstanding:
ucs_free(self->outstanding);
err:
return status;
}
static ucs_status_t ucp_fill_resources(ucp_context_h context,
const ucp_config_t *config)
{
unsigned num_tl_resources;
unsigned num_pd_resources;
uct_pd_resource_desc_t *pd_rscs;
ucs_status_t status;
ucp_rsc_index_t i;
unsigned pd_index;
uct_pd_h pd;
/* if we got here then num_resources > 0.
* if the user's device list is empty, there is no match */
if (0 == config->devices.count) {
ucs_error("The device list is empty. Please specify the devices you would like to use "
"or omit the UCX_DEVICES so that the default will be used.");
status = UCS_ERR_NO_ELEM;
goto err;
}
/* if we got here then num_resources > 0.
* if the user's tls list is empty, there is no match */
if (0 == config->tls.count) {
ucs_error("The TLs list is empty. Please specify the transports you would like to use "
"or omit the UCX_TLS so that the default will be used.");
status = UCS_ERR_NO_ELEM;
goto err;
}
/* List protection domain resources */
status = uct_query_pd_resources(&pd_rscs, &num_pd_resources);
if (status != UCS_OK) {
goto err;
}
/* Error check: Make sure there is at least one PD */
if (num_pd_resources == 0) {
ucs_error("No pd resources found");
status = UCS_ERR_NO_DEVICE;
goto err_release_pd_resources;
}
if (num_pd_resources >= UCP_MAX_PDS) {
ucs_error("Only up to %ld PDs are supported", UCP_MAX_PDS);
status = UCS_ERR_EXCEEDS_LIMIT;
goto err_release_pd_resources;
}
context->num_pds = 0;
context->pd_rscs = NULL;
context->pds = NULL;
context->pd_attrs = NULL;
context->num_tls = 0;
context->tl_rscs = NULL;
/* Allocate array of PD resources we would actually use */
context->pd_rscs = ucs_calloc(num_pd_resources, sizeof(*context->pd_rscs),
"ucp_pd_resources");
if (context->pd_rscs == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err_free_context_resources;
}
/* Allocate array of protection domains */
context->pds = ucs_calloc(num_pd_resources, sizeof(*context->pds), "ucp_pds");
if (context->pds == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err_free_context_resources;
}
/* Allocate array of protection domains attributes */
context->pd_attrs = ucs_calloc(num_pd_resources, sizeof(*context->pd_attrs),
"ucp_pd_attrs");
if (context->pd_attrs == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err_free_context_resources;
}
/* Open all protection domains, keep only those which have at least one TL
* resources selected on them.
*/
pd_index = 0;
for (i = 0; i < num_pd_resources; ++i) {
status = uct_pd_open(pd_rscs[i].pd_name, &pd);
if (status != UCS_OK) {
goto err_free_context_resources;
}
context->pd_rscs[pd_index] = pd_rscs[i];
context->pds[pd_index] = pd;
/* Save PD attributes */
status = uct_pd_query(pd, &context->pd_attrs[pd_index]);
if (status != UCS_OK) {
goto err_free_context_resources;
}
/* Add communication resources of each PD */
status = ucp_add_tl_resources(context, pd, pd_index, config,
&num_tl_resources);
if (status != UCS_OK) {
goto err_free_context_resources;
}
/* If the PD does not have transport resources, don't use it */
if (num_tl_resources > 0) {
++pd_index;
++context->num_pds;
} else {
ucs_debug("closing pd %s because it has no selected transport resources",
pd_rscs[i].pd_name);
uct_pd_close(pd);
}
}
/* Error check: Make sure there is at least one transport */
if (0 == context->num_tls) {
ucs_error("There are no available resources matching the configured criteria");
status = UCS_ERR_NO_DEVICE;
goto err_free_context_resources;
}
/* Error check: Make sure there are no too many transports */
if (context->num_tls >= UCP_MAX_TLS) {
ucs_error("Exceeded resources limit (%u requested, up to %d are supported)",
context->num_tls, UCP_MAX_TLS);
status = UCS_ERR_EXCEEDS_LIMIT;
goto err_free_context_resources;
}
uct_release_pd_resource_list(pd_rscs);
return UCS_OK;
err_free_context_resources:
ucp_free_resources(context);
err_release_pd_resources:
uct_release_pd_resource_list(pd_rscs);
err:
return status;
}
ucs_status_t ucp_address_unpack(const void *buffer, uint64_t *remote_uuid_p,
char *remote_name, size_t max,
unsigned *address_count_p,
ucp_address_entry_t **address_list_p)
{
ucp_address_entry_t *address_list, *address;
const uct_device_addr_t *dev_addr;
ucp_rsc_index_t dev_index;
ucp_rsc_index_t md_index;
unsigned address_count;
int last_dev, last_tl, ep_addr_present;
int empty_dev;
uint64_t md_flags;
size_t dev_addr_len;
size_t iface_addr_len;
size_t ep_addr_len;
uint8_t md_byte;
const void *ptr;
const void *aptr;
ptr = buffer;
*remote_uuid_p = *(uint64_t*)ptr;
ptr += sizeof(uint64_t);
aptr = ucp_address_unpack_string(ptr, remote_name, max);
address_count = 0;
/* Count addresses */
ptr = aptr;
do {
if (*(uint8_t*)ptr == UCP_NULL_RESOURCE) {
break;
}
/* md_index */
empty_dev = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_EMPTY;
++ptr;
/* device address length */
dev_addr_len = (*(uint8_t*)ptr) & ~UCP_ADDRESS_FLAG_LAST;
last_dev = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LAST;
++ptr;
ptr += dev_addr_len;
last_tl = empty_dev;
while (!last_tl) {
ptr += sizeof(uint16_t); /* tl_name_csum */
ptr += sizeof(ucp_address_packed_iface_attr_t); /* iface attr */
/* iface and ep address lengths */
iface_addr_len = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LEN_MASK;
last_tl = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LAST;
ep_addr_present = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_EP_ADDR;
ptr += 1 + iface_addr_len;
if (ep_addr_present) {
ep_addr_len = *(uint8_t*)ptr;
ptr += 1 + ep_addr_len;
}
++address_count;
ucs_assert(address_count <= UCP_MAX_RESOURCES);
}
} while (!last_dev);
/* Allocate address list */
address_list = ucs_calloc(address_count, sizeof(*address_list),
"ucp_address_list");
if (address_list == NULL) {
return UCS_ERR_NO_MEMORY;
}
/* Unpack addresses */
address = address_list;
ptr = aptr;
dev_index = 0;
do {
if (*(uint8_t*)ptr == UCP_NULL_RESOURCE) {
break;
}
/* md_index */
md_byte = (*(uint8_t*)ptr);
md_index = md_byte & UCP_ADDRESS_FLAG_MD_MASK;
md_flags = (md_byte & UCP_ADDRESS_FLAG_MD_ALLOC) ? UCT_MD_FLAG_ALLOC : 0;
md_flags |= (md_byte & UCP_ADDRESS_FLAG_MD_REG) ? UCT_MD_FLAG_REG : 0;
empty_dev = md_byte & UCP_ADDRESS_FLAG_EMPTY;
++ptr;
/* device address length */
dev_addr_len = (*(uint8_t*)ptr) & ~UCP_ADDRESS_FLAG_LAST;
last_dev = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LAST;
++ptr;
dev_addr = ptr;
ptr += dev_addr_len;
last_tl = empty_dev;
while (!last_tl) {
/* tl name */
address->tl_name_csum = *(uint16_t*)ptr;
ptr += sizeof(uint16_t);
/* tl_name_csum */
ucp_address_unpack_iface_attr(&address->iface_attr, ptr);
ptr += sizeof(ucp_address_packed_iface_attr_t);
/* tl address length */
iface_addr_len = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LEN_MASK;
last_tl = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_LAST;
ep_addr_present = (*(uint8_t*)ptr) & UCP_ADDRESS_FLAG_EP_ADDR;
++ptr;
address->dev_addr = (dev_addr_len > 0) ? dev_addr : NULL;
address->md_index = md_index;
address->dev_index = dev_index;
address->md_flags = md_flags;
address->iface_addr = (iface_addr_len > 0) ? ptr : NULL;
ptr += iface_addr_len;
if (ep_addr_present) {
ep_addr_len = *(uint8_t*)ptr;
address->ep_addr = (ep_addr_len > 0) ? ptr + 1 : NULL;
ptr += 1 + ep_addr_len;
} else {
address->ep_addr = NULL;
}
ucs_trace("unpack addr[%d] : md_flags 0x%"PRIx64" tl_flags 0x%"PRIx64" bw %e ovh %e "
"lat_ovh %e dev_priority %d",
(int)(address - address_list),
address->md_flags, address->iface_attr.cap_flags,
address->iface_attr.bandwidth, address->iface_attr.overhead,
address->iface_attr.lat_ovh,
address->iface_attr.priority);
++address;
}
++dev_index;
} while (!last_dev);
*address_count_p = address_count;
*address_list_p = address_list;
return UCS_OK;
}
static uct_ib_mem_t *uct_ib_memh_alloc()
{
return ucs_calloc(1, sizeof(uct_ib_mem_t), "ib_memh");
}