static const char *
ucp_wireup_get_missing_flag_desc(uint64_t flags, uint64_t required_flags,
const char ** flag_descs)
{
ucs_assert((required_flags & (~flags)) != 0);
return flag_descs[ucs_ffs64(required_flags & (~flags))];
}
static void ucp_ep_flush_progress(ucp_request_t *req)
{
ucp_ep_h ep = req->send.ep;
ucp_lane_index_t lane;
ucs_status_t status;
uct_ep_h uct_ep;
ucs_trace("ep %p: progress flush req %p, lanes 0x%x count %d", ep, req,
req->send.flush.lanes, req->send.uct_comp.count);
while (req->send.flush.lanes) {
/* Search for next lane to start flush */
lane = ucs_ffs64(req->send.flush.lanes);
uct_ep = ep->uct_eps[lane];
if (uct_ep == NULL) {
req->send.flush.lanes &= ~UCS_BIT(lane);
--req->send.uct_comp.count;
continue;
}
/* Start flush operation on UCT endpoint */
status = uct_ep_flush(uct_ep, 0, &req->send.uct_comp);
ucs_trace("flushing ep %p lane[%d]: %s", ep, lane,
ucs_status_string(status));
if (status == UCS_OK) {
req->send.flush.lanes &= ~UCS_BIT(lane);
--req->send.uct_comp.count;
} else if (status == UCS_INPROGRESS) {
req->send.flush.lanes &= ~UCS_BIT(lane);
} else if (status == UCS_ERR_NO_RESOURCE) {
if (req->send.lane != UCP_NULL_LANE) {
ucs_trace("ep %p: not adding pending flush %p on lane %d, "
"because it's already pending on lane %d",
ep, req, lane, req->send.lane);
break;
}
req->send.lane = lane;
status = uct_ep_pending_add(uct_ep, &req->send.uct);
ucs_trace("adding pending flush on ep %p lane[%d]: %s", ep, lane,
ucs_status_string(status));
if (status == UCS_OK) {
req->send.flush.lanes &= ~UCS_BIT(lane);
} else if (status != UCS_ERR_BUSY) {
ucp_ep_flush_error(req, status);
}
} else {
ucp_ep_flush_error(req, status);
}
}
}
ucs_status_t ucp_ep_rkey_unpack(ucp_ep_h ep, void *rkey_buffer, ucp_rkey_h *rkey_p)
{
unsigned remote_pd_index, remote_pd_gap;
unsigned rkey_index;
unsigned pd_count;
ucs_status_t status;
ucp_rkey_h rkey;
uint8_t pd_size;
ucp_pd_map_t pd_map;
void *p;
/* Count the number of remote PDs in the rkey buffer */
p = rkey_buffer;
/* Read remote PD map */
pd_map = *(ucp_pd_map_t*)p;
ucs_trace("unpacking rkey with pd_map 0x%x", pd_map);
if (pd_map == 0) {
/* Dummy key return ok */
*rkey_p = &ucp_mem_dummy_rkey;
return UCS_OK;
}
pd_count = ucs_count_one_bits(pd_map);
p += sizeof(ucp_pd_map_t);
/* Allocate rkey handle which holds UCT rkeys for all remote PDs.
* We keep all of them to handle a future transport switch.
*/
rkey = ucs_malloc(sizeof(*rkey) + (sizeof(rkey->uct[0]) * pd_count), "ucp_rkey");
if (rkey == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err;
}
rkey->pd_map = 0;
remote_pd_index = 0; /* Index of remote PD */
rkey_index = 0; /* Index of the rkey in the array */
/* Unpack rkey of each UCT PD */
while (pd_map > 0) {
pd_size = *((uint8_t*)p++);
/* Use bit operations to iterate through the indices of the remote PDs
* as provided in the pd_map. pd_map always holds a bitmap of PD indices
* that remain to be used. Every time we find the "gap" until the next
* valid PD index using ffs operation. If some rkeys cannot be unpacked,
* we remove them from the local map.
*/
remote_pd_gap = ucs_ffs64(pd_map); /* Find the offset for next PD index */
remote_pd_index += remote_pd_gap; /* Calculate next index of remote PD*/
pd_map >>= remote_pd_gap; /* Remove the gap from the map */
ucs_assert(pd_map & 1);
/* Unpack only reachable rkeys */
if (UCS_BIT(remote_pd_index) & ucp_ep_config(ep)->key.reachable_pd_map) {
ucs_assert(rkey_index < pd_count);
status = uct_rkey_unpack(p, &rkey->uct[rkey_index]);
if (status != UCS_OK) {
ucs_error("Failed to unpack remote key from remote pd[%d]: %s",
remote_pd_index, ucs_status_string(status));
goto err_destroy;
}
ucs_trace("rkey[%d] for remote pd %d is 0x%lx", rkey_index,
remote_pd_index, rkey->uct[rkey_index].rkey);
rkey->pd_map |= UCS_BIT(remote_pd_index);
++rkey_index;
}
++remote_pd_index;
pd_map >>= 1;
p += pd_size;
}
if (rkey->pd_map == 0) {
ucs_debug("The unpacked rkey from the destination is unreachable");
status = UCS_ERR_UNREACHABLE;
goto err_destroy;
}
*rkey_p = rkey;
return UCS_OK;
err_destroy:
ucp_rkey_destroy(rkey);
err:
return status;
}
static void ucp_wireup_print_config(ucp_context_h context,
const ucp_ep_config_key_t *key,
const char *title,
uint8_t *addr_indices)
{
char lane_info[128], *p, *endp;
ucp_lane_index_t lane, amo_index;
ucp_rsc_index_t rsc_index;
ucp_md_map_t md_map;
if (!ucs_log_enabled(UCS_LOG_LEVEL_DEBUG)) {
return;
}
ucs_debug("%s: am_lane %d wirep_lane %d rma_lane_map 0x%"PRIx64
" amo_lane_map 0x%"PRIx64" reachable_mds 0x%x",
title, key->am_lane, key->wireup_msg_lane,
key->rma_lane_map, key->amo_lane_map,
key->reachable_md_map);
for (lane = 0; lane < key->num_lanes; ++lane) {
p = lane_info;
endp = lane_info + sizeof(lane_info);
rsc_index = key->lanes[lane];
if (addr_indices != NULL) {
snprintf(p, endp - p, "->addr[%d] ", addr_indices[lane]);
p += strlen(p);
}
if (key->am_lane == lane) {
snprintf(p, endp - p, "[am]");
p += strlen(p);
}
md_map = ucp_lane_map_get_lane(key->rma_lane_map, lane);
if (md_map) {
snprintf(p, endp - p, "[rma->md%d]", ucs_ffs64(md_map));
p += strlen(p);
}
amo_index = ucp_ep_get_amo_lane_index(key, lane);
if (amo_index != UCP_NULL_LANE) {
md_map = ucp_lane_map_get_lane(key->amo_lane_map, amo_index);
if (md_map) {
snprintf(p, endp - p, "[amo[%d]->md%d]", amo_index, ucs_ffs64(md_map));
p += strlen(p);
}
}
if (key->wireup_msg_lane == lane) {
snprintf(p, endp - p, "[wireup]");
p += strlen(p);
}
ucs_debug("%s: lane[%d] using rsc[%d] "UCT_TL_RESOURCE_DESC_FMT " %s",
title, lane, rsc_index,
UCT_TL_RESOURCE_DESC_ARG(&context->tl_rscs[rsc_index].tl_rsc),
lane_info);
}
}
static ucs_status_t ucp_add_tl_resources(ucp_context_h context,
uct_pd_h pd, ucp_rsc_index_t pd_index,
const ucp_config_t *config,
unsigned *num_resources_p)
{
uint64_t used_devices_mask, mask, config_devices_mask;
uct_tl_resource_desc_t *tl_resources;
ucp_tl_resource_desc_t *tmp;
unsigned num_resources;
ucs_status_t status;
ucp_rsc_index_t i;
*num_resources_p = 0;
/* check what are the available uct resources */
status = uct_pd_query_tl_resources(pd, &tl_resources, &num_resources);
if (status != UCS_OK) {
ucs_error("Failed to query resources: %s", ucs_status_string(status));
goto err;
}
if (num_resources == 0) {
ucs_debug("No tl resources found for pd %s", context->pd_rscs[pd_index].pd_name);
goto out_free_resources;
}
tmp = ucs_realloc(context->tl_rscs,
sizeof(*context->tl_rscs) * (context->num_tls + num_resources),
"ucp resources");
if (tmp == NULL) {
ucs_error("Failed to allocate resources");
status = UCS_ERR_NO_MEMORY;
goto err_free_resources;
}
/* mask of all devices from configuration which were used */
used_devices_mask = 0;
/* copy only the resources enabled by user configuration */
context->tl_rscs = tmp;
for (i = 0; i < num_resources; ++i) {
if (ucp_is_resource_enabled(&tl_resources[i], config, &mask)) {
context->tl_rscs[context->num_tls].tl_rsc = tl_resources[i];
context->tl_rscs[context->num_tls].pd_index = pd_index;
++context->num_tls;
used_devices_mask |= mask;
++(*num_resources_p);
}
}
/* if all devices should be used, check that */
config_devices_mask = UCS_MASK_SAFE(config->devices.count);
if (config->force_all_devices && (used_devices_mask != config_devices_mask)) {
i = ucs_ffs64(used_devices_mask ^ config_devices_mask);
ucs_error("device %s is not available", config->devices.names[i]);
status = UCS_ERR_NO_DEVICE;
goto err_free_resources;
}
out_free_resources:
uct_release_tl_resource_list(tl_resources);
return UCS_OK;
err_free_resources:
uct_release_tl_resource_list(tl_resources);
err:
return status;
}