static ucs_status_t ucp_wireup_connect_local(ucp_ep_h ep, const uint8_t *tli,
unsigned address_count,
const ucp_address_entry_t *address_list)
{
ucp_worker_h worker = ep->worker;
const ucp_address_entry_t *address;
ucp_rsc_index_t rsc_index;
ucp_lane_index_t lane, amo_index;
ucs_status_t status;
ucp_md_map_t UCS_V_UNUSED md_map;
ucs_trace("ep %p: connect local transports", ep);
for (lane = 0; lane < ucp_ep_num_lanes(ep); ++lane) {
rsc_index = ucp_ep_get_rsc_index(ep, lane);
if (!ucp_worker_is_tl_p2p(worker, rsc_index)) {
continue;
}
address = &address_list[tli[lane]];
ucs_assert(address->tl_addr_len > 0);
/* Check that if the lane is used for RMA/AMO, destination md index matches */
md_map = ucp_lane_map_get_lane(ucp_ep_config(ep)->key.rma_lane_map, lane);
ucs_assertv((md_map == 0) || (md_map == UCS_BIT(address->md_index)),
"lane=%d ai=%d md_map=0x%x md_index=%d", lane, tli[lane],
md_map, address->md_index);
amo_index = ucp_ep_get_amo_lane_index(&ucp_ep_config(ep)->key, lane);
if (amo_index != UCP_NULL_LANE) {
md_map = ucp_lane_map_get_lane(ucp_ep_config(ep)->key.amo_lane_map,
amo_index);
ucs_assertv((md_map == 0) || (md_map == UCS_BIT(address->md_index)),
"lane=%d ai=%d md_map=0x%x md_index=%d", lane, tli[lane],
md_map, address->md_index);
}
status = uct_ep_connect_to_ep(ep->uct_eps[lane], address->dev_addr,
address->ep_addr);
if (status != UCS_OK) {
return status;
}
}
return UCS_OK;
}
ucp_md_map_t ucp_ep_config_get_amo_md_map(const ucp_ep_config_key_t *key,
ucp_lane_index_t lane)
{
ucp_lane_index_t amo_lane= ucp_ep_get_amo_lane_index(key, lane);
if (amo_lane != UCP_NULL_LANE) {
return ucp_lane_map_get_lane(key->amo_lane_map, amo_lane);
} else {
return 0;
}
}
static void ucp_wireup_print_config(ucp_context_h context,
const ucp_ep_config_key_t *key,
const char *title,
uint8_t *addr_indices)
{
char lane_info[128], *p, *endp;
ucp_lane_index_t lane, amo_index;
ucp_rsc_index_t rsc_index;
ucp_md_map_t md_map;
if (!ucs_log_enabled(UCS_LOG_LEVEL_DEBUG)) {
return;
}
ucs_debug("%s: am_lane %d wirep_lane %d rma_lane_map 0x%"PRIx64
" amo_lane_map 0x%"PRIx64" reachable_mds 0x%x",
title, key->am_lane, key->wireup_msg_lane,
key->rma_lane_map, key->amo_lane_map,
key->reachable_md_map);
for (lane = 0; lane < key->num_lanes; ++lane) {
p = lane_info;
endp = lane_info + sizeof(lane_info);
rsc_index = key->lanes[lane];
if (addr_indices != NULL) {
snprintf(p, endp - p, "->addr[%d] ", addr_indices[lane]);
p += strlen(p);
}
if (key->am_lane == lane) {
snprintf(p, endp - p, "[am]");
p += strlen(p);
}
md_map = ucp_lane_map_get_lane(key->rma_lane_map, lane);
if (md_map) {
snprintf(p, endp - p, "[rma->md%d]", ucs_ffs64(md_map));
p += strlen(p);
}
amo_index = ucp_ep_get_amo_lane_index(key, lane);
if (amo_index != UCP_NULL_LANE) {
md_map = ucp_lane_map_get_lane(key->amo_lane_map, amo_index);
if (md_map) {
snprintf(p, endp - p, "[amo[%d]->md%d]", amo_index, ucs_ffs64(md_map));
p += strlen(p);
}
}
if (key->wireup_msg_lane == lane) {
snprintf(p, endp - p, "[wireup]");
p += strlen(p);
}
ucs_debug("%s: lane[%d] using rsc[%d] "UCT_TL_RESOURCE_DESC_FMT " %s",
title, lane, rsc_index,
UCT_TL_RESOURCE_DESC_ARG(&context->tl_rscs[rsc_index].tl_rsc),
lane_info);
}
}
ucp_md_map_t ucp_ep_config_get_rma_md_map(const ucp_ep_config_key_t *key,
ucp_lane_index_t lane)
{
return ucp_lane_map_get_lane(key->rma_lane_map, lane);
}
void ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config)
{
ucp_context_h context = worker->context;
ucp_ep_rma_config_t *rma_config;
uct_iface_attr_t *iface_attr;
ucp_rsc_index_t rsc_index;
uct_md_attr_t *md_attr;
ucp_lane_index_t lane;
double zcopy_thresh, numerator, denumerator;
size_t rndv_thresh;
/* Default settings */
config->zcopy_thresh = SIZE_MAX;
config->sync_zcopy_thresh = -1;
config->bcopy_thresh = context->config.ext.bcopy_thresh;
config->rndv_thresh = SIZE_MAX;
config->sync_rndv_thresh = SIZE_MAX;
config->max_rndv_get_zcopy = SIZE_MAX;
config->p2p_lanes = 0;
/* Collect p2p lanes */
for (lane = 0; lane < config->key.num_lanes; ++lane) {
rsc_index = config->key.lanes[lane];
if ((rsc_index != UCP_NULL_RESOURCE) &&
ucp_worker_is_tl_p2p(worker, rsc_index))
{
config->p2p_lanes |= UCS_BIT(lane);
}
}
/* Configuration for active messages */
if (config->key.am_lane != UCP_NULL_LANE) {
lane = config->key.am_lane;
rsc_index = config->key.lanes[lane];
if (rsc_index != UCP_NULL_RESOURCE) {
iface_attr = &worker->iface_attrs[rsc_index];
md_attr = &context->md_attrs[context->tl_rscs[rsc_index].md_index];
if (iface_attr->cap.flags & UCT_IFACE_FLAG_AM_SHORT) {
config->max_eager_short = iface_attr->cap.am.max_short -
sizeof(ucp_eager_hdr_t);
config->max_am_short = iface_attr->cap.am.max_short -
sizeof(uint64_t);
}
if (iface_attr->cap.flags & UCT_IFACE_FLAG_AM_BCOPY) {
config->max_am_bcopy = iface_attr->cap.am.max_bcopy;
}
if ((iface_attr->cap.flags & UCT_IFACE_FLAG_AM_ZCOPY) &&
(md_attr->cap.flags & UCT_MD_FLAG_REG))
{
config->max_am_zcopy = iface_attr->cap.am.max_zcopy;
if (context->config.ext.zcopy_thresh == UCS_CONFIG_MEMUNITS_AUTO) {
/* auto */
zcopy_thresh = md_attr->reg_cost.overhead / (
(1.0 / context->config.ext.bcopy_bw) -
(1.0 / iface_attr->bandwidth) -
md_attr->reg_cost.growth);
if (zcopy_thresh < 0) {
config->zcopy_thresh = SIZE_MAX;
config->sync_zcopy_thresh = -1;
} else {
config->zcopy_thresh = zcopy_thresh;
config->sync_zcopy_thresh = zcopy_thresh;
}
} else {
config->zcopy_thresh = context->config.ext.zcopy_thresh;
config->sync_zcopy_thresh = context->config.ext.zcopy_thresh;
}
}
} else {
config->max_am_bcopy = UCP_MIN_BCOPY; /* Stub endpoint */
}
}
/* Configuration for remote memory access */
for (lane = 0; lane < config->key.num_lanes; ++lane) {
if (ucp_lane_map_get_lane(config->key.rma_lane_map, lane) == 0) {
continue;
}
rma_config = &config->rma[lane];
rsc_index = config->key.lanes[lane];
iface_attr = &worker->iface_attrs[rsc_index];
if (rsc_index != UCP_NULL_RESOURCE) {
if (iface_attr->cap.flags & UCT_IFACE_FLAG_PUT_SHORT) {
rma_config->max_put_short = iface_attr->cap.put.max_short;
}
if (iface_attr->cap.flags & UCT_IFACE_FLAG_PUT_BCOPY) {
rma_config->max_put_bcopy = iface_attr->cap.put.max_bcopy;
}
if (iface_attr->cap.flags & UCT_IFACE_FLAG_GET_BCOPY) {
rma_config->max_get_bcopy = iface_attr->cap.get.max_bcopy;
}
} else {
rma_config->max_put_bcopy = UCP_MIN_BCOPY; /* Stub endpoint */
}
}
/* Configuration for Rendezvous data */
if (config->key.rndv_lane != UCP_NULL_LANE) {
lane = config->key.rndv_lane;
rsc_index = config->key.lanes[lane];
if (rsc_index != UCP_NULL_RESOURCE) {
iface_attr = &worker->iface_attrs[rsc_index];
md_attr = &context->md_attrs[context->tl_rscs[rsc_index].md_index];
ucs_assert_always(iface_attr->cap.flags & UCT_IFACE_FLAG_GET_ZCOPY);
if (context->config.ext.rndv_thresh == UCS_CONFIG_MEMUNITS_AUTO) {
/* auto */
/* Make UCX calculate the rndv threshold on its own.
* We do it by finding the message size at which rndv and eager_zcopy get
* the same latency. Starting this message size (rndv_thresh), rndv's latency
* would be lower and the reached bandwidth would be higher.
* The latency function for eager_zcopy is:
* [ reg_cost.overhead + size * md_attr->reg_cost.growth +
* max(size/bw , size/bcopy_bw) + overhead ]
* The latency function for Rendezvous is:
* [ reg_cost.overhead + size * md_attr->reg_cost.growth + latency + overhead +
* reg_cost.overhead + size * md_attr->reg_cost.growth + overhead + latency +
* size/bw + latency + overhead + latency ]
* Isolating the 'size' yields the rndv_thresh.
* The used latency functions for eager_zcopy and rndv are also specified in
* the UCX wiki */
numerator = ((2 * iface_attr->overhead) + (4 * iface_attr->latency) +
md_attr->reg_cost.overhead);
denumerator = (ucs_max((1.0 / iface_attr->bandwidth),(1.0 / context->config.ext.bcopy_bw)) -
md_attr->reg_cost.growth - (1.0 / iface_attr->bandwidth));
if (denumerator > 0) {
rndv_thresh = numerator / denumerator;
ucs_trace("rendezvous threshold is %zu ( = %f / %f)",
rndv_thresh, numerator, denumerator);
} else {
rndv_thresh = context->config.ext.rndv_thresh_fallback;
ucs_trace("rendezvous threshold is %zu", rndv_thresh);
}
/* for the 'auto' mode in the rndv_threshold, we enforce the usage of rndv
* to a value that can be set by the user.
* to disable rndv, need to set a high value for the rndv_threshold
* (without the 'auto' mode)*/
config->rndv_thresh = rndv_thresh;
config->sync_rndv_thresh = rndv_thresh;
config->max_rndv_get_zcopy = iface_attr->cap.get.max_zcopy;
} else {
config->rndv_thresh = context->config.ext.rndv_thresh;
config->sync_rndv_thresh = context->config.ext.rndv_thresh;
config->max_rndv_get_zcopy = iface_attr->cap.get.max_zcopy;
ucs_trace("rendezvous threshold is %zu", config->rndv_thresh);
}
} else {
ucs_debug("rendezvous protocol is not supported ");
}
}
}
