/* Called with lock held */
static ucs_status_t ucm_mmap_test(int events)
{
static int installed_events = 0;
ucm_event_handler_t handler;
int out_events;
void *p;
if (ucs_test_all_flags(installed_events, events)) {
/* All requested events are already installed */
return UCS_OK;
}
/* Install a temporary event handler which will add the supported event
* type to out_events bitmap.
*/
handler.events = events;
handler.priority = -1;
handler.cb = ucm_mmap_event_test_callback;
handler.arg = &out_events;
out_events = 0;
ucm_event_handler_add(&handler);
if (events & (UCM_EVENT_MMAP|UCM_EVENT_MUNMAP|UCM_EVENT_MREMAP)) {
p = mmap(NULL, 0, 0, 0, -1 ,0);
p = mremap(p, 0, 0, 0);
munmap(p, 0);
}
if (events & (UCM_EVENT_SHMAT|UCM_EVENT_SHMDT)) {
p = shmat(0, NULL, 0);
shmdt(p);
}
if (events & UCM_EVENT_SBRK) {
(void)sbrk(0);
}
ucm_event_handler_remove(&handler);
/* TODO check address / stop all threads */
installed_events |= out_events;
ucm_debug("mmap test: got 0x%x out of 0x%x, total: 0x%x", out_events, events,
installed_events);
/* Return success iff we caught all wanted events */
if (!ucs_test_all_flags(out_events, events)) {
return UCS_ERR_UNSUPPORTED;
}
return UCS_OK;
}
void ucp_tag_eager_sync_completion(ucp_request_t *req, uint16_t flag)
{
static const uint16_t all_completed = UCP_REQUEST_FLAG_LOCAL_COMPLETED |
UCP_REQUEST_FLAG_REMOTE_COMPLETED;
ucs_assertv(!(req->flags & flag), "req->flags=%d flag=%d", req->flags, flag);
req->flags |= flag;
if (ucs_test_all_flags(req->flags, all_completed)) {
ucp_request_complete(req, req->cb.send, UCS_OK);
}
}
static int ucs_malloc_is_ready(int events)
{
/*
* If malloc hooks are installed - we're good here.
* Otherwise, we have to make sure all events are indeed working - because
* we can't be sure what the existing implementation is doing.
* The implication of this is that in some cases (e.g infinite mmap threshold)
* we will install out memory hooks, even though it may not be required.
*/
return ucm_malloc_hook_state.hook_called ||
ucs_test_all_flags(ucm_malloc_hook_state.installed_events, events);
}
static ucs_status_t ucp_wireup_add_am_lane(ucp_ep_h ep, unsigned address_count,
const ucp_address_entry_t *address_list,
ucp_wireup_lane_desc_t *lane_descs,
ucp_lane_index_t *num_lanes_p)
{
ucp_wireup_criteria_t criteria;
ucp_rsc_index_t rsc_index;
ucp_lane_index_t lane;
ucs_status_t status;
unsigned addr_index;
double score;
int need_am;
/* Check if we need active messages, for wireup */
if (!(ucp_ep_get_context_features(ep) & UCP_FEATURE_TAG)) {
need_am = 0;
for (lane = 0; lane < *num_lanes_p; ++lane) {
need_am = need_am || ucp_worker_is_tl_p2p(ep->worker,
lane_descs[lane].rsc_index);
}
if (!need_am) {
return UCS_OK;
}
}
/* Select one lane for active messages */
criteria.title = "active messages";
criteria.local_md_flags = 0;
criteria.remote_md_flags = 0;
criteria.remote_iface_flags = UCT_IFACE_FLAG_AM_BCOPY |
UCT_IFACE_FLAG_AM_CB_SYNC;
criteria.local_iface_flags = UCT_IFACE_FLAG_AM_BCOPY;
criteria.calc_score = ucp_wireup_am_score_func;
if (ucs_test_all_flags(ucp_ep_get_context_features(ep), UCP_FEATURE_TAG |
UCP_FEATURE_WAKEUP)) {
criteria.remote_iface_flags |= UCT_IFACE_FLAG_WAKEUP;
}
status = ucp_wireup_select_transport(ep, address_list, address_count, &criteria,
-1, 1, &rsc_index, &addr_index, &score);
if (status != UCS_OK) {
return status;
}
ucp_wireup_add_lane_desc(lane_descs, num_lanes_p, rsc_index, addr_index,
address_list[addr_index].md_index, score,
UCP_WIREUP_LANE_USAGE_AM);
return UCS_OK;
}
static ucs_status_t ucp_wireup_add_rndv_lane(ucp_ep_h ep, unsigned address_count,
const ucp_address_entry_t *address_list,
ucp_wireup_lane_desc_t *lane_descs,
ucp_lane_index_t *num_lanes_p)
{
ucp_wireup_criteria_t criteria;
ucp_rsc_index_t rsc_index;
ucs_status_t status;
unsigned addr_index;
double score;
if (!(ucp_ep_get_context_features(ep) & UCP_FEATURE_TAG)) {
return UCS_OK;
}
/* Select one lane for the Rendezvous protocol (for the actual data. not for rts) */
criteria.title = "rendezvous";
criteria.local_md_flags = UCT_MD_FLAG_REG;
criteria.remote_md_flags = UCT_MD_FLAG_REG; /* TODO not all ucts need reg on remote side */
criteria.remote_iface_flags = UCT_IFACE_FLAG_GET_ZCOPY;
criteria.local_iface_flags = UCT_IFACE_FLAG_GET_ZCOPY |
UCT_IFACE_FLAG_PENDING;
criteria.calc_score = ucp_wireup_rndv_score_func;
if (ucs_test_all_flags(ucp_ep_get_context_features(ep), UCP_FEATURE_WAKEUP)) {
criteria.remote_iface_flags |= UCT_IFACE_FLAG_WAKEUP;
}
status = ucp_wireup_select_transport(ep, address_list, address_count, &criteria,
-1, 0, &rsc_index, &addr_index, &score);
if ((status == UCS_OK) &&
/* a temporary workaround to prevent the ugni uct from using rndv */
(strstr(ep->worker->context->tl_rscs[rsc_index].tl_rsc.tl_name, "ugni") == NULL)) {
ucp_wireup_add_lane_desc(lane_descs, num_lanes_p, rsc_index, addr_index,
address_list[addr_index].md_index, score,
UCP_WIREUP_LANE_USAGE_RNDV);
}
return UCS_OK;
}
static int ucp_wireup_check_flags(const uct_tl_resource_desc_t *resource,
uint64_t flags, uint64_t required_flags,
const char *title, const char ** flag_descs,
char *reason, size_t max)
{
const char *missing_flag_desc;
if (ucs_test_all_flags(flags, required_flags)) {
return 1;
}
if (required_flags) {
missing_flag_desc = ucp_wireup_get_missing_flag_desc(flags, required_flags,
flag_descs);
ucs_trace(UCT_TL_RESOURCE_DESC_FMT " : not suitable for %s, no %s",
UCT_TL_RESOURCE_DESC_ARG(resource), title,
missing_flag_desc);
snprintf(reason, max, UCT_TL_RESOURCE_DESC_FMT" - no %s",
UCT_TL_RESOURCE_DESC_ARG(resource), missing_flag_desc);
}
return 0;
}
static ucs_status_t uct_perf_test_check_capabilities(ucx_perf_params_t *params,
uct_iface_h iface)
{
uct_iface_attr_t attr;
ucs_status_t status;
uint64_t required_flags;
size_t min_size, max_size, max_iov, message_size;
status = uct_iface_query(iface, &attr);
if (status != UCS_OK) {
return status;
}
min_size = 0;
max_iov = 1;
message_size = ucx_perf_get_message_size(params);
switch (params->command) {
case UCX_PERF_CMD_AM:
required_flags = __get_flag(params->uct.data_layout, UCT_IFACE_FLAG_AM_SHORT,
UCT_IFACE_FLAG_AM_BCOPY, UCT_IFACE_FLAG_AM_ZCOPY);
required_flags |= UCT_IFACE_FLAG_CB_SYNC;
min_size = __get_max_size(params->uct.data_layout, 0, 0,
attr.cap.am.min_zcopy);
max_size = __get_max_size(params->uct.data_layout, attr.cap.am.max_short,
attr.cap.am.max_bcopy, attr.cap.am.max_zcopy);
max_iov = attr.cap.am.max_iov;
break;
case UCX_PERF_CMD_PUT:
required_flags = __get_flag(params->uct.data_layout, UCT_IFACE_FLAG_PUT_SHORT,
UCT_IFACE_FLAG_PUT_BCOPY, UCT_IFACE_FLAG_PUT_ZCOPY);
min_size = __get_max_size(params->uct.data_layout, 0, 0,
attr.cap.put.min_zcopy);
max_size = __get_max_size(params->uct.data_layout, attr.cap.put.max_short,
attr.cap.put.max_bcopy, attr.cap.put.max_zcopy);
max_iov = attr.cap.put.max_iov;
break;
case UCX_PERF_CMD_GET:
required_flags = __get_flag(params->uct.data_layout, 0,
UCT_IFACE_FLAG_GET_BCOPY, UCT_IFACE_FLAG_GET_ZCOPY);
min_size = __get_max_size(params->uct.data_layout, 0, 0,
attr.cap.get.min_zcopy);
max_size = __get_max_size(params->uct.data_layout, 0,
attr.cap.get.max_bcopy, attr.cap.get.max_zcopy);
max_iov = attr.cap.get.max_iov;
break;
case UCX_PERF_CMD_ADD:
required_flags = __get_atomic_flag(message_size, UCT_IFACE_FLAG_ATOMIC_ADD32,
UCT_IFACE_FLAG_ATOMIC_ADD64);
max_size = 8;
break;
case UCX_PERF_CMD_FADD:
required_flags = __get_atomic_flag(message_size, UCT_IFACE_FLAG_ATOMIC_FADD32,
UCT_IFACE_FLAG_ATOMIC_FADD64);
max_size = 8;
break;
case UCX_PERF_CMD_SWAP:
required_flags = __get_atomic_flag(message_size, UCT_IFACE_FLAG_ATOMIC_SWAP32,
UCT_IFACE_FLAG_ATOMIC_SWAP64);
max_size = 8;
break;
case UCX_PERF_CMD_CSWAP:
required_flags = __get_atomic_flag(message_size, UCT_IFACE_FLAG_ATOMIC_CSWAP32,
UCT_IFACE_FLAG_ATOMIC_CSWAP64);
max_size = 8;
break;
default:
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Invalid test command");
}
return UCS_ERR_INVALID_PARAM;
}
status = ucx_perf_test_check_params(params);
if (status != UCS_OK) {
return status;
}
if (!ucs_test_all_flags(attr.cap.flags, required_flags) || !required_flags) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Device does not support required operation");
}
return UCS_ERR_UNSUPPORTED;
}
if (message_size < min_size) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Message size too small");
}
return UCS_ERR_UNSUPPORTED;
}
if (message_size > max_size) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Message size too big");
}
return UCS_ERR_UNSUPPORTED;
}
if (params->command == UCX_PERF_CMD_AM) {
if ((params->uct.data_layout == UCT_PERF_DATA_LAYOUT_SHORT) &&
(params->am_hdr_size != sizeof(uint64_t)))
{
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Short AM header size must be 8 bytes");
}
return UCS_ERR_INVALID_PARAM;
}
if ((params->uct.data_layout == UCT_PERF_DATA_LAYOUT_ZCOPY) &&
(params->am_hdr_size > attr.cap.am.max_hdr))
{
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("AM header size too big");
}
return UCS_ERR_UNSUPPORTED;
}
if (params->am_hdr_size > message_size) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("AM header size larger than message size");
}
return UCS_ERR_INVALID_PARAM;
}
if (params->uct.fc_window > UCT_PERF_TEST_MAX_FC_WINDOW) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("AM flow-control window too large (should be <= %d)",
UCT_PERF_TEST_MAX_FC_WINDOW);
}
return UCS_ERR_INVALID_PARAM;
}
if ((params->flags & UCX_PERF_TEST_FLAG_ONE_SIDED) &&
(params->flags & UCX_PERF_TEST_FLAG_VERBOSE))
{
ucs_warn("Running active-message test with on-sided progress");
}
}
if (UCT_PERF_DATA_LAYOUT_ZCOPY == params->uct.data_layout) {
if (params->msg_size_cnt > max_iov) {
if ((params->flags & UCX_PERF_TEST_FLAG_VERBOSE) ||
!params->msg_size_cnt) {
ucs_error("Wrong number of IOV entries. Requested is %lu, "
"should be in the range 1...%lu", params->msg_size_cnt,
max_iov);
}
return UCS_ERR_UNSUPPORTED;
}
/* if msg_size_cnt == 1 the message size checked above */
if ((UCX_PERF_CMD_AM == params->command) && (params->msg_size_cnt > 1)) {
if (params->am_hdr_size > params->msg_size_list[0]) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("AM header size (%lu) larger than the first IOV "
"message size (%lu)", params->am_hdr_size,
params->msg_size_list[0]);
}
return UCS_ERR_INVALID_PARAM;
}
}
}
return UCS_OK;
}
/**
* Select a local and remote transport
*/
static UCS_F_NOINLINE ucs_status_t
ucp_wireup_select_transport(ucp_ep_h ep, const ucp_address_entry_t *address_list,
unsigned address_count, const ucp_wireup_criteria_t *criteria,
uint64_t remote_md_map, int show_error,
ucp_rsc_index_t *rsc_index_p, unsigned *dst_addr_index_p,
double *score_p)
{
ucp_worker_h worker = ep->worker;
ucp_context_h context = worker->context;
uct_tl_resource_desc_t *resource;
const ucp_address_entry_t *ae;
ucp_rsc_index_t rsc_index;
double score, best_score;
char tls_info[256];
char *p, *endp;
uct_iface_attr_t *iface_attr;
uct_md_attr_t *md_attr;
uint64_t addr_index_map;
unsigned addr_index;
int reachable;
int found;
found = 0;
best_score = 0.0;
p = tls_info;
endp = tls_info + sizeof(tls_info) - 1;
tls_info[0] = '\0';
/* Check which remote addresses satisfy the criteria */
addr_index_map = 0;
for (ae = address_list; ae < address_list + address_count; ++ae) {
addr_index = ae - address_list;
if (!(remote_md_map & UCS_BIT(ae->md_index))) {
ucs_trace("addr[%d]: not in use, because on md[%d]", addr_index,
ae->md_index);
continue;
}
if (!ucs_test_all_flags(ae->md_flags, criteria->remote_md_flags)) {
ucs_trace("addr[%d]: no %s", addr_index,
ucp_wireup_get_missing_flag_desc(ae->md_flags,
criteria->remote_md_flags,
ucp_wireup_md_flags));
continue;
}
if (!ucs_test_all_flags(ae->iface_attr.cap_flags, criteria->remote_iface_flags)) {
ucs_trace("addr[%d]: no %s", addr_index,
ucp_wireup_get_missing_flag_desc(ae->iface_attr.cap_flags,
criteria->remote_iface_flags,
ucp_wireup_iface_flags));
continue;
}
addr_index_map |= UCS_BIT(addr_index);
}
/* For each local resource try to find the best remote address to connect to.
* Pick the best local resource to satisfy the criteria.
* best one has the highest score (from the dedicated score_func) and
* has a reachable tl on the remote peer */
for (rsc_index = 0; rsc_index < context->num_tls; ++rsc_index) {
resource = &context->tl_rscs[rsc_index].tl_rsc;
iface_attr = &worker->iface_attrs[rsc_index];
md_attr = &context->md_attrs[context->tl_rscs[rsc_index].md_index];
/* Check that local md and interface satisfy the criteria */
if (!ucp_wireup_check_flags(resource, md_attr->cap.flags,
criteria->local_md_flags, criteria->title,
ucp_wireup_md_flags, p, endp - p) ||
!ucp_wireup_check_flags(resource, iface_attr->cap.flags,
criteria->local_iface_flags, criteria->title,
ucp_wireup_iface_flags, p, endp - p))
{
p += strlen(p);
snprintf(p, endp - p, ", ");
p += strlen(p);
continue;
}
reachable = 0;
for (ae = address_list; ae < address_list + address_count; ++ae) {
if (!(addr_index_map & UCS_BIT(ae - address_list)) ||
!ucp_wireup_is_reachable(worker, rsc_index, ae))
{
/* Must be reachable device address, on same transport */
continue;
}
reachable = 1;
score = criteria->calc_score(md_attr, iface_attr, &ae->iface_attr);
ucs_assert(score >= 0.0);
ucs_trace(UCT_TL_RESOURCE_DESC_FMT "->addr[%zd] : %s score %.2f",
UCT_TL_RESOURCE_DESC_ARG(resource), ae - address_list,
criteria->title, score);
if (!found || (score > best_score)) {
*rsc_index_p = rsc_index;
*dst_addr_index_p = ae - address_list;
*score_p = score;
best_score = score;
found = 1;
}
}
/* If a local resource cannot reach any of the remote addresses, generate
* debug message.
*/
if (!reachable) {
snprintf(p, endp - p, UCT_TL_RESOURCE_DESC_FMT" - cannot reach remote worker, ",
UCT_TL_RESOURCE_DESC_ARG(resource));
p += strlen(p);
}
}
if (p >= tls_info + 2) {
*(p - 2) = '\0'; /* trim last "," */
}
if (!found) {
if (show_error) {
ucs_error("No %s transport to %s: %s", criteria->title,
ucp_ep_peer_name(ep), tls_info);
}
return UCS_ERR_UNREACHABLE;
}
ucs_trace("ep %p: selected for %s: " UCT_TL_RESOURCE_DESC_FMT
" -> '%s' address[%d],md[%d] score %.2f", ep, criteria->title,
UCT_TL_RESOURCE_DESC_ARG(&context->tl_rscs[*rsc_index_p].tl_rsc),
ucp_ep_peer_name(ep), *dst_addr_index_p,
address_list[*dst_addr_index_p].md_index, best_score);
return UCS_OK;
}