int ucs_get_first_cpu()
{
int first_cpu, total_cpus, ret;
cpu_set_t mask;
ret = sysconf(_SC_NPROCESSORS_CONF);
if (ret < 0) {
ucs_error("failed to get local cpu count: %m");
return ret;
}
total_cpus = ret;
CPU_ZERO(&mask);
ret = sched_getaffinity(0, sizeof(mask), &mask);
if (ret < 0) {
ucs_error("failed to get process affinity: %m");
return ret;
}
for (first_cpu = 0; first_cpu < total_cpus; ++first_cpu) {
if (CPU_ISSET(first_cpu, &mask)) {
return first_cpu;
}
}
return total_cpus;
}
static ucs_status_t uct_posix_shm_open(const char *file_name, size_t length, int *shm_fd)
{
ucs_status_t status;
/* Create shared memory object and set its size */
*shm_fd = shm_open(file_name, O_CREAT | O_RDWR | O_EXCL,
UCT_MM_POSIX_SHM_OPEN_MODE);
if (*shm_fd == -1) {
ucs_error("Error returned from shm_open %m. File name is: %s",
file_name);
status = UCS_ERR_SHMEM_SEGMENT;
goto err;
}
if (ftruncate(*shm_fd, length) == -1) {
ucs_error("Error returned from ftruncate %m");
status = UCS_ERR_SHMEM_SEGMENT;
goto err_shm_unlink;
}
return UCS_OK;
err_shm_unlink:
close(*shm_fd);
if (shm_unlink(file_name) != 0) {
ucs_warn("unable to shm_unlink the shared memory segment");
}
err:
return status;
}
static ucs_status_t ucx_perf_test_check_params(ucx_perf_params_t *params)
{
size_t it;
if (ucx_perf_get_message_size(params) < 1) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Message size too small, need to be at least 1");
}
return UCS_ERR_INVALID_PARAM;
}
if (params->max_outstanding < 1) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("max_outstanding, need to be at least 1");
}
return UCS_ERR_INVALID_PARAM;
}
/* check if particular message size fit into stride size */
if (params->iov_stride) {
for (it = 0; it < params->msg_size_cnt; ++it) {
if (params->msg_size_list[it] > params->iov_stride) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Buffer size %lu bigger than stride %lu",
params->msg_size_list[it], params->iov_stride);
}
return UCS_ERR_INVALID_PARAM;
}
}
}
return UCS_OK;
}
static ucs_status_t uct_posix_open(const char *file_name, size_t length, int *shm_fd)
{
ucs_status_t status;
/* use open with the given path */
*shm_fd = open(file_name, O_CREAT | O_RDWR | O_EXCL, UCT_MM_POSIX_SHM_OPEN_MODE);
if (*shm_fd == -1) {
ucs_error("Error returned from open %m . File name is: %s", file_name);
status = UCS_ERR_SHMEM_SEGMENT;
goto err;
}
if (ftruncate(*shm_fd, length) == -1) {
ucs_error("Error returned from ftruncate %m");
status = UCS_ERR_SHMEM_SEGMENT;
goto err_close;
}
return UCS_OK;
err_close:
close(*shm_fd);
if (unlink(file_name) != 0) {
ucs_warn("unable to unlink the shared memory segment");
}
err:
return status;
}
ucs_status_t ugni_activate_iface(uct_ugni_iface_t *iface)
{
ucs_status_t status;
gni_return_t ugni_rc;
uint32_t pe_address;
if(iface->activated) {
return UCS_OK;
}
status = uct_ugni_init_nic(0, &iface->domain_id,
&iface->cdm_handle, &iface->nic_handle,
&pe_address);
if (UCS_OK != status) {
ucs_error("Failed to UGNI NIC, Error status: %d", status);
return status;
}
ucs_debug("Made ugni interface. iface->dev->nic_addr = %i iface->domain_id = %i", iface->dev->address, iface->domain_id);
ugni_rc = GNI_CqCreate(iface->nic_handle, UCT_UGNI_LOCAL_CQ, 0,
GNI_CQ_NOBLOCK,
NULL, NULL, &iface->local_cq);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CqCreate failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
iface->activated = true;
/* iface is activated */
return UCS_OK;
}
/* add new handler to the table */
static ucs_status_t ucs_async_handler_add(ucs_async_handler_t *handler)
{
int hash_extra_status;
ucs_status_t status;
khiter_t hash_it;
pthread_rwlock_wrlock(&ucs_async_global_context.handlers_lock);
ucs_assert_always(handler->refcount == 1);
hash_it = kh_put(ucs_async_handler, &ucs_async_global_context.handlers,
handler->id, &hash_extra_status);
if (hash_extra_status == -1) {
ucs_error("Failed to add async handler " UCS_ASYNC_HANDLER_FMT " to hash",
UCS_ASYNC_HANDLER_ARG(handler));
status = UCS_ERR_NO_MEMORY;
goto out_unlock;
} else if (hash_extra_status == 0) {
ucs_error("Async handler " UCS_ASYNC_HANDLER_FMT " exists - cannot add %s()",
UCS_ASYNC_HANDLER_ARG(kh_value(&ucs_async_global_context.handlers, hash_it)),
ucs_debug_get_symbol_name(handler->cb));
status = UCS_ERR_ALREADY_EXISTS;
goto out_unlock;
}
ucs_assert_always(!ucs_async_handler_kh_is_end(hash_it));
kh_value(&ucs_async_global_context.handlers, hash_it) = handler;
ucs_debug("added async handler " UCS_ASYNC_HANDLER_FMT " to hash",
UCS_ASYNC_HANDLER_ARG(handler));
status = UCS_OK;
out_unlock:
pthread_rwlock_unlock(&ucs_async_global_context.handlers_lock);
return status;
}
ucs_status_t ugni_activate_iface(uct_ugni_iface_t *iface)
{
int rc;
gni_return_t ugni_rc;
if(iface->activated) {
return UCS_OK;
}
rc = uct_ugni_init_nic(0, &iface->domain_id,
&iface->cdm_handle, &iface->nic_handle,
&iface->pe_address);
if (UCS_OK != rc) {
ucs_error("Failed to UGNI NIC, Error status: %d", rc);
return rc;
}
ugni_rc = GNI_CqCreate(iface->nic_handle, UCT_UGNI_LOCAL_CQ, 0,
GNI_CQ_NOBLOCK,
NULL, NULL, &iface->local_cq);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CqCreate failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
iface->activated = true;
/* iface is activated */
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_self_iface_t, uct_md_h md, uct_worker_h worker,
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
ucs_status_t status;
uct_self_iface_config_t *self_config = 0;
ucs_trace_func("Creating a loop-back transport self=%p rxh=%lu",
self, params->rx_headroom);
if (strcmp(params->dev_name, UCT_SELF_NAME) != 0) {
ucs_error("No device was found: %s", params->dev_name);
return UCS_ERR_NO_DEVICE;
}
UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_self_iface_ops, md, worker,
tl_config UCS_STATS_ARG(params->stats_root)
UCS_STATS_ARG(UCT_SELF_NAME));
self_config = ucs_derived_of(tl_config, uct_self_iface_config_t);
self->id = ucs_generate_uuid((uintptr_t)self);
self->rx_headroom = params->rx_headroom;
self->data_length = self_config->super.max_bcopy;
self->release_desc.cb = uct_self_iface_release_desc;
/* create a memory pool for data transferred */
status = uct_iface_mpool_init(&self->super,
&self->msg_desc_mp,
sizeof(uct_recv_desc_t) + self->rx_headroom +
self->data_length,
sizeof(uct_recv_desc_t) + self->rx_headroom,
UCS_SYS_CACHE_LINE_SIZE,
&self_config->mp,
256,
ucs_empty_function,
"self_msg_desc");
if (UCS_OK != status) {
ucs_error("Failed to create a memory pool for the loop-back transport");
goto err;
}
/* set the message descriptor for the loop-back */
self->msg_cur_desc = ucs_mpool_get(&self->msg_desc_mp);
VALGRIND_MAKE_MEM_DEFINED(self->msg_cur_desc, sizeof(*(self->msg_cur_desc)));
if (NULL == self->msg_cur_desc) {
ucs_error("Failed to get the first descriptor in loop-back MP storage");
status = UCS_ERR_NO_RESOURCE;
goto destroy_mpool;
}
ucs_debug("Created a loop-back iface. id=0x%lx, desc=%p, len=%u, tx_hdr=%lu",
self->id, self->msg_cur_desc, self->data_length, self->rx_headroom);
return UCS_OK;
destroy_mpool:
ucs_mpool_cleanup(&self->msg_desc_mp, 1);
err:
return status;
}
static ucs_status_t recieve_datagram(uct_ugni_udt_iface_t *iface, uint64_t id, uct_ugni_udt_ep_t **ep_out)
{
uint32_t rem_addr, rem_id;
gni_post_state_t post_state;
gni_return_t ugni_rc;
uct_ugni_udt_ep_t *ep;
gni_ep_handle_t gni_ep;
uct_ugni_udt_desc_t *desc;
uct_ugni_udt_header_t *header;
ucs_trace_func("iface=%p, id=%lx", iface, id);
if (UCT_UGNI_UDT_ANY == id) {
ep = NULL;
gni_ep = iface->ep_any;
desc = iface->desc_any;
} else {
ep = ucs_derived_of(uct_ugni_iface_lookup_ep(&iface->super, id),
uct_ugni_udt_ep_t);
gni_ep = ep->super.ep;
desc = ep->posted_desc;
}
*ep_out = ep;
uct_ugni_device_lock(&iface->super.cdm);
ugni_rc = GNI_EpPostDataWaitById(gni_ep, id, -1, &post_state, &rem_addr, &rem_id);
uct_ugni_device_unlock(&iface->super.cdm);
if (ucs_unlikely(GNI_RC_SUCCESS != ugni_rc)) {
ucs_error("GNI_EpPostDataWaitById, id=%lu Error status: %s %d",
id, gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
if (GNI_POST_TERMINATED == post_state) {
return UCS_ERR_CANCELED;
}
if (GNI_POST_COMPLETED != post_state) {
ucs_error("GNI_EpPostDataWaitById gave unexpected response: %u", post_state);
return UCS_ERR_IO_ERROR;
}
if (UCT_UGNI_UDT_ANY != id) {
--iface->super.outstanding;
}
header = uct_ugni_udt_get_rheader(desc, iface);
ucs_trace("Got datagram id: %lu type: %i len: %i am_id: %i", id, header->type, header->length, header->am_id);
if (UCT_UGNI_UDT_PAYLOAD != header->type) {
/* ack message, no data */
ucs_assert_always(NULL != ep);
ucs_mpool_put(ep->posted_desc);
uct_ugni_check_flush(ep->desc_flush_group);
ep->posted_desc = NULL;
return UCS_OK;
}
return UCS_INPROGRESS;
}
static ucs_status_t uct_ugni_rkey_unpack(uct_md_component_t *mdc, const void *rkey_buffer,
uct_rkey_t *rkey_p, void **handle_p)
{
const uint64_t *ptr = rkey_buffer;
gni_mem_handle_t *mem_hndl = NULL;
uint64_t magic = 0;
ucs_debug("Unpacking [ %"PRIx64" %"PRIx64" %"PRIx64"]", ptr[0], ptr[1], ptr[2]);
magic = ptr[0];
if (magic != UCT_UGNI_RKEY_MAGIC) {
ucs_error("Failed to identify key. Expected %llx but received %"PRIx64"",
UCT_UGNI_RKEY_MAGIC, magic);
return UCS_ERR_UNSUPPORTED;
}
mem_hndl = ucs_malloc(sizeof(gni_mem_handle_t), "gni_mem_handle_t");
if (NULL == mem_hndl) {
ucs_error("Failed to allocate memory for gni_mem_handle_t");
return UCS_ERR_NO_MEMORY;
}
mem_hndl->qword1 = ptr[1];
mem_hndl->qword2 = ptr[2];
*rkey_p = (uintptr_t)mem_hndl;
*handle_p = NULL;
return UCS_OK;
}
static ucs_status_t uct_xpmem_detach(uct_mm_remote_seg_t *mm_desc)
{
xpmem_apid_t apid = mm_desc->cookie;
void *address;
int ret;
address = ucs_align_down_pow2_ptr(mm_desc->address, ucs_get_page_size());
ucs_trace("xpmem detaching address %p", address);
ret = xpmem_detach(address);
if (ret < 0) {
ucs_error("Failed to xpmem_detach: %m");
return UCS_ERR_IO_ERROR;
}
VALGRIND_MAKE_MEM_UNDEFINED(mm_desc->address, mm_desc->length);
ucs_trace("xpmem releasing segment apid 0x%llx", apid);
ret = xpmem_release(apid);
if (ret < 0) {
ucs_error("Failed to release xpmem segment apid 0x%llx", apid);
return UCS_ERR_IO_ERROR;
}
return UCS_OK;
}
static void uct_ugni_udt_clean_wildcard(uct_ugni_udt_iface_t *iface)
{
gni_return_t ugni_rc;
uint32_t rem_addr, rem_id;
gni_post_state_t post_state;
uct_ugni_device_lock(&iface->super.cdm);
ugni_rc = GNI_EpPostDataCancelById(iface->ep_any, UCT_UGNI_UDT_ANY);
if (GNI_RC_SUCCESS != ugni_rc) {
uct_ugni_device_unlock(&iface->super.cdm);
ucs_error("GNI_EpPostDataCancel failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return;
}
ugni_rc = GNI_EpPostDataTestById(iface->ep_any, UCT_UGNI_UDT_ANY, &post_state, &rem_addr, &rem_id);
if (GNI_RC_SUCCESS != ugni_rc) {
if (GNI_RC_NO_MATCH != ugni_rc) {
uct_ugni_device_unlock(&iface->super.cdm);
ucs_error("GNI_EpPostDataTestById failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return;
}
} else {
if (GNI_POST_PENDING == post_state) {
ugni_rc = GNI_EpPostDataWaitById(iface->ep_any, UCT_UGNI_UDT_ANY, -1, &post_state, &rem_addr, &rem_id);
}
}
ugni_rc = GNI_EpDestroy(iface->ep_any);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_EpDestroy failed, Error status: %s %d\n",
gni_err_str[ugni_rc], ugni_rc);
}
uct_ugni_device_unlock(&iface->super.cdm);
}
static ucs_status_t uct_ugni_smsg_mbox_reg(uct_ugni_smsg_iface_t *iface, uct_ugni_smsg_mbox_t *mbox)
{
gni_return_t ugni_rc;
void *address = (mbox+1);
if (0 == iface->bytes_per_mbox) {
ucs_error("Unexpected length %zu", iface->bytes_per_mbox);
return UCS_ERR_INVALID_PARAM;
}
uct_ugni_cdm_lock(&iface->super.cdm);
ugni_rc = GNI_MemRegister(uct_ugni_iface_nic_handle(&iface->super), (uint64_t)address,
iface->bytes_per_mbox, iface->remote_cq,
GNI_MEM_READWRITE,
-1, &(mbox->gni_mem));
uct_ugni_cdm_unlock(&iface->super.cdm);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_MemRegister failed (addr %p, size %zu), Error status: %s %d",
address, iface->bytes_per_mbox, gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
mbox->base_address = (uintptr_t)address;
return UCS_OK;
}
ucs_status_t ucx_perf_run(ucx_perf_params_t *params, ucx_perf_result_t *result)
{
ucx_perf_context_t *perf;
ucs_status_t status;
if (params->command == UCX_PERF_CMD_LAST) {
ucs_error("Test is not selected");
status = UCS_ERR_INVALID_PARAM;
goto out;
}
if ((params->api != UCX_PERF_API_UCT) && (params->api != UCX_PERF_API_UCP)) {
ucs_error("Invalid test API parameter (should be UCT or UCP)");
status = UCS_ERR_INVALID_PARAM;
goto out;
}
perf = malloc(sizeof(*perf));
if (perf == NULL) {
status = UCS_ERR_NO_MEMORY;
goto out;
}
ucx_perf_test_reset(perf, params);
status = ucx_perf_funcs[params->api].setup(perf, params);
if (status != UCS_OK) {
goto out_free;
}
if (UCS_THREAD_MODE_SINGLE == params->thread_mode) {
if (params->warmup_iter > 0) {
ucx_perf_set_warmup(perf, params);
status = ucx_perf_funcs[params->api].run(perf);
if (status != UCS_OK) {
goto out_cleanup;
}
rte_call(perf, barrier);
ucx_perf_test_reset(perf, params);
}
/* Run test */
status = ucx_perf_funcs[params->api].run(perf);
rte_call(perf, barrier);
if (status == UCS_OK) {
ucx_perf_calc_result(perf, result);
rte_call(perf, report, result, perf->params.report_arg, 1);
}
} else {
status = ucx_perf_thread_spawn(perf, result);
}
out_cleanup:
ucx_perf_funcs[params->api].cleanup(perf);
out_free:
free(perf);
out:
return status;
}
static void process_mbox(uct_ugni_smsg_iface_t *iface, uct_ugni_smsg_ep_t *ep){
ucs_status_t status;
uint8_t tag;
void *data_ptr;
gni_return_t ugni_rc;
uct_ugni_smsg_header_t *header;
void *user_data;
pthread_mutex_lock(&uct_ugni_global_lock);
while(1){
tag = GNI_SMSG_ANY_TAG;
ugni_rc = GNI_SmsgGetNextWTag(ep->super.ep, (void **)&data_ptr, &tag);
/* Yes, GNI_RC_NOT_DONE means that you're done with the smsg mailbox */
if(GNI_RC_NOT_DONE == ugni_rc){
pthread_mutex_unlock(&uct_ugni_global_lock);
return;
}
if(GNI_RC_SUCCESS != ugni_rc){
ucs_error("Unhandled smsg error: %s %d", gni_err_str[ugni_rc], ugni_rc);
pthread_mutex_unlock(&uct_ugni_global_lock);
return;
}
if(NULL == data_ptr){
ucs_error("Empty data pointer in smsg.");
pthread_mutex_unlock(&uct_ugni_global_lock);
return;
}
header = (uct_ugni_smsg_header_t *)data_ptr;
user_data = (void *)(header + 1);
void *user_desc = iface->user_desc+1;
uct_iface_trace_am(&iface->super.super, UCT_AM_TRACE_TYPE_RECV,
tag, user_data, header->length, "RX: AM");
pthread_mutex_unlock(&uct_ugni_global_lock);
status = uct_iface_invoke_am(&iface->super.super, tag, user_data,
header->length, user_desc);
pthread_mutex_lock(&uct_ugni_global_lock);
if(status != UCS_OK){
uct_recv_desc_iface(user_desc) = &iface->super.super.super;
UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_desc,
iface->user_desc, iface->user_desc = NULL);
}
ugni_rc = GNI_SmsgRelease(ep->super.ep);
if(GNI_RC_SUCCESS != ugni_rc){
ucs_error("Unhandled smsg error in GNI_SmsgRelease: %s %d", gni_err_str[ugni_rc], ugni_rc);
pthread_mutex_unlock(&uct_ugni_global_lock);
return;
}
}
}
ucs_status_t ucp_ep_new(ucp_worker_h worker, uint64_t dest_uuid,
const char *peer_name, const char *message,
ucp_ep_h *ep_p)
{
ucs_status_t status;
ucp_ep_config_key_t key;
ucp_ep_h ep;
khiter_t hash_it;
int hash_extra_status = 0;
ep = ucs_calloc(1, sizeof(*ep), "ucp ep");
if (ep == NULL) {
ucs_error("Failed to allocate ep");
status = UCS_ERR_NO_MEMORY;
goto err;
}
/* EP configuration without any lanes */
memset(&key, 0, sizeof(key));
key.rma_lane_map = 0;
key.amo_lane_map = 0;
key.reachable_md_map = 0;
key.am_lane = UCP_NULL_RESOURCE;
key.rndv_lane = UCP_NULL_RESOURCE;
key.wireup_msg_lane = UCP_NULL_LANE;
key.num_lanes = 0;
memset(key.amo_lanes, UCP_NULL_LANE, sizeof(key.amo_lanes));
ep->worker = worker;
ep->dest_uuid = dest_uuid;
ep->cfg_index = ucp_worker_get_ep_config(worker, &key);
ep->am_lane = UCP_NULL_LANE;
ep->flags = 0;
#if ENABLE_DEBUG_DATA
ucs_snprintf_zero(ep->peer_name, UCP_WORKER_NAME_MAX, "%s", peer_name);
#endif
hash_it = kh_put(ucp_worker_ep_hash, &worker->ep_hash, dest_uuid,
&hash_extra_status);
if (ucs_unlikely(hash_it == kh_end(&worker->ep_hash))) {
ucs_error("Hash failed with ep %p to %s 0x%"PRIx64"->0x%"PRIx64" %s "
"with status %d", ep, peer_name, worker->uuid, ep->dest_uuid,
message, hash_extra_status);
status = UCS_ERR_NO_RESOURCE;
goto err_free_ep;
}
kh_value(&worker->ep_hash, hash_it) = ep;
*ep_p = ep;
ucs_debug("created ep %p to %s 0x%"PRIx64"->0x%"PRIx64" %s", ep, peer_name,
worker->uuid, ep->dest_uuid, message);
return UCS_OK;
err_free_ep:
ucs_free(ep);
err:
return status;
}
static ucs_status_t
uct_gdr_copy_mem_reg_internal(uct_md_h uct_md, void *address, size_t length,
unsigned flags, uct_gdr_copy_mem_t *mem_hndl)
{
uct_gdr_copy_md_t *md = ucs_derived_of(uct_md, uct_gdr_copy_md_t);
CUdeviceptr d_ptr = ((CUdeviceptr )(char *) address);
gdr_mh_t mh;
void *bar_ptr;
gdr_info_t info;
int ret;
if (!length) {
mem_hndl->mh = 0;
return UCS_OK;
}
ret = gdr_pin_buffer(md->gdrcpy_ctx, d_ptr, length, 0, 0, &mh);
if (ret) {
ucs_error("gdr_pin_buffer failed. length :%lu ret:%d", length, ret);
goto err;
}
ret = gdr_map(md->gdrcpy_ctx, mh, &bar_ptr, length);
if (ret) {
ucs_error("gdr_map failed. length :%lu ret:%d", length, ret);
goto unpin_buffer;
}
ret = gdr_get_info(md->gdrcpy_ctx, mh, &info);
if (ret) {
ucs_error("gdr_get_info failed. ret:%d", ret);
goto unmap_buffer;
}
mem_hndl->mh = mh;
mem_hndl->info = info;
mem_hndl->bar_ptr = bar_ptr;
mem_hndl->reg_size = length;
ucs_trace("registered memory:%p..%p length:%lu info.va:0x%"PRIx64" bar_ptr:%p",
address, address + length, length, info.va, bar_ptr);
return UCS_OK;
unmap_buffer:
ret = gdr_unmap(md->gdrcpy_ctx, mem_hndl->mh, mem_hndl->bar_ptr, mem_hndl->reg_size);
if (ret) {
ucs_warn("gdr_unmap failed. unpin_size:%lu ret:%d", mem_hndl->reg_size, ret);
}
unpin_buffer:
ret = gdr_unpin_buffer(md->gdrcpy_ctx, mh);
if (ret) {
ucs_warn("gdr_unpin_buffer failed. ret;%d", ret);
}
err:
return UCS_ERR_IO_ERROR;
}
static ucs_status_t ucp_add_tl_resources(ucp_context_h context,
uct_pd_h pd, ucp_rsc_index_t pd_index,
const ucp_config_t *config,
unsigned *num_resources_p,
uint64_t *masks)
{
uct_tl_resource_desc_t *tl_resources;
ucp_tl_resource_desc_t *tmp;
unsigned num_resources;
ucs_status_t status;
ucp_rsc_index_t i;
*num_resources_p = 0;
/* check what are the available uct resources */
status = uct_pd_query_tl_resources(pd, &tl_resources, &num_resources);
if (status != UCS_OK) {
ucs_error("Failed to query resources: %s", ucs_status_string(status));
goto err;
}
if (num_resources == 0) {
ucs_debug("No tl resources found for pd %s", context->pd_rscs[pd_index].pd_name);
goto out_free_resources;
}
tmp = ucs_realloc(context->tl_rscs,
sizeof(*context->tl_rscs) * (context->num_tls + num_resources),
"ucp resources");
if (tmp == NULL) {
ucs_error("Failed to allocate resources");
status = UCS_ERR_NO_MEMORY;
goto err_free_resources;
}
/* copy only the resources enabled by user configuration */
context->tl_rscs = tmp;
for (i = 0; i < num_resources; ++i) {
if (ucp_is_resource_enabled(&tl_resources[i], config, masks)) {
context->tl_rscs[context->num_tls].tl_rsc = tl_resources[i];
context->tl_rscs[context->num_tls].pd_index = pd_index;
++context->num_tls;
++(*num_resources_p);
}
}
out_free_resources:
uct_release_tl_resource_list(tl_resources);
return UCS_OK;
err_free_resources:
uct_release_tl_resource_list(tl_resources);
err:
return status;
}
/* add new handler to the table */
static ucs_status_t ucs_async_handler_add(int min_id, int max_id,
ucs_async_handler_t *handler)
{
int hash_extra_status;
ucs_status_t status;
khiter_t hash_it;
int i, id;
pthread_rwlock_wrlock(&ucs_async_global_context.handlers_lock);
handler->id = -1;
ucs_assert_always(handler->refcount == 1);
/*
* Search for an empty key in the range [min_id, max_id)
* ucs_async_global_context.handler_id is used to generate "unique" keys.
*/
for (i = min_id; i < max_id; ++i) {
id = min_id + (ucs_atomic_fadd32(&ucs_async_global_context.handler_id, 1) %
(max_id - min_id));
hash_it = kh_put(ucs_async_handler, &ucs_async_global_context.handlers,
id, &hash_extra_status);
if (hash_extra_status == -1) {
ucs_error("Failed to add async handler " UCS_ASYNC_HANDLER_FMT
" to hash", UCS_ASYNC_HANDLER_ARG(handler));
status = UCS_ERR_NO_MEMORY;
goto out_unlock;
} else if (hash_extra_status != 0) {
handler->id = id;
ucs_assert(id != -1);
break;
}
}
if (handler->id == -1) {
ucs_error("Cannot add async handler %s() - id range [%d..%d) is full",
ucs_debug_get_symbol_name(handler->cb), min_id, max_id);
status = UCS_ERR_ALREADY_EXISTS;
goto out_unlock;
}
ucs_assert_always(!ucs_async_handler_kh_is_end(hash_it));
kh_value(&ucs_async_global_context.handlers, hash_it) = handler;
ucs_debug("added async handler " UCS_ASYNC_HANDLER_FMT " to hash",
UCS_ASYNC_HANDLER_ARG(handler));
status = UCS_OK;
out_unlock:
pthread_rwlock_unlock(&ucs_async_global_context.handlers_lock);
return status;
}
static uint64_t ucs_get_mac_address()
{
static uint64_t mac_address = 0;
struct ifreq ifr, *it, *end;
struct ifconf ifc;
char buf[1024];
int sock;
if (mac_address == 0) {
sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
ucs_error("failed to create socket: %m");
return 0;
}
ifc.ifc_len = sizeof(buf);
ifc.ifc_buf = buf;
if (ioctl(sock, SIOCGIFCONF, &ifc) == -1) {
ucs_error("ioctl(SIOCGIFCONF) failed: %m");
close(sock);
return 0;
}
it = ifc.ifc_req;
end = it + (ifc.ifc_len / sizeof *it);
for (it = ifc.ifc_req; it != end; ++it) {
strcpy(ifr.ifr_name, it->ifr_name);
if (ioctl(sock, SIOCGIFFLAGS, &ifr) != 0) {
ucs_error("ioctl(SIOCGIFFLAGS) failed: %m");
close(sock);
return 0;
}
if (!(ifr.ifr_flags & IFF_LOOPBACK)) {
if (ioctl(sock, SIOCGIFHWADDR, &ifr) != 0) {
ucs_error("ioctl(SIOCGIFHWADDR) failed: %m");
close(sock);
return 0;
}
memcpy(&mac_address, ifr.ifr_hwaddr.sa_data, 6);
break;
}
}
close(sock);
ucs_trace("MAC address is 0x%012"PRIX64, mac_address);
}
return mac_address;
}
static ucs_status_t ucp_perf_test_fill_params(ucx_perf_params_t *params,
ucp_params_t *ucp_params)
{
ucs_status_t status, message_size;
message_size = ucx_perf_get_message_size(params);
switch (params->command) {
case UCX_PERF_CMD_PUT:
case UCX_PERF_CMD_GET:
ucp_params->features |= UCP_FEATURE_RMA;
break;
case UCX_PERF_CMD_ADD:
case UCX_PERF_CMD_FADD:
case UCX_PERF_CMD_SWAP:
case UCX_PERF_CMD_CSWAP:
if (message_size == sizeof(uint32_t)) {
ucp_params->features |= UCP_FEATURE_AMO32;
} else if (message_size == sizeof(uint64_t)) {
ucp_params->features |= UCP_FEATURE_AMO64;
} else {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Atomic size should be either 32 or 64 bit");
}
return UCS_ERR_INVALID_PARAM;
}
break;
case UCX_PERF_CMD_TAG:
ucp_params->features |= UCP_FEATURE_TAG;
ucp_params->field_mask |= UCP_PARAM_FIELD_REQUEST_SIZE;
ucp_params->request_size = sizeof(ucp_perf_request_t);
break;
case UCX_PERF_CMD_STREAM:
ucp_params->features |= UCP_FEATURE_STREAM;
ucp_params->field_mask |= UCP_PARAM_FIELD_REQUEST_SIZE;
ucp_params->request_size = sizeof(ucp_perf_request_t);
break;
default:
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Invalid test command");
}
return UCS_ERR_INVALID_PARAM;
}
status = ucx_perf_test_check_params(params);
if (status != UCS_OK) {
return status;
}
return UCS_OK;
}
static ucs_status_t uct_ugni_fetch_pmi()
{
int spawned = 0,
rc;
if(job_info.initialized) {
return UCS_OK;
}
/* Fetch information from Cray's PMI */
rc = PMI_Init(&spawned);
if (PMI_SUCCESS != rc) {
ucs_error("PMI_Init failed, Error status: %d", rc);
return UCS_ERR_IO_ERROR;
}
ucs_debug("PMI spawned %d", spawned);
rc = PMI_Get_size(&job_info.pmi_num_of_ranks);
if (PMI_SUCCESS != rc) {
ucs_error("PMI_Get_size failed, Error status: %d", rc);
return UCS_ERR_IO_ERROR;
}
ucs_debug("PMI size %d", job_info.pmi_num_of_ranks);
rc = PMI_Get_rank(&job_info.pmi_rank_id);
if (PMI_SUCCESS != rc) {
ucs_error("PMI_Get_rank failed, Error status: %d", rc);
return UCS_ERR_IO_ERROR;
}
ucs_debug("PMI rank %d", job_info.pmi_rank_id);
rc = get_ptag(&job_info.ptag);
if (UCS_OK != rc) {
ucs_error("get_ptag failed, Error status: %d", rc);
return rc;
}
ucs_debug("PMI ptag %d", job_info.ptag);
rc = get_cookie(&job_info.cookie);
if (UCS_OK != rc) {
ucs_error("get_cookie failed, Error status: %d", rc);
return rc;
}
ucs_debug("PMI cookie %d", job_info.cookie);
/* Context and domain is activated */
job_info.initialized = true;
ucs_debug("UGNI job info was activated");
return UCS_OK;
}
static ucs_status_t uct_gdr_copy_mem_reg(uct_md_h uct_md, void *address, size_t length,
unsigned flags, uct_mem_h *memh_p)
{
uct_gdr_copy_mem_t *mem_hndl = NULL;
size_t reg_size;
void *ptr;
ucs_status_t status;
mem_hndl = ucs_malloc(sizeof(uct_gdr_copy_mem_t), "gdr_copy handle");
if (NULL == mem_hndl) {
ucs_error("failed to allocate memory for gdr_copy_mem_t");
return UCS_ERR_NO_MEMORY;
}
reg_size = (length + GPU_PAGE_SIZE - 1) & GPU_PAGE_MASK;
ptr = (void *) ((uintptr_t)address & GPU_PAGE_MASK);
status = uct_gdr_copy_mem_reg_internal(uct_md, ptr, reg_size, 0, mem_hndl);
if (status != UCS_OK) {
ucs_free(mem_hndl);
return status;
}
*memh_p = mem_hndl;
return UCS_OK;
}
static ucs_status_t ucp_ep_new(ucp_worker_h worker, uint64_t dest_uuid,
const char *peer_name, const char *message,
ucp_ep_h *ep_p)
{
ucp_ep_h ep;
ep = ucs_calloc(1, sizeof(*ep), "ucp ep");
if (ep == NULL) {
ucs_error("Failed to allocate ep");
return UCS_ERR_NO_MEMORY;
}
ep->worker = worker;
ep->dest_uuid = dest_uuid;
ep->rma_dst_pdi = UCP_NULL_RESOURCE;
ep->amo_dst_pdi = UCP_NULL_RESOURCE;
ep->cfg_index = 0;
ep->flags = 0;
#if ENABLE_DEBUG_DATA
ucs_snprintf_zero(ep->peer_name, UCP_WORKER_NAME_MAX, "%s", peer_name);
#endif
sglib_hashed_ucp_ep_t_add(worker->ep_hash, ep);
*ep_p = ep;
ucs_debug("created ep %p to %s 0x%"PRIx64"->0x%"PRIx64" %s", ep, peer_name,
worker->uuid, ep->dest_uuid, message);
return UCS_OK;
}
static ucs_status_t progress_local_cq(uct_ugni_smsg_iface_t *iface){
gni_return_t ugni_rc;
gni_cq_entry_t event_data;
uct_ugni_smsg_desc_t message_data;
uct_ugni_smsg_desc_t *message_pointer;
ugni_rc = GNI_CqGetEvent(iface->super.local_cq, &event_data);
if(GNI_RC_NOT_DONE == ugni_rc){
return UCS_OK;
}
if((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)){
/* TODO: handle overruns */
ucs_error("Error posting data. CQ overrun = %d", (int)GNI_CQ_OVERRUN(event_data));
return UCS_ERR_NO_RESOURCE;
}
message_data.msg_id = GNI_CQ_GET_MSG_ID(event_data);
message_pointer = sglib_hashed_uct_ugni_smsg_desc_t_find_member(iface->smsg_list,&message_data);
ucs_assert(NULL != message_pointer);
message_pointer->ep->outstanding--;
iface->super.outstanding--;
uct_ugni_ep_check_flush(message_pointer->ep);
sglib_hashed_uct_ugni_smsg_desc_t_delete(iface->smsg_list,message_pointer);
ucs_mpool_put(message_pointer);
return UCS_INPROGRESS;
}
ucs_status_t progress_remote_cq(uct_ugni_smsg_iface_t *iface)
{
gni_return_t ugni_rc;
gni_cq_entry_t event_data;
uct_ugni_ep_t tl_ep;
uct_ugni_ep_t *ugni_ep;
uct_ugni_smsg_ep_t *ep;
ugni_rc = GNI_CqGetEvent(iface->remote_cq, &event_data);
if(GNI_RC_NOT_DONE == ugni_rc){
return UCS_OK;
}
if (GNI_RC_SUCCESS != ugni_rc || !GNI_CQ_STATUS_OK(event_data) || GNI_CQ_OVERRUN(event_data)) {
if(GNI_RC_ERROR_RESOURCE == ugni_rc || (GNI_RC_SUCCESS == ugni_rc && GNI_CQ_OVERRUN(event_data))){
ucs_debug("Detected remote CQ overrun. ungi_rc = %d [%s]", ugni_rc, gni_err_str[ugni_rc]);
uct_ugni_smsg_handle_remote_overflow(iface);
return UCS_OK;
}
ucs_error("GNI_CqGetEvent falied with unhandled error. Error status %s %d ",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
tl_ep.hash_key = GNI_CQ_GET_INST_ID(event_data);
ugni_ep = sglib_hashed_uct_ugni_ep_t_find_member(iface->super.eps, &tl_ep);
ep = ucs_derived_of(ugni_ep, uct_ugni_smsg_ep_t);
process_mbox(iface, ep);
return UCS_INPROGRESS;
}
static UCS_F_NOINLINE void
uct_ud_mlx5_iface_post_recv(uct_ud_mlx5_iface_t *iface)
{
unsigned batch = iface->super.config.rx_max_batch;
struct mlx5_wqe_data_seg *rx_wqes;
uint16_t pi, next_pi, count;
uct_ib_iface_recv_desc_t *desc;
rx_wqes = iface->rx.wq.wqes;
pi = iface->rx.wq.rq_wqe_counter & iface->rx.wq.mask;
for (count = 0; count < batch; count ++) {
next_pi = (pi + 1) & iface->rx.wq.mask;
ucs_prefetch(rx_wqes + next_pi);
UCT_TL_IFACE_GET_RX_DESC(&iface->super.super.super, &iface->super.rx.mp,
desc, break);
rx_wqes[pi].lkey = htonl(desc->lkey);
rx_wqes[pi].addr = htonll((uintptr_t)uct_ib_iface_recv_desc_hdr(&iface->super.super, desc));
pi = next_pi;
}
if (ucs_unlikely(count == 0)) {
ucs_error("iface(%p) failed to post receive wqes", iface);
return;
}
pi = iface->rx.wq.rq_wqe_counter + count;
iface->rx.wq.rq_wqe_counter = pi;
iface->super.rx.available -= count;
ucs_memory_cpu_fence();
*iface->rx.wq.dbrec = htonl(pi);
}
ucs_status_t ucp_wireup_msg_progress(uct_pending_req_t *self)
{
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
ucp_ep_h ep = req->send.ep;
ssize_t packed_len;
if (req->send.wireup.type == UCP_WIREUP_MSG_REQUEST) {
if (ep->flags & UCP_EP_FLAG_REMOTE_CONNECTED) {
ucs_trace("ep %p: not sending wireup message - remote already connected",
ep);
goto out;
}
}
/* send the active message */
if (req->send.wireup.type == UCP_WIREUP_MSG_ACK) {
req->send.lane = ucp_ep_get_am_lane(ep);
} else {
req->send.lane = ucp_ep_get_wireup_msg_lane(ep);
}
packed_len = uct_ep_am_bcopy(ep->uct_eps[req->send.lane], UCP_AM_ID_WIREUP,
ucp_wireup_msg_pack, req);
if (packed_len < 0) {
if (packed_len != UCS_ERR_NO_RESOURCE) {
ucs_error("failed to send wireup: %s", ucs_status_string(packed_len));
}
return (ucs_status_t)packed_len;
}
out:
ucp_request_complete_send(req, UCS_OK);
return UCS_OK;
}
ucs_status_t uct_ugni_query_tl_resources(uct_pd_h pd, const char *tl_name,
uct_tl_resource_desc_t **resource_p,
unsigned *num_resources_p)
{
uct_tl_resource_desc_t *resources;
int num_devices = job_info.num_devices;
uct_ugni_device_t *devs = job_info.devices;
int i;
ucs_status_t status = UCS_OK;
pthread_mutex_lock(&uct_ugni_global_lock);
resources = ucs_calloc(job_info.num_devices, sizeof(uct_tl_resource_desc_t),
"resource desc");
if (NULL == resources) {
ucs_error("Failed to allocate memory");
num_devices = 0;
resources = NULL;
status = UCS_ERR_NO_MEMORY;
goto error;
}
for (i = 0; i < job_info.num_devices; i++) {
uct_ugni_device_get_resource(tl_name, &devs[i], &resources[i]);
}
error:
*num_resources_p = num_devices;
*resource_p = resources;
pthread_mutex_unlock(&uct_ugni_global_lock);
return status;
}
static inline ucs_status_t uct_ugni_post_rdma(uct_ugni_rdma_iface_t *iface,
uct_ugni_ep_t *ep,
uct_ugni_base_desc_t *rdma)
{
gni_return_t ugni_rc;
if (ucs_unlikely(!uct_ugni_can_send(ep))) {
ucs_mpool_put(rdma);
return UCS_ERR_NO_RESOURCE;
}
ugni_rc = GNI_PostRdma(ep->ep, &rdma->desc);
if (ucs_unlikely(GNI_RC_SUCCESS != ugni_rc)) {
ucs_mpool_put(rdma);
if(GNI_RC_ERROR_RESOURCE == ugni_rc || GNI_RC_ERROR_NOMEM == ugni_rc) {
ucs_debug("GNI_PostRdma failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_RESOURCE;
} else {
ucs_error("GNI_PostRdma failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
}
++ep->outstanding;
++iface->super.outstanding;
return UCS_INPROGRESS;
}
