ucs_status_t uct_iface_mpool_init(uct_base_iface_t *iface, ucs_mpool_t *mp, size_t elem_size, size_t align_offset, size_t alignment, const uct_iface_mpool_config_t *config, unsigned grow, uct_iface_mpool_init_obj_cb_t init_obj_cb, const char *name) { unsigned elems_per_chunk; ucs_status_t status; elems_per_chunk = (config->bufs_grow != 0) ? config->bufs_grow : grow; status = ucs_mpool_init(mp, sizeof(uct_iface_mp_priv_t), elem_size, align_offset, alignment, elems_per_chunk, config->max_bufs, &uct_iface_mpool_ops, name); if (status != UCS_OK) { return status; } uct_iface_mp_priv(mp)->iface = iface; uct_iface_mp_priv(mp)->init_obj_cb = init_obj_cb; return UCS_OK; }
static UCS_CLASS_INIT_FUNC(uct_ugni_smsg_iface_t, uct_md_h md, uct_worker_h worker, const uct_iface_params_t *params, const uct_iface_config_t *tl_config) { uct_ugni_iface_config_t *config = ucs_derived_of(tl_config, uct_ugni_iface_config_t); ucs_status_t status; gni_return_t ugni_rc; unsigned int bytes_per_mbox; gni_smsg_attr_t smsg_attr; pthread_mutex_lock(&uct_ugni_global_lock); UCS_CLASS_CALL_SUPER_INIT(uct_ugni_iface_t, md, worker, params, &uct_ugni_smsg_iface_ops, &config->super UCS_STATS_ARG(NULL)); /* Setting initial configuration */ self->config.smsg_seg_size = 2048; self->config.rx_headroom = params->rx_headroom; self->config.smsg_max_retransmit = 16; self->config.smsg_max_credit = 8; self->smsg_id = 0; smsg_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT; smsg_attr.mbox_maxcredit = self->config.smsg_max_credit; smsg_attr.msg_maxsize = self->config.smsg_seg_size; ugni_rc = GNI_SmsgBufferSizeNeeded(&(smsg_attr), &bytes_per_mbox); self->bytes_per_mbox = ucs_align_up_pow2(bytes_per_mbox, ucs_get_page_size()); if (ugni_rc != GNI_RC_SUCCESS) { ucs_error("Smsg buffer size calculation failed"); status = UCS_ERR_INVALID_PARAM; goto exit; } status = ucs_mpool_init(&self->free_desc, 0, self->config.smsg_seg_size + sizeof(uct_ugni_smsg_desc_t), 0, UCS_SYS_CACHE_LINE_SIZE, /* alignment */ 128 , /* grow */ config->mpool.max_bufs, /* max buffers */ &uct_ugni_smsg_desc_mpool_ops, "UGNI-SMSG-DESC"); if (UCS_OK != status) { ucs_error("Desc Mpool creation failed"); goto exit; } status = ucs_mpool_init(&self->free_mbox, 0, self->bytes_per_mbox + sizeof(uct_ugni_smsg_mbox_t), sizeof(uct_ugni_smsg_mbox_t), UCS_SYS_CACHE_LINE_SIZE, /* alignment */ 128, /* grow */ config->mpool.max_bufs, /* max buffers */ &uct_ugni_smsg_mbox_mpool_ops, "UGNI-SMSG-MBOX"); if (UCS_OK != status) { ucs_error("Mbox Mpool creation failed"); goto clean_desc; } UCT_TL_IFACE_GET_TX_DESC(&self->super.super, &self->free_desc, self->user_desc, self->user_desc = NULL); status = ugni_smsg_activate_iface(self); if (UCS_OK != status) { ucs_error("Failed to activate the interface"); goto clean_mbox; } ugni_rc = GNI_SmsgSetMaxRetrans(self->super.nic_handle, self->config.smsg_max_retransmit); if (ugni_rc != GNI_RC_SUCCESS) { ucs_error("Smsg setting max retransmit count failed."); status = UCS_ERR_INVALID_PARAM; goto clean_iface; } /* TBD: eventually the uct_ugni_progress has to be moved to * udt layer so each ugni layer will have own progress */ uct_worker_progress_register(worker, uct_ugni_smsg_progress, self); pthread_mutex_unlock(&uct_ugni_global_lock); return UCS_OK; clean_iface: ugni_smsg_deactivate_iface(self); clean_desc: ucs_mpool_put(self->user_desc); ucs_mpool_cleanup(&self->free_desc, 1); clean_mbox: ucs_mpool_cleanup(&self->free_mbox, 1); exit: ucs_error("Failed to activate interface"); pthread_mutex_unlock(&uct_ugni_global_lock); return status; }
static UCS_CLASS_INIT_FUNC(uct_ugni_rdma_iface_t, uct_pd_h pd, uct_worker_h worker, const char *dev_name, size_t rx_headroom, const uct_iface_config_t *tl_config) { uct_ugni_rdma_iface_config_t *config = ucs_derived_of(tl_config, uct_ugni_rdma_iface_config_t); ucs_status_t status; pthread_mutex_lock(&uct_ugni_global_lock); UCS_CLASS_CALL_SUPER_INIT(uct_ugni_iface_t, pd, worker, dev_name, &uct_ugni_rdma_iface_ops, &config->super UCS_STATS_ARG(NULL)); /* Setting initial configuration */ self->config.fma_seg_size = UCT_UGNI_MAX_FMA; self->config.rdma_max_size = UCT_UGNI_MAX_RDMA; status = ucs_mpool_init(&self->free_desc, 0, sizeof(uct_ugni_base_desc_t), 0, /* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ 128, /* grow */ config->mpool.max_bufs, /* max buffers */ &uct_ugni_rdma_desc_mpool_ops, "UGNI-DESC-ONLY"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto exit; } status = ucs_mpool_init(&self->free_desc_get, 0, sizeof(uct_ugni_rdma_fetch_desc_t), 0, /* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ 128 , /* grow */ config->mpool.max_bufs, /* max buffers */ &uct_ugni_rdma_desc_mpool_ops, "UGNI-GET-DESC-ONLY"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto clean_desc; } status = ucs_mpool_init(&self->free_desc_buffer, 0, sizeof(uct_ugni_base_desc_t) + self->config.fma_seg_size, sizeof(uct_ugni_base_desc_t), /* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ 128 , /* grow */ config->mpool.max_bufs, /* max buffers */ &uct_ugni_rdma_desc_mpool_ops, "UGNI-DESC-BUFFER"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto clean_desc_get; } status = uct_iface_mpool_init(&self->super.super, &self->free_desc_famo, sizeof(uct_ugni_rdma_fetch_desc_t) + 8, sizeof(uct_ugni_rdma_fetch_desc_t),/* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ &config->mpool, /* mpool config */ 128 , /* grow */ uct_ugni_base_desc_key_init, /* memory/key init */ "UGNI-DESC-FAMO"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto clean_buffer; } status = uct_iface_mpool_init(&self->super.super, &self->free_desc_get_buffer, sizeof(uct_ugni_rdma_fetch_desc_t) + self->config.fma_seg_size, sizeof(uct_ugni_rdma_fetch_desc_t), /* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ &config->mpool, /* mpool config */ 128 , /* grow */ uct_ugni_base_desc_key_init, /* memory/key init */ "UGNI-DESC-GET"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto clean_famo; } status = ugni_activate_iface(&self->super); if (UCS_OK != status) { ucs_error("Failed to activate the interface"); goto clean_get_buffer; } if(GNI_DEVICE_ARIES == self->super.dev->type) { uct_ugni_rdma_iface_ops.ep_atomic_swap64 = uct_ugni_ep_atomic_swap64; uct_ugni_rdma_iface_ops.ep_atomic_add32 = uct_ugni_ep_atomic_add32; uct_ugni_rdma_iface_ops.ep_atomic_fadd32 = uct_ugni_ep_atomic_fadd32; uct_ugni_rdma_iface_ops.ep_atomic_cswap32 = uct_ugni_ep_atomic_cswap32; uct_ugni_rdma_iface_ops.ep_atomic_swap32 = uct_ugni_ep_atomic_swap32; } /* TBD: eventually the uct_ugni_progress has to be moved to * rdma layer so each ugni layer will have own progress */ uct_worker_progress_register(worker, uct_ugni_progress, self); pthread_mutex_unlock(&uct_ugni_global_lock); return UCS_OK; clean_get_buffer: ucs_mpool_cleanup(&self->free_desc_get_buffer, 1); clean_famo: ucs_mpool_cleanup(&self->free_desc_famo, 1); clean_buffer: ucs_mpool_cleanup(&self->free_desc_buffer, 1); clean_desc_get: ucs_mpool_cleanup(&self->free_desc_get, 1); clean_desc: ucs_mpool_cleanup(&self->free_desc, 1); exit: ucs_error("Failed to activate interface"); pthread_mutex_unlock(&uct_ugni_global_lock); return status; }
static UCS_CLASS_INIT_FUNC(uct_ugni_udt_iface_t, uct_md_h md, uct_worker_h worker, const uct_iface_params_t *params, const uct_iface_config_t *tl_config) { uct_ugni_iface_config_t *config = ucs_derived_of(tl_config, uct_ugni_iface_config_t); ucs_status_t status; uct_ugni_udt_desc_t *desc; gni_return_t ugni_rc; int rc; UCS_CLASS_CALL_SUPER_INIT(uct_ugni_iface_t, md, worker, params, &uct_ugni_udt_iface_ops, &config->super UCS_STATS_ARG(NULL)); /* Setting initial configuration */ self->config.udt_seg_size = GNI_DATAGRAM_MAXSIZE; self->config.rx_headroom = params->rx_headroom; self->release_desc.cb = uct_ugni_udt_iface_release_desc; status = ucs_async_pipe_create(&self->event_pipe); if (UCS_OK != status) { ucs_error("Pipe creation failed"); goto exit; } status = ucs_mpool_init(&self->free_desc, 0, uct_ugni_udt_get_diff(self) + self->config.udt_seg_size * 2, uct_ugni_udt_get_diff(self), UCS_SYS_CACHE_LINE_SIZE, /* alignment */ 128, /* grow */ config->mpool.max_bufs, /* max buffers */ &uct_ugni_udt_desc_mpool_ops, "UGNI-UDT-DESC"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto clean_pipe; } ugni_rc = GNI_EpCreate(uct_ugni_udt_iface_nic_handle(self), NULL, &self->ep_any); if (GNI_RC_SUCCESS != ugni_rc) { ucs_error("GNI_EpCreate failed, Error status: %s %d", gni_err_str[ugni_rc], ugni_rc); status = UCS_ERR_NO_DEVICE; goto clean_free_desc; } UCT_TL_IFACE_GET_TX_DESC(&self->super.super, &self->free_desc, desc, goto clean_ep); ucs_debug("First wildcard desc is %p", desc); /* Init any desc */ self->desc_any = desc; status = uct_ugni_udt_ep_any_post(self); if (UCS_OK != status) { /* We can't continue if we can't post the first receive */ ucs_error("Failed to post wildcard request"); goto clean_any_desc; } status = ucs_async_set_event_handler(self->super.super.worker->async->mode, ucs_async_pipe_rfd(&self->event_pipe), POLLIN, uct_ugni_proccess_datagram_pipe, self, self->super.super.worker->async); if (UCS_OK != status) { goto clean_cancel_desc; } pthread_mutex_init(&self->device_lock, NULL); pthread_cond_init(&self->device_condition, NULL); self->events_ready = 0; rc = pthread_create(&self->event_thread, NULL, uct_ugni_udt_device_thread, self); if(0 != rc) { goto clean_remove_event; } return UCS_OK; clean_remove_event: ucs_async_pipe_destroy(&self->event_pipe); clean_cancel_desc: uct_ugni_udt_clean_wildcard(self); clean_any_desc: ucs_mpool_put(self->desc_any); clean_ep: ugni_rc = GNI_EpDestroy(self->ep_any); if (GNI_RC_SUCCESS != ugni_rc) { ucs_warn("GNI_EpDestroy failed, Error status: %s %d", gni_err_str[ugni_rc], ugni_rc); } clean_free_desc: ucs_mpool_cleanup(&self->free_desc, 1); clean_pipe: ucs_async_pipe_destroy(&self->event_pipe); exit: uct_ugni_cleanup_base_iface(&self->super); ucs_error("Failed to activate interface"); return status; }