static UCS_CLASS_INIT_FUNC(uct_self_iface_t, uct_md_h md, uct_worker_h worker, const uct_iface_params_t *params, const uct_iface_config_t *tl_config) { ucs_status_t status; uct_self_iface_config_t *self_config = 0; ucs_trace_func("Creating a loop-back transport self=%p rxh=%lu", self, params->rx_headroom); if (strcmp(params->dev_name, UCT_SELF_NAME) != 0) { ucs_error("No device was found: %s", params->dev_name); return UCS_ERR_NO_DEVICE; } UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_self_iface_ops, md, worker, tl_config UCS_STATS_ARG(params->stats_root) UCS_STATS_ARG(UCT_SELF_NAME)); self_config = ucs_derived_of(tl_config, uct_self_iface_config_t); self->id = ucs_generate_uuid((uintptr_t)self); self->rx_headroom = params->rx_headroom; self->data_length = self_config->super.max_bcopy; self->release_desc.cb = uct_self_iface_release_desc; /* create a memory pool for data transferred */ status = uct_iface_mpool_init(&self->super, &self->msg_desc_mp, sizeof(uct_recv_desc_t) + self->rx_headroom + self->data_length, sizeof(uct_recv_desc_t) + self->rx_headroom, UCS_SYS_CACHE_LINE_SIZE, &self_config->mp, 256, ucs_empty_function, "self_msg_desc"); if (UCS_OK != status) { ucs_error("Failed to create a memory pool for the loop-back transport"); goto err; } /* set the message descriptor for the loop-back */ self->msg_cur_desc = ucs_mpool_get(&self->msg_desc_mp); VALGRIND_MAKE_MEM_DEFINED(self->msg_cur_desc, sizeof(*(self->msg_cur_desc))); if (NULL == self->msg_cur_desc) { ucs_error("Failed to get the first descriptor in loop-back MP storage"); status = UCS_ERR_NO_RESOURCE; goto destroy_mpool; } ucs_debug("Created a loop-back iface. id=0x%lx, desc=%p, len=%u, tx_hdr=%lu", self->id, self->msg_cur_desc, self->data_length, self->rx_headroom); return UCS_OK; destroy_mpool: ucs_mpool_cleanup(&self->msg_desc_mp, 1); err: return status; }
ucs_status_t uct_rc_verbs_iface_common_init(uct_rc_verbs_iface_common_t *iface, uct_rc_iface_t *rc_iface, uct_rc_verbs_iface_common_config_t *config, uct_rc_iface_config_t *rc_config) { ucs_status_t status; size_t am_hdr_size; memset(iface->inl_sge, 0, sizeof(iface->inl_sge)); /* Configuration */ am_hdr_size = ucs_max(config->max_am_hdr, sizeof(uct_rc_hdr_t)); iface->config.short_desc_size = ucs_max(UCT_RC_MAX_ATOMIC_SIZE, am_hdr_size); /* Create AM headers and Atomic mempool */ status = uct_iface_mpool_init(&rc_iface->super.super, &iface->short_desc_mp, sizeof(uct_rc_iface_send_desc_t) + iface->config.short_desc_size, sizeof(uct_rc_iface_send_desc_t), UCS_SYS_CACHE_LINE_SIZE, &rc_config->super.tx.mp, rc_iface->config.tx_qp_len, uct_rc_iface_send_desc_init, "rc_verbs_short_desc"); if (status != UCS_OK) { return status; } return UCS_OK; }
ucs_status_t uct_ib_iface_recv_mpool_init(uct_ib_iface_t *iface, const uct_ib_iface_config_t *config, const char *name, ucs_mpool_t *mp) { unsigned grow; if (config->rx.queue_len < 1024) { grow = 1024; } else { /* We want to have some free (+10%) elements to avoid mem pool expansion */ grow = ucs_min( (int)(1.1 * config->rx.queue_len + 0.5), config->rx.mp.max_bufs); } return uct_iface_mpool_init(&iface->super, mp, iface->config.rx_payload_offset + iface->config.seg_size, iface->config.rx_hdr_offset, UCS_SYS_CACHE_LINE_SIZE, &config->rx.mp, grow, uct_ib_iface_recv_desc_init, name); }
ucs_status_t uct_rc_verbs_iface_common_init(uct_rc_verbs_iface_common_t *iface, uct_rc_iface_t *rc_iface, uct_rc_verbs_iface_common_config_t *config, uct_rc_iface_config_t *rc_config, size_t max_hdr_size) { memset(iface->inl_sge, 0, sizeof(iface->inl_sge)); ucs_status_t status; /* Configuration */ iface->config.short_desc_size = ucs_max(UCT_RC_MAX_ATOMIC_SIZE, max_hdr_size); /* Create AM headers and Atomic mempool */ status = uct_iface_mpool_init(&rc_iface->super.super, &iface->short_desc_mp, sizeof(uct_rc_iface_send_desc_t) + iface->config.short_desc_size, sizeof(uct_rc_iface_send_desc_t), UCS_SYS_CACHE_LINE_SIZE, &rc_config->super.tx.mp, rc_iface->config.tx_qp_len, uct_rc_iface_send_desc_init, "rc_verbs_short_desc"); if (status != UCS_OK) { return status; } iface->config.notag_hdr_size = 0; iface->am_inl_hdr = ucs_mpool_get(&iface->short_desc_mp); if (iface->am_inl_hdr == NULL) { ucs_error("Failed to allocate AM short header"); return UCS_ERR_NO_MEMORY; } return UCS_OK; }
static UCS_CLASS_INIT_FUNC(uct_ugni_rdma_iface_t, uct_pd_h pd, uct_worker_h worker, const char *dev_name, size_t rx_headroom, const uct_iface_config_t *tl_config) { uct_ugni_rdma_iface_config_t *config = ucs_derived_of(tl_config, uct_ugni_rdma_iface_config_t); ucs_status_t status; pthread_mutex_lock(&uct_ugni_global_lock); UCS_CLASS_CALL_SUPER_INIT(uct_ugni_iface_t, pd, worker, dev_name, &uct_ugni_rdma_iface_ops, &config->super UCS_STATS_ARG(NULL)); /* Setting initial configuration */ self->config.fma_seg_size = UCT_UGNI_MAX_FMA; self->config.rdma_max_size = UCT_UGNI_MAX_RDMA; status = ucs_mpool_init(&self->free_desc, 0, sizeof(uct_ugni_base_desc_t), 0, /* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ 128, /* grow */ config->mpool.max_bufs, /* max buffers */ &uct_ugni_rdma_desc_mpool_ops, "UGNI-DESC-ONLY"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto exit; } status = ucs_mpool_init(&self->free_desc_get, 0, sizeof(uct_ugni_rdma_fetch_desc_t), 0, /* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ 128 , /* grow */ config->mpool.max_bufs, /* max buffers */ &uct_ugni_rdma_desc_mpool_ops, "UGNI-GET-DESC-ONLY"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto clean_desc; } status = ucs_mpool_init(&self->free_desc_buffer, 0, sizeof(uct_ugni_base_desc_t) + self->config.fma_seg_size, sizeof(uct_ugni_base_desc_t), /* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ 128 , /* grow */ config->mpool.max_bufs, /* max buffers */ &uct_ugni_rdma_desc_mpool_ops, "UGNI-DESC-BUFFER"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto clean_desc_get; } status = uct_iface_mpool_init(&self->super.super, &self->free_desc_famo, sizeof(uct_ugni_rdma_fetch_desc_t) + 8, sizeof(uct_ugni_rdma_fetch_desc_t),/* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ &config->mpool, /* mpool config */ 128 , /* grow */ uct_ugni_base_desc_key_init, /* memory/key init */ "UGNI-DESC-FAMO"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto clean_buffer; } status = uct_iface_mpool_init(&self->super.super, &self->free_desc_get_buffer, sizeof(uct_ugni_rdma_fetch_desc_t) + self->config.fma_seg_size, sizeof(uct_ugni_rdma_fetch_desc_t), /* alignment offset */ UCS_SYS_CACHE_LINE_SIZE, /* alignment */ &config->mpool, /* mpool config */ 128 , /* grow */ uct_ugni_base_desc_key_init, /* memory/key init */ "UGNI-DESC-GET"); if (UCS_OK != status) { ucs_error("Mpool creation failed"); goto clean_famo; } status = ugni_activate_iface(&self->super); if (UCS_OK != status) { ucs_error("Failed to activate the interface"); goto clean_get_buffer; } if(GNI_DEVICE_ARIES == self->super.dev->type) { uct_ugni_rdma_iface_ops.ep_atomic_swap64 = uct_ugni_ep_atomic_swap64; uct_ugni_rdma_iface_ops.ep_atomic_add32 = uct_ugni_ep_atomic_add32; uct_ugni_rdma_iface_ops.ep_atomic_fadd32 = uct_ugni_ep_atomic_fadd32; uct_ugni_rdma_iface_ops.ep_atomic_cswap32 = uct_ugni_ep_atomic_cswap32; uct_ugni_rdma_iface_ops.ep_atomic_swap32 = uct_ugni_ep_atomic_swap32; } /* TBD: eventually the uct_ugni_progress has to be moved to * rdma layer so each ugni layer will have own progress */ uct_worker_progress_register(worker, uct_ugni_progress, self); pthread_mutex_unlock(&uct_ugni_global_lock); return UCS_OK; clean_get_buffer: ucs_mpool_cleanup(&self->free_desc_get_buffer, 1); clean_famo: ucs_mpool_cleanup(&self->free_desc_famo, 1); clean_buffer: ucs_mpool_cleanup(&self->free_desc_buffer, 1); clean_desc_get: ucs_mpool_cleanup(&self->free_desc_get, 1); clean_desc: ucs_mpool_cleanup(&self->free_desc, 1); exit: ucs_error("Failed to activate interface"); pthread_mutex_unlock(&uct_ugni_global_lock); return status; }