static UCS_CLASS_INIT_FUNC(uct_rc_mlx5_ep_t, uct_iface_h tl_iface)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_mlx5_iface_t);
uct_ib_mlx5_qp_info_t qp_info;
ucs_status_t status;
UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super);
status = uct_ib_mlx5_get_qp_info(self->super.qp, &qp_info);
if (status != UCS_OK) {
ucs_error("Failed to get mlx5 QP information");
return status;
}
if ((qp_info.bf.size == 0) || !ucs_is_pow2(qp_info.bf.size) ||
(qp_info.sq.stride != MLX5_SEND_WQE_BB) ||
!ucs_is_pow2(qp_info.sq.wqe_cnt))
{
ucs_error("mlx5 device parameters not suitable for transport");
return UCS_ERR_IO_ERROR;
}
self->qp_num = self->super.qp->qp_num;
self->tx.qstart = qp_info.sq.buf;
self->tx.qend = qp_info.sq.buf + (MLX5_SEND_WQE_BB * qp_info.sq.wqe_cnt);
self->tx.seg = self->tx.qstart;
self->tx.sw_pi = 0;
self->tx.prev_sw_pi = -1;
self->tx.max_pi = uct_rc_mlx5_calc_max_pi(iface, self->tx.prev_sw_pi);
self->tx.bf_reg = qp_info.bf.reg;
self->tx.bf_size = qp_info.bf.size;
self->tx.dbrec = &qp_info.dbrec[MLX5_SND_DBR];
memset(self->tx.qstart, 0, self->tx.qend - self->tx.qstart);
return UCS_OK;
}
static int ucm_posix_memalign(void **memptr, size_t alignment, size_t size)
{
void *ptr;
if (!ucs_is_pow2(alignment)) {
return EINVAL;
}
ptr = ucm_memalign_impl(alignment, size, "posix_memalign");
if (ptr == NULL) {
return ENOMEM;
}
*memptr = ptr;
return 0;
}
ucs_status_t ucs_mpool_init(ucs_mpool_t *mp, size_t priv_size,
size_t elem_size, size_t align_offset, size_t alignment,
unsigned elems_per_chunk, unsigned max_elems,
ucs_mpool_ops_t *ops, const char *name)
{
/* Check input values */
if ((elem_size == 0) || (align_offset > elem_size) ||
(alignment == 0) || !ucs_is_pow2(alignment) ||
(elems_per_chunk == 0) || (max_elems < elems_per_chunk))
{
ucs_error("Invalid memory pool parameter(s)");
return UCS_ERR_INVALID_PARAM;
}
mp->data = ucs_malloc(sizeof(*mp->data) + priv_size, "mpool_data");
if (mp->data == NULL) {
ucs_error("Failed to allocate memory pool slow-path area");
return UCS_ERR_NO_MEMORY;
}
mp->freelist = NULL;
mp->data->elem_size = sizeof(ucs_mpool_elem_t) + elem_size;
mp->data->alignment = alignment;
mp->data->align_offset = sizeof(ucs_mpool_elem_t) + align_offset;
mp->data->quota = max_elems;
mp->data->tail = NULL;
mp->data->chunk_size = sizeof(ucs_mpool_chunk_t) + alignment +
elems_per_chunk * ucs_mpool_elem_total_size(mp->data);
mp->data->chunks = NULL;
mp->data->ops = ops;
mp->data->name = strdup(name);
VALGRIND_CREATE_MEMPOOL(mp, 0, 0);
ucs_debug("mpool %s: align %u, maxelems %u, elemsize %u",
ucs_mpool_name(mp), mp->data->alignment, max_elems, mp->data->elem_size);
return UCS_OK;
}
ucs_status_t uct_ib_iface_query(uct_ib_iface_t *iface, size_t xport_hdr_len,
uct_iface_attr_t *iface_attr)
{
uct_ib_device_t *dev = uct_ib_iface_device(iface);
static const unsigned ib_port_widths[] = {
[0] = 1,
[1] = 4,
[2] = 8,
[3] = 12
};
uint8_t active_width, active_speed, active_mtu;
double encoding, signal_rate, wire_speed;
size_t mtu, width, extra_pkt_len;
int i = 0;
active_width = uct_ib_iface_port_attr(iface)->active_width;
active_speed = uct_ib_iface_port_attr(iface)->active_speed;
active_mtu = uct_ib_iface_port_attr(iface)->active_mtu;
/* Get active width */
if (!ucs_is_pow2(active_width) ||
(active_width < 1) || (ucs_ilog2(active_width) > 3))
{
ucs_error("Invalid active_width on %s:%d: %d",
UCT_IB_IFACE_ARG(iface), active_width);
return UCS_ERR_IO_ERROR;
}
memset(iface_attr, 0, sizeof(*iface_attr));
iface_attr->device_addr_len = iface->addr_size;
switch (active_speed) {
case 1: /* SDR */
iface_attr->latency = 5000e-9;
signal_rate = 2.5e9;
encoding = 8.0/10.0;
break;
case 2: /* DDR */
iface_attr->latency = 2500e-9;
signal_rate = 5.0e9;
encoding = 8.0/10.0;
break;
case 4: /* QDR */
iface_attr->latency = 1300e-9;
signal_rate = 10.0e9;
encoding = 8.0/10.0;
break;
case 8: /* FDR10 */
iface_attr->latency = 700e-9;
signal_rate = 10.3125e9;
encoding = 64.0/66.0;
break;
case 16: /* FDR */
iface_attr->latency = 700e-9;
signal_rate = 14.0625e9;
encoding = 64.0/66.0;
break;
case 32: /* EDR */
iface_attr->latency = 600e-9;
signal_rate = 25.78125e9;
encoding = 64.0/66.0;
break;
default:
ucs_error("Invalid active_speed on %s:%d: %d",
UCT_IB_IFACE_ARG(iface), active_speed);
return UCS_ERR_IO_ERROR;
}
/* Wire speed calculation: Width * SignalRate * Encoding */
width = ib_port_widths[ucs_ilog2(active_width)];
wire_speed = (width * signal_rate * encoding) / 8.0;
/* Calculate packet overhead */
mtu = ucs_min(uct_ib_mtu_value(active_mtu),
iface->config.seg_size);
extra_pkt_len = UCT_IB_BTH_LEN + xport_hdr_len + UCT_IB_ICRC_LEN + UCT_IB_VCRC_LEN + UCT_IB_DELIM_LEN;
if (IBV_PORT_IS_LINK_LAYER_ETHERNET(uct_ib_iface_port_attr(iface))) {
extra_pkt_len += UCT_IB_GRH_LEN + UCT_IB_ROCE_LEN;
} else {
/* TODO check if UCT_IB_DELIM_LEN is present in RoCE as well */
extra_pkt_len += UCT_IB_LRH_LEN;
}
iface_attr->bandwidth = (wire_speed * mtu) / (mtu + extra_pkt_len);
/* Set priority of current device */
iface_attr->priority = 0;
while (uct_ib_device_info_table[i].vendor_part_id != 0) {
if (uct_ib_device_info_table[i].vendor_part_id == dev->dev_attr.vendor_part_id) {
iface_attr->priority = uct_ib_device_info_table[i].priority;
break;
}
i++;
}
return UCS_OK;
}
ucs_status_t uct_ib_iface_query(uct_ib_iface_t *iface, size_t xport_hdr_len,
uct_iface_attr_t *iface_attr)
{
uct_ib_device_t *dev = uct_ib_iface_device(iface);
static const unsigned ib_port_widths[] = {
[0] = 1,
[1] = 4,
[2] = 8,
[3] = 12
};
uint8_t active_width, active_speed, active_mtu;
double encoding, signal_rate, wire_speed;
size_t mtu, width, extra_pkt_len;
if (!uct_ib_device_is_port_ib(dev, iface->port_num)) {
return UCS_ERR_UNSUPPORTED;
}
active_width = uct_ib_iface_port_attr(iface)->active_width;
active_speed = uct_ib_iface_port_attr(iface)->active_speed;
active_mtu = uct_ib_iface_port_attr(iface)->active_mtu;
/* Get active width */
if (!ucs_is_pow2(active_width) ||
(active_width < 1) || (ucs_ilog2(active_width) > 3))
{
ucs_error("Invalid active_width on %s:%d: %d",
uct_ib_device_name(dev), iface->port_num, active_width);
return UCS_ERR_IO_ERROR;
}
memset(iface_attr, 0, sizeof(*iface_attr));
iface_attr->device_addr_len = iface->addr_size;
switch (active_speed) {
case 1: /* SDR */
iface_attr->latency = 5000e-9;
signal_rate = 2.5e9;
encoding = 8.0/10.0;
break;
case 2: /* DDR */
iface_attr->latency = 2500e-9;
signal_rate = 5.0e9;
encoding = 8.0/10.0;
break;
case 4: /* QDR */
iface_attr->latency = 1300e-9;
signal_rate = 10.0e9;
encoding = 8.0/10.0;
break;
case 8: /* FDR10 */
iface_attr->latency = 700e-9;
signal_rate = 10.3125e9;
encoding = 64.0/66.0;
break;
case 16: /* FDR */
iface_attr->latency = 700e-9;
signal_rate = 14.0625e9;
encoding = 64.0/66.0;
break;
case 32: /* EDR */
iface_attr->latency = 600e-9;
signal_rate = 25.78125e9;
encoding = 64.0/66.0;
break;
default:
ucs_error("Invalid active_speed on %s:%d: %d",
uct_ib_device_name(dev), iface->port_num, active_speed);
return UCS_ERR_IO_ERROR;
}
/* Wire speed calculation: Width * SignalRate * Encoding */
width = ib_port_widths[ucs_ilog2(active_width)];
wire_speed = (width * signal_rate * encoding) / 8.0;
/* Calculate packet overhead */
mtu = ucs_min(uct_ib_mtu_value(active_mtu),
iface->config.seg_size);
extra_pkt_len = UCT_IB_LRH_LEN + UCT_IB_BTH_LEN + xport_hdr_len +
UCT_IB_ICRC_LEN + UCT_IB_VCRC_LEN + UCT_IB_DELIM_LEN;
iface_attr->bandwidth = (wire_speed * mtu) / (mtu + extra_pkt_len);
return UCS_OK;
}