static void
rdma_trans_destroy(void *a)
{
Rdmatrans *rdma;
struct ibv_qp_attr attr;
rdma = a;
if (rdma->connected)
rdma_disconnect(rdma->cm_id);
if (rdma->qp) {
attr.qp_state = IBV_QPS_ERR;
ibv_modify_qp(rdma->qp, &attr, IBV_QP_STATE);
ibv_destroy_qp(rdma->qp);
}
if (rdma->cq)
ibv_destroy_cq(rdma->cq);
if (rdma->ch)
ibv_destroy_comp_channel(rdma->ch);
if (rdma->snd_mr)
ibv_dereg_mr(rdma->snd_mr);
if (rdma->snd_buf)
free(rdma->snd_buf);
if (rdma->rcv_mr)
ibv_dereg_mr(rdma->rcv_mr);
if (rdma->rcv_buf)
free(rdma->rcv_buf);
if (rdma->pd)
ibv_dealloc_pd(rdma->pd);
if (rdma->cm_id)
rdma_destroy_id(rdma->cm_id);
}
/// The IBConnectionGroup default destructor.
virtual ~IBConnectionGroup()
{
for (auto& c : conn_)
c = nullptr;
if (listen_id_) {
int err = rdma_destroy_id(listen_id_);
if (err) {
L_(error) << "rdma_destroy_id() failed";
}
listen_id_ = nullptr;
}
if (cq_) {
int err = ibv_destroy_cq(cq_);
if (err) {
L_(error) << "ibv_destroy_cq() failed";
}
cq_ = nullptr;
}
if (pd_) {
int err = ibv_dealloc_pd(pd_);
if (err) {
L_(error) << "ibv_dealloc_pd() failed";
}
pd_ = nullptr;
}
rdma_destroy_event_channel(ec_);
}
static void free_sa_qp_cache(void)
{
struct mca_btl_openib_sa_qp_cache *cache, *tmp;
cache = sa_qp_cache;
while (NULL != cache) {
/* free cache data */
if (cache->device_name)
free(cache->device_name);
if (NULL != cache->qp)
ibv_destroy_qp(cache->qp);
if (NULL != cache->ah)
ibv_destroy_ah(cache->ah);
if (NULL != cache->cq)
ibv_destroy_cq(cache->cq);
if (NULL != cache->mr)
ibv_dereg_mr(cache->mr);
if (NULL != cache->pd)
ibv_dealloc_pd(cache->pd);
tmp = cache->next;
free(cache);
cache = tmp;
}
sa_qp_cache = NULL;
}
static void destroy_node(struct cmatest_node *node)
{
if (!node->cma_id)
return;
if (node->cma_id->qp)
rdma_destroy_qp(node->cma_id);
if (node->cq[SEND_CQ_INDEX])
ibv_destroy_cq(node->cq[SEND_CQ_INDEX]);
if (node->cq[RECV_CQ_INDEX])
ibv_destroy_cq(node->cq[RECV_CQ_INDEX]);
if (node->mem) {
ibv_dereg_mr(node->mr);
free(node->mem);
}
if (node->pd)
ibv_dealloc_pd(node->pd);
/* Destroy the RDMA ID after all device resources */
rdma_destroy_id(node->cma_id);
}
static void rping_free_qp(struct rping_cb *cb)
{
ibv_destroy_qp(cb->qp);
ibv_destroy_cq(cb->cq);
ibv_destroy_comp_channel(cb->channel);
ibv_dealloc_pd(cb->pd);
}
RDMAAdapter::~RDMAAdapter() {
StopInternalThread();
CHECK(!ibv_destroy_cq(cq_)) << "Failed to destroy CQ";
CHECK(!ibv_destroy_comp_channel(channel_)) << "Failed to destroy channel";
CHECK(!ibv_dealloc_pd(pd_)) << "Failed to deallocate PD";
CHECK(!ibv_close_device(context_)) << "Failed to release context";
}
void xfer_rdma_destroy_ctx(struct xfer_context *ctx) {
rdma_destroy_qp(ctx->cm_id);
ibv_destroy_cq(ctx->cq);
ibv_destroy_comp_channel(ctx->ch);
ibv_dereg_mr(ctx->send_mr);
ibv_dereg_mr(ctx->recv_mr);
ibv_dealloc_pd(ctx->pd);
free(ctx);
}
ProtectionDomain& ProtectionDomain::operator=(ProtectionDomain&& other) {
if (mDomain != nullptr && ibv_dealloc_pd(mDomain)) {
throw std::system_error(errno, std::generic_category());
}
mDomain = other.mDomain;
other.mDomain = nullptr;
return *this;
}
static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f)
{
struct rdmaio_data *rd = td->io_ops->data;
struct ibv_send_wr *bad_wr;
/* unregister rdma buffer */
/*
* Client sends notification to the server side
*/
/* refer to: http://linux.die.net/man/7/rdma_cm */
if ((rd->is_client == 1) && ((rd->rdma_protocol == FIO_RDMA_MEM_WRITE)
|| (rd->rdma_protocol ==
FIO_RDMA_MEM_READ))) {
if (ibv_post_send(rd->qp, &rd->sq_wr, &bad_wr) != 0) {
log_err("fio: ibv_post_send fail");
return 1;
}
dprint(FD_IO, "fio: close information sent success\n");
rdma_poll_wait(td, IBV_WC_SEND);
}
if (rd->is_client == 1)
rdma_disconnect(rd->cm_id);
else {
rdma_disconnect(rd->child_cm_id);
#if 0
rdma_disconnect(rd->cm_id);
#endif
}
#if 0
if (get_next_channel_event(td, rd->cm_channel, RDMA_CM_EVENT_DISCONNECTED) != 0) {
log_err("fio: wait for RDMA_CM_EVENT_DISCONNECTED\n");
return 1;
}
#endif
ibv_destroy_cq(rd->cq);
ibv_destroy_qp(rd->qp);
if (rd->is_client == 1)
rdma_destroy_id(rd->cm_id);
else {
rdma_destroy_id(rd->child_cm_id);
rdma_destroy_id(rd->cm_id);
}
ibv_destroy_comp_channel(rd->channel);
ibv_dealloc_pd(rd->pd);
return 0;
}
int __ibv_dealloc_pd_1_0(struct ibv_pd_1_0 *pd)
{
int ret;
ret = ibv_dealloc_pd(pd->real_pd);
if (ret)
return ret;
free(pd);
return 0;
}
int __ibv_dealloc_pd_1_0(struct ibv_pd_1_0 *pd)
{ fprintf(stderr, "%s:%s:%d \n", __func__, __FILE__, __LINE__);
int ret;
ret = ibv_dealloc_pd(pd->real_pd);
if (ret)
return ret;
free(pd);
return 0;
}
int BClientContext::destroy_context () {
if (qp) TEST_NZ(ibv_destroy_qp (qp));
if (recv_memory_mr) TEST_NZ (ibv_dereg_mr (recv_memory_mr));
if (lock_result_mr) TEST_NZ (ibv_dereg_mr (lock_result_mr));
if (cq) TEST_NZ (ibv_destroy_cq (cq));
if (pd) TEST_NZ (ibv_dealloc_pd (pd));
if (ib_ctx) TEST_NZ (ibv_close_device (ib_ctx));
if (sockfd >= 0) TEST_NZ (close (sockfd));
return 0;
}
int opal_common_verbs_qp_test(struct ibv_context *device_context, int flags)
{
int rc = OPAL_SUCCESS;
struct ibv_pd *pd = NULL;
struct ibv_cq *cq = NULL;
/* Bozo check */
if (NULL == device_context ||
(0 == (flags & (OPAL_COMMON_VERBS_FLAGS_RC | OPAL_COMMON_VERBS_FLAGS_UD)))) {
return OPAL_ERR_BAD_PARAM;
}
/* Try to make both the PD and CQ */
pd = ibv_alloc_pd(device_context);
if (NULL == pd) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
cq = ibv_create_cq(device_context, 2, NULL, NULL, 0);
if (NULL == cq) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
goto out;
}
/* Now try to make the QP(s) of the desired type(s) */
if (flags & OPAL_COMMON_VERBS_FLAGS_RC &&
!make_qp(pd, cq, IBV_QPT_RC)) {
rc = OPAL_ERR_NOT_SUPPORTED;
goto out;
}
if (flags & OPAL_COMMON_VERBS_FLAGS_NOT_RC &&
make_qp(pd, cq, IBV_QPT_RC)) {
rc = OPAL_ERR_TYPE_MISMATCH;
goto out;
}
if (flags & OPAL_COMMON_VERBS_FLAGS_UD &&
!make_qp(pd, cq, IBV_QPT_UD)) {
rc = OPAL_ERR_NOT_SUPPORTED;
goto out;
}
out:
/* Free the PD and/or CQ */
if (NULL != pd) {
ibv_dealloc_pd(pd);
}
if (NULL != cq) {
ibv_destroy_cq(cq);
}
return rc;
}
static int resources_destroy(struct resources *res)
{
int rc = 0;
if (res->qp) {
if (ibv_destroy_qp(res->qp)) {
fprintf(stderr, "failed to destroy QP\n");
rc = 1;
}
}
if (res->mr) {
if (ibv_dereg_mr(res->mr)) {
fprintf(stderr, "failed to deregister MR\n");
rc = 1;
}
}
if (res->buf)
free(res->buf);
if (res->cq) {
if (ibv_destroy_cq(res->cq)) {
fprintf(stderr, "failed to destroy CQ\n");
rc = 1;
}
}
if (res->pd) {
if (ibv_dealloc_pd(res->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
rc = 1;
}
}
if (res->ib_ctx) {
if (ibv_close_device(res->ib_ctx)) {
fprintf(stderr, "failed to close device context\n");
rc = 1;
}
}
if (res->sock >= 0) {
if (close(res->sock)) {
fprintf(stderr, "failed to close socket\n");
rc = 1;
}
}
return rc;
}
static int rping_setup_qp(struct rping_cb *cb, struct rdma_cm_id *cm_id)
{
int ret;
cb->pd = ibv_alloc_pd(cm_id->verbs);
if (!cb->pd) {
fprintf(stderr, "ibv_alloc_pd failed\n");
return errno;
}
DEBUG_LOG("created pd %p\n", cb->pd);
cb->channel = ibv_create_comp_channel(cm_id->verbs);
if (!cb->channel) {
fprintf(stderr, "ibv_create_comp_channel failed\n");
ret = errno;
goto err1;
}
DEBUG_LOG("created channel %p\n", cb->channel);
cb->cq = ibv_create_cq(cm_id->verbs, RPING_SQ_DEPTH * 2, cb,
cb->channel, 0);
if (!cb->cq) {
fprintf(stderr, "ibv_create_cq failed\n");
ret = errno;
goto err2;
}
DEBUG_LOG("created cq %p\n", cb->cq);
ret = ibv_req_notify_cq(cb->cq, 0);
if (ret) {
fprintf(stderr, "ibv_create_cq failed\n");
ret = errno;
goto err3;
}
ret = rping_create_qp(cb);
if (ret) {
perror("rdma_create_qp");
goto err3;
}
DEBUG_LOG("created qp %p\n", cb->qp);
return 0;
err3:
ibv_destroy_cq(cb->cq);
err2:
ibv_destroy_comp_channel(cb->channel);
err1:
ibv_dealloc_pd(cb->pd);
return ret;
}
static inline int fi_ibv_get_qp_cap(struct ibv_context *ctx,
struct fi_info *info)
{
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_qp *qp;
struct ibv_qp_init_attr init_attr;
int ret = 0;
pd = ibv_alloc_pd(ctx);
if (!pd) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_alloc_pd", errno);
return -errno;
}
cq = ibv_create_cq(ctx, 1, NULL, NULL, 0);
if (!cq) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_cq", errno);
ret = -errno;
goto err1;
}
memset(&init_attr, 0, sizeof init_attr);
init_attr.send_cq = cq;
init_attr.recv_cq = cq;
init_attr.cap.max_send_wr = verbs_default_tx_size;
init_attr.cap.max_recv_wr = verbs_default_rx_size;
init_attr.cap.max_send_sge = verbs_default_tx_iov_limit;
init_attr.cap.max_recv_sge = verbs_default_rx_iov_limit;
init_attr.cap.max_inline_data = verbs_default_inline_size;
init_attr.qp_type = IBV_QPT_RC;
qp = ibv_create_qp(pd, &init_attr);
if (!qp) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_qp", errno);
ret = -errno;
goto err2;
}
info->tx_attr->inject_size = init_attr.cap.max_inline_data;
ibv_destroy_qp(qp);
err2:
ibv_destroy_cq(cq);
err1:
ibv_dealloc_pd(pd);
return ret;
}
/** ========================================================================= */
static void destroy_sa_qp(struct oib_port *port)
{
int i;
// if the user just unregistered trap messages those messages may still
// be on this list, wait 5 seconds for the thread to handle the response.
for (i = 0; i < 5000; i++) {
if (!LIST_EMPTY(&port->pending_reg_msg_head)) {
usleep(1000);
}
else {
DBGPRINT("destroy_sa_qp: wait %d ms for LIST_EMPTY\n", i);
break;
}
}
stop_ud_cq_monitor(port);
join_port_thread(port);
/* Free any remaining unregistration messages */
if (!LIST_EMPTY(&port->pending_reg_msg_head)) {
OUTPUT_ERROR("Ignoring Pending Notice un-registation requests\n");
oib_sa_remove_all_pending_reg_msgs(port);
}
if (port->sa_ah)
ibv_destroy_ah(port->sa_ah);
if (port->sa_qp)
ibv_destroy_qp(port->sa_qp);
for (i = 0; i<port->num_userspace_recv_buf; i++)
if (port->recv_bufs)
ibv_dereg_mr(port->recv_bufs[i].mr);
if (port->sa_qp_pd)
ibv_dealloc_pd(port->sa_qp_pd);
if (port->sa_qp_cq)
ibv_destroy_cq(port->sa_qp_cq);
if (port->recv_bufs) {
free(port->recv_bufs);
port->recv_bufs = NULL;
}
if (port->sa_qp_comp_channel)
ibv_destroy_comp_channel(port->sa_qp_comp_channel);
}
static void
rdmasniff_cleanup(pcap_t *handle)
{
struct pcap_rdmasniff *priv = handle->priv;
ibv_dereg_mr(priv->mr);
ibv_destroy_flow(priv->flow);
ibv_destroy_qp(priv->qp);
ibv_destroy_cq(priv->cq);
ibv_dealloc_pd(priv->pd);
ibv_destroy_comp_channel(priv->channel);
ibv_close_device(priv->context);
free(priv->oneshot_buffer);
pcap_cleanup_live_common(handle);
}
static int ibw_conn_priv_destruct(struct ibw_conn_priv *pconn)
{
DEBUG(DEBUG_DEBUG, ("ibw_conn_priv_destruct(%p, cmid: %p)\n",
pconn, pconn->cm_id));
/* pconn->wr_index is freed by talloc */
/* pconn->wr_index[i] are freed by talloc */
/*
* tevent_fd must be removed before the fd is closed
*/
TALLOC_FREE(pconn->verbs_channel_event);
/* destroy verbs */
if (pconn->cm_id!=NULL && pconn->cm_id->qp!=NULL) {
rdma_destroy_qp(pconn->cm_id);
pconn->cm_id->qp = NULL;
}
if (pconn->cq!=NULL) {
ibv_destroy_cq(pconn->cq);
pconn->cq = NULL;
}
if (pconn->verbs_channel!=NULL) {
ibv_destroy_comp_channel(pconn->verbs_channel);
pconn->verbs_channel = NULL;
}
/* free memory regions */
ibw_free_mr(&pconn->buf_send, &pconn->mr_send);
ibw_free_mr(&pconn->buf_recv, &pconn->mr_recv);
if (pconn->pd) {
ibv_dealloc_pd(pconn->pd);
pconn->pd = NULL;
DEBUG(DEBUG_DEBUG, ("pconn=%p pd deallocated\n", pconn));
}
if (pconn->cm_id) {
rdma_destroy_id(pconn->cm_id);
pconn->cm_id = NULL;
DEBUG(DEBUG_DEBUG, ("pconn=%p cm_id destroyed\n", pconn));
}
return 0;
}
static ucs_status_t uct_ib_mlx5_check_dc(uct_ib_device_t *dev)
{
ucs_status_t status = UCS_OK;
struct ibv_context *ctx = dev->ibv_context;
struct ibv_qp_init_attr_ex qp_attr = {};
struct mlx5dv_qp_init_attr dv_attr = {};
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_qp *qp;
pd = ibv_alloc_pd(ctx);
if (pd == NULL) {
ucs_error("ibv_alloc_pd() failed: %m");
return UCS_ERR_IO_ERROR;
}
cq = ibv_create_cq(ctx, 1, NULL, NULL, 0);
if (cq == NULL) {
ucs_error("ibv_create_cq() failed: %m");
status = UCS_ERR_IO_ERROR;
goto err_cq;
}
qp_attr.send_cq = cq;
qp_attr.recv_cq = cq;
qp_attr.cap.max_send_wr = 1;
qp_attr.cap.max_send_sge = 1;
qp_attr.qp_type = IBV_QPT_DRIVER;
qp_attr.comp_mask = IBV_QP_INIT_ATTR_PD;
qp_attr.pd = pd;
dv_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_DC;
dv_attr.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI;
/* create DCI qp successful means DC is supported */
qp = mlx5dv_create_qp(ctx, &qp_attr, &dv_attr);
if (qp) {
ibv_destroy_qp(qp);
dev->flags |= UCT_IB_DEVICE_FLAG_DC;
}
ibv_destroy_cq(cq);
err_cq:
ibv_dealloc_pd(pd);
return status;
}
static int fi_ibv_domain_close(fid_t fid)
{
struct fi_ibv_domain *domain;
int ret;
domain = container_of(fid, struct fi_ibv_domain, domain_fid.fid);
if (domain->pd) {
ret = ibv_dealloc_pd(domain->pd);
if (ret)
return -ret;
domain->pd = NULL;
}
fi_freeinfo(domain->info);
free(domain);
return 0;
}
int
rd_close_2(DEVICE *dev)
{
if (dev->pd)
ibv_dealloc_pd(dev->pd);
if (dev->channel)
ibv_destroy_comp_channel(dev->channel);
#if 0
if (!Req.use_cm)
#endif
ib_close2(dev);
memset(dev, 0, sizeof(*dev));
return 0;
}
int
resource_destroy(resource_t *res)
{
int rc = 0;
// Delete QP
if (res->qp && ibv_destroy_qp(res->qp)){
fprintf(stderr, "failed to destroy QP\n");
rc = 1;
}
// Deregister MR
if(res->mr_list && res->mr_size > 0){
int i;
for(i=0; i<res->mr_size; i++){
ibv_dereg_mr(res->mr_list[i]);
}
free(res->mr_list);
}
if(res->buf){
free(res->buf);
}
// Delete CQ
if (res->scq && ibv_destroy_cq(res->scq)){
fprintf(stderr, "failed to destroy SCQ\n");
rc = 1;
}
if (res->rcq && ibv_destroy_cq(res->rcq)){
fprintf(stderr, "failed to destroy RCQ\n");
rc = 1;
}
if(res->comp_ch && ibv_destroy_comp_channel(res->comp_ch)){
fprintf(stderr, "failed to destroy Complete CH\n");
rc = 1;
}
// Deallocate PD
if (res->pd && ibv_dealloc_pd(res->pd)){
fprintf(stderr, "failed to deallocate PD\n");
rc = 1;
}
if (res->ib_ctx && ibv_close_device(res->ib_ctx)){
fprintf(stderr, "failed to close device context\n");
rc = 1;
}
return rc;
}
static int destroy_ctx_resources(struct pingpong_context *ctx) {
int test_result = 0;
if (ibv_destroy_qp(ctx->qp)) {
fprintf(stderr, "failed to destroy QP\n");
test_result = 1;
}
if (ibv_destroy_cq(ctx->cq)) {
fprintf(stderr, "failed to destroy CQ\n");
test_result = 1;
}
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "failed to deregister MR\n");
test_result = 1;
}
if (ctx->channel) {
if (ibv_destroy_comp_channel(ctx->channel)) {
fprintf(stderr, "failed to destroy channel \n");
test_result = 1;
}
}
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
test_result = 1;
}
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "failed to close device context\n");
test_result = 1;
}
free(ctx->buf);
free(ctx);
free(tstamp);
return test_result;
}
static int destroy_ctx_resources(struct pingpong_context *ctx,int num_qps) {
int i;
int test_result = 0;
for (i = 0; i < num_qps; i++) {
if (ibv_destroy_qp(ctx->qp[i])) {
fprintf(stderr, "failed to destroy QP\n");
test_result = 1;
}
}
if (ibv_destroy_cq(ctx->cq)) {
fprintf(stderr, "failed to destroy CQ\n");
test_result = 1;
}
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "failed to deregister MR\n");
test_result = 1;
}
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
test_result = 1;
}
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "failed to close device context\n");
test_result = 1;
}
free(ctx->buf);
free(ctx->qp);
free(ctx->scnt);
free(ctx->ccnt);
free(ctx);
return test_result;
}
void release_resources(void)
{
int i;
// Destroy the registration cache
reg_cache_destroy(nprocs);
for(i = 0; i < nprocs ; i++) {
if (me != i) {
if(ibv_destroy_qp(conn.qp[i])) {
printf("Exiting\n");
exit(1);
}
}
}
#if 0
if(ibv_destroy_cq(hca.cq)) {
fprintf(stderr,"Couldn't destroy cq %s\n",
ibv_get_device_name(hca.ib_dev));
}
if(ibv_dealloc_pd(hca.pd)) {
fprintf(stderr,"Couldn't free pd %s\n",
ibv_get_device_name(hca.ib_dev));
}
#endif
free(conn.qp);
free(conn.lid);
free(conn.qp_num);
free(rbuf.qp_num);
free(rbuf.lid);
#if 0
free(rbuf.rkey);
free(rbuf.buf);
#endif
}
/**
* DPDK callback to close the device.
*
* Destroy all queues and objects, free memory.
*
* @param dev
* Pointer to Ethernet device structure.
*/
static void
mlx5_dev_close(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
void *tmp;
unsigned int i;
priv_lock(priv);
DEBUG("%p: closing device \"%s\"",
(void *)dev,
((priv->ctx != NULL) ? priv->ctx->device->name : ""));
/* In case mlx5_dev_stop() has not been called. */
priv_dev_interrupt_handler_uninstall(priv, dev);
priv_allmulticast_disable(priv);
priv_promiscuous_disable(priv);
priv_mac_addrs_disable(priv);
priv_destroy_hash_rxqs(priv);
/* Prevent crashes when queues are still in use. */
dev->rx_pkt_burst = removed_rx_burst;
dev->tx_pkt_burst = removed_tx_burst;
if (priv->rxqs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
usleep(1000);
for (i = 0; (i != priv->rxqs_n); ++i) {
tmp = (*priv->rxqs)[i];
if (tmp == NULL)
continue;
(*priv->rxqs)[i] = NULL;
rxq_cleanup(tmp);
rte_free(tmp);
}
priv->rxqs_n = 0;
priv->rxqs = NULL;
}
if (priv->txqs != NULL) {
/* XXX race condition if mlx5_tx_burst() is still running. */
usleep(1000);
for (i = 0; (i != priv->txqs_n); ++i) {
tmp = (*priv->txqs)[i];
if (tmp == NULL)
continue;
(*priv->txqs)[i] = NULL;
txq_cleanup(tmp);
rte_free(tmp);
}
priv->txqs_n = 0;
priv->txqs = NULL;
}
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
claim_zero(ibv_dealloc_pd(priv->pd));
claim_zero(ibv_close_device(priv->ctx));
} else
assert(priv->ctx == NULL);
if (priv->rss_conf != NULL) {
for (i = 0; (i != hash_rxq_init_n); ++i)
rte_free((*priv->rss_conf)[i]);
rte_free(priv->rss_conf);
}
priv_unlock(priv);
memset(priv, 0, sizeof(*priv));
}
/**
* DPDK callback to register a PCI device.
*
* This function creates an Ethernet device for each port of a given
* PCI device.
*
* @param[in] pci_drv
* PCI driver structure (mlx5_driver).
* @param[in] pci_dev
* PCI device information.
*
* @return
* 0 on success, negative errno value on failure.
*/
static int
mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
{
struct ibv_device **list;
struct ibv_device *ibv_dev;
int err = 0;
struct ibv_context *attr_ctx = NULL;
struct ibv_device_attr device_attr;
unsigned int vf;
int idx;
int i;
(void)pci_drv;
assert(pci_drv == &mlx5_driver.pci_drv);
/* Get mlx5_dev[] index. */
idx = mlx5_dev_idx(&pci_dev->addr);
if (idx == -1) {
ERROR("this driver cannot support any more adapters");
return -ENOMEM;
}
DEBUG("using driver device index %d", idx);
/* Save PCI address. */
mlx5_dev[idx].pci_addr = pci_dev->addr;
list = ibv_get_device_list(&i);
if (list == NULL) {
assert(errno);
if (errno == ENOSYS) {
WARN("cannot list devices, is ib_uverbs loaded?");
return 0;
}
return -errno;
}
assert(i >= 0);
/*
* For each listed device, check related sysfs entry against
* the provided PCI ID.
*/
while (i != 0) {
struct rte_pci_addr pci_addr;
--i;
DEBUG("checking device \"%s\"", list[i]->name);
if (mlx5_ibv_device_to_pci_addr(list[i], &pci_addr))
continue;
if ((pci_dev->addr.domain != pci_addr.domain) ||
(pci_dev->addr.bus != pci_addr.bus) ||
(pci_dev->addr.devid != pci_addr.devid) ||
(pci_dev->addr.function != pci_addr.function))
continue;
vf = ((pci_dev->id.device_id ==
PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) ||
(pci_dev->id.device_id ==
PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF));
INFO("PCI information matches, using device \"%s\" (VF: %s)",
list[i]->name, (vf ? "true" : "false"));
attr_ctx = ibv_open_device(list[i]);
err = errno;
break;
}
if (attr_ctx == NULL) {
ibv_free_device_list(list);
switch (err) {
case 0:
WARN("cannot access device, is mlx5_ib loaded?");
return 0;
case EINVAL:
WARN("cannot use device, are drivers up to date?");
return 0;
}
assert(err > 0);
return -err;
}
ibv_dev = list[i];
DEBUG("device opened");
if (ibv_query_device(attr_ctx, &device_attr))
goto error;
INFO("%u port(s) detected", device_attr.phys_port_cnt);
for (i = 0; i < device_attr.phys_port_cnt; i++) {
uint32_t port = i + 1; /* ports are indexed from one */
uint32_t test = (1 << i);
struct ibv_context *ctx = NULL;
struct ibv_port_attr port_attr;
struct ibv_pd *pd = NULL;
struct priv *priv = NULL;
struct rte_eth_dev *eth_dev;
#ifdef HAVE_EXP_QUERY_DEVICE
struct ibv_exp_device_attr exp_device_attr;
#endif /* HAVE_EXP_QUERY_DEVICE */
struct ether_addr mac;
#ifdef HAVE_EXP_QUERY_DEVICE
exp_device_attr.comp_mask =
IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS |
IBV_EXP_DEVICE_ATTR_RX_HASH;
#endif /* HAVE_EXP_QUERY_DEVICE */
DEBUG("using port %u (%08" PRIx32 ")", port, test);
ctx = ibv_open_device(ibv_dev);
if (ctx == NULL)
goto port_error;
/* Check port status. */
err = ibv_query_port(ctx, port, &port_attr);
if (err) {
ERROR("port query failed: %s", strerror(err));
goto port_error;
}
if (port_attr.state != IBV_PORT_ACTIVE)
DEBUG("port %d is not active: \"%s\" (%d)",
port, ibv_port_state_str(port_attr.state),
port_attr.state);
/* Allocate protection domain. */
pd = ibv_alloc_pd(ctx);
if (pd == NULL) {
ERROR("PD allocation failure");
err = ENOMEM;
goto port_error;
}
mlx5_dev[idx].ports |= test;
/* from rte_ethdev.c */
priv = rte_zmalloc("ethdev private structure",
sizeof(*priv),
RTE_CACHE_LINE_SIZE);
if (priv == NULL) {
ERROR("priv allocation failure");
err = ENOMEM;
goto port_error;
}
priv->ctx = ctx;
priv->device_attr = device_attr;
priv->port = port;
priv->pd = pd;
priv->mtu = ETHER_MTU;
#ifdef HAVE_EXP_QUERY_DEVICE
if (ibv_exp_query_device(ctx, &exp_device_attr)) {
ERROR("ibv_exp_query_device() failed");
goto port_error;
}
priv->hw_csum =
((exp_device_attr.exp_device_cap_flags &
IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) &&
(exp_device_attr.exp_device_cap_flags &
IBV_EXP_DEVICE_RX_CSUM_IP_PKT));
DEBUG("checksum offloading is %ssupported",
(priv->hw_csum ? "" : "not "));
priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags &
IBV_EXP_DEVICE_VXLAN_SUPPORT);
DEBUG("L2 tunnel checksum offloads are %ssupported",
(priv->hw_csum_l2tun ? "" : "not "));
priv->ind_table_max_size = exp_device_attr.rx_hash_caps.max_rwq_indirection_table_size;
DEBUG("maximum RX indirection table size is %u",
priv->ind_table_max_size);
#else /* HAVE_EXP_QUERY_DEVICE */
priv->ind_table_max_size = RSS_INDIRECTION_TABLE_SIZE;
#endif /* HAVE_EXP_QUERY_DEVICE */
priv->vf = vf;
/* Allocate and register default RSS hash keys. */
priv->rss_conf = rte_calloc(__func__, hash_rxq_init_n,
sizeof((*priv->rss_conf)[0]), 0);
if (priv->rss_conf == NULL) {
err = ENOMEM;
goto port_error;
}
err = rss_hash_rss_conf_new_key(priv,
rss_hash_default_key,
rss_hash_default_key_len,
ETH_RSS_PROTO_MASK);
if (err)
goto port_error;
/* Configure the first MAC address by default. */
if (priv_get_mac(priv, &mac.addr_bytes)) {
ERROR("cannot get MAC address, is mlx5_en loaded?"
" (errno: %s)", strerror(errno));
goto port_error;
}
INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
priv->port,
mac.addr_bytes[0], mac.addr_bytes[1],
mac.addr_bytes[2], mac.addr_bytes[3],
mac.addr_bytes[4], mac.addr_bytes[5]);
/* Register MAC and broadcast addresses. */
claim_zero(priv_mac_addr_add(priv, 0,
(const uint8_t (*)[ETHER_ADDR_LEN])
mac.addr_bytes));
claim_zero(priv_mac_addr_add(priv, (RTE_DIM(priv->mac) - 1),
&(const uint8_t [ETHER_ADDR_LEN])
{ "\xff\xff\xff\xff\xff\xff" }));
#ifndef NDEBUG
{
char ifname[IF_NAMESIZE];
if (priv_get_ifname(priv, &ifname) == 0)
DEBUG("port %u ifname is \"%s\"",
priv->port, ifname);
else
DEBUG("port %u ifname is unknown", priv->port);
}
#endif
/* Get actual MTU if possible. */
priv_get_mtu(priv, &priv->mtu);
DEBUG("port %u MTU is %u", priv->port, priv->mtu);
/* from rte_ethdev.c */
{
char name[RTE_ETH_NAME_MAX_LEN];
snprintf(name, sizeof(name), "%s port %u",
ibv_get_device_name(ibv_dev), port);
eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_PCI);
}
if (eth_dev == NULL) {
ERROR("can not allocate rte ethdev");
err = ENOMEM;
goto port_error;
}
eth_dev->data->dev_private = priv;
eth_dev->pci_dev = pci_dev;
eth_dev->driver = &mlx5_driver;
eth_dev->data->rx_mbuf_alloc_failed = 0;
eth_dev->data->mtu = ETHER_MTU;
priv->dev = eth_dev;
eth_dev->dev_ops = &mlx5_dev_ops;
eth_dev->data->mac_addrs = priv->mac;
TAILQ_INIT(ð_dev->link_intr_cbs);
/* Bring Ethernet device up. */
DEBUG("forcing Ethernet interface up");
priv_set_flags(priv, ~IFF_UP, IFF_UP);
continue;
port_error:
rte_free(priv->rss_conf);
rte_free(priv);
if (pd)
claim_zero(ibv_dealloc_pd(pd));
if (ctx)
claim_zero(ibv_close_device(ctx));
break;
}
ProtectionDomain::~ProtectionDomain() {
if (mDomain != nullptr && ibv_dealloc_pd(mDomain)) {
std::error_code ec(errno, std::generic_category());
LOG_ERROR("Failed to deallocate protection domain [error = %1% %2%]", ec, ec.message());
}
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
verbs_runtime_query(mca_base_module_t **module,
int *priority,
const char *hint)
{
int rc = OSHMEM_SUCCESS;
openib_device_t my_device;
openib_device_t *device = &my_device;
int num_devs = 0;
int i = 0;
*priority = 0;
*module = NULL;
memset(device, 0, sizeof(*device));
#ifdef HAVE_IBV_GET_DEVICE_LIST
device->ib_devs = ibv_get_device_list(&num_devs);
#else
#error unsupported ibv_get_device_list in infiniband/verbs.h
#endif
if (num_devs == 0 || !device->ib_devs) {
return OSHMEM_ERR_NOT_SUPPORTED;
}
/* Open device */
if (NULL != mca_sshmem_verbs_component.hca_name) {
for (i = 0; i < num_devs; i++) {
if (0 == strcmp(mca_sshmem_verbs_component.hca_name, ibv_get_device_name(device->ib_devs[i]))) {
device->ib_dev = device->ib_devs[i];
break;
}
}
} else {
device->ib_dev = device->ib_devs[0];
}
if (NULL == device->ib_dev) {
rc = OSHMEM_ERR_NOT_FOUND;
goto out;
}
if (NULL == (device->ib_dev_context = ibv_open_device(device->ib_dev))) {
rc = OSHMEM_ERR_RESOURCE_BUSY;
goto out;
}
/* Obtain device attributes */
if (ibv_query_device(device->ib_dev_context, &device->ib_dev_attr)) {
rc = OSHMEM_ERR_RESOURCE_BUSY;
goto out;
}
/* Allocate the protection domain for the device */
device->ib_pd = ibv_alloc_pd(device->ib_dev_context);
if (NULL == device->ib_pd) {
rc = OSHMEM_ERR_RESOURCE_BUSY;
goto out;
}
/* Allocate memory */
if (!rc) {
void *addr = NULL;
size_t size = getpagesize();
struct ibv_mr *ib_mr = NULL;
uint64_t access_flag = IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ;
uint64_t exp_access_flag = 0;
OBJ_CONSTRUCT(&device->ib_mr_array, opal_value_array_t);
opal_value_array_init(&device->ib_mr_array, sizeof(struct ibv_mr *));
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
exp_access_flag = IBV_EXP_ACCESS_ALLOCATE_MR |
IBV_EXP_ACCESS_SHARED_MR_USER_READ |
IBV_EXP_ACCESS_SHARED_MR_USER_WRITE;
#endif /* MPAGE_ENABLE */
struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag|exp_access_flag, 0};
ib_mr = ibv_exp_reg_mr(&in);
if (NULL == ib_mr) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
} else {
device->ib_mr_shared = ib_mr;
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
}
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
if (!rc) {
struct ibv_exp_reg_shared_mr_in in_smr;
access_flag = IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ|
IBV_EXP_ACCESS_NO_RDMA;
addr = (void *)mca_sshmem_base_start_address;
mca_sshmem_verbs_fill_shared_mr(&in_smr, device->ib_pd, device->ib_mr_shared->handle, addr, access_flag);
ib_mr = ibv_exp_reg_shared_mr(&in_smr);
if (NULL == ib_mr) {
mca_sshmem_verbs_component.has_shared_mr = 0;
} else {
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
mca_sshmem_verbs_component.has_shared_mr = 1;
}
}
#endif /* MPAGE_ENABLE */
}
/* all is well - rainbows and butterflies */
if (!rc) {
*priority = mca_sshmem_verbs_component.priority;
*module = (mca_base_module_t *)&mca_sshmem_verbs_module.super;
}
out:
if (device) {
if (opal_value_array_get_size(&device->ib_mr_array)) {
struct ibv_mr** array;
struct ibv_mr* ib_mr = NULL;
array = OPAL_VALUE_ARRAY_GET_BASE(&device->ib_mr_array, struct ibv_mr *);
while (opal_value_array_get_size(&device->ib_mr_array) > 0) {
ib_mr = array[0];
ibv_dereg_mr(ib_mr);
opal_value_array_remove_item(&device->ib_mr_array, 0);
}
if (device->ib_mr_shared) {
device->ib_mr_shared = NULL;
}
OBJ_DESTRUCT(&device->ib_mr_array);
}
if (device->ib_pd) {
ibv_dealloc_pd(device->ib_pd);
device->ib_pd = NULL;
}
if(device->ib_dev_context) {
ibv_close_device(device->ib_dev_context);
device->ib_dev_context = NULL;
}
if(device->ib_devs) {
ibv_free_device_list(device->ib_devs);
device->ib_devs = NULL;
}
}
return rc;
}