void rdma_backend_fini(RdmaBackendDev *backend_dev)
{
mad_fini(backend_dev);
g_hash_table_destroy(ah_hash);
ibv_destroy_comp_channel(backend_dev->channel);
ibv_close_device(backend_dev->context);
}
static void
rdma_trans_destroy(void *a)
{
Rdmatrans *rdma;
struct ibv_qp_attr attr;
rdma = a;
if (rdma->connected)
rdma_disconnect(rdma->cm_id);
if (rdma->qp) {
attr.qp_state = IBV_QPS_ERR;
ibv_modify_qp(rdma->qp, &attr, IBV_QP_STATE);
ibv_destroy_qp(rdma->qp);
}
if (rdma->cq)
ibv_destroy_cq(rdma->cq);
if (rdma->ch)
ibv_destroy_comp_channel(rdma->ch);
if (rdma->snd_mr)
ibv_dereg_mr(rdma->snd_mr);
if (rdma->snd_buf)
free(rdma->snd_buf);
if (rdma->rcv_mr)
ibv_dereg_mr(rdma->rcv_mr);
if (rdma->rcv_buf)
free(rdma->rcv_buf);
if (rdma->pd)
ibv_dealloc_pd(rdma->pd);
if (rdma->cm_id)
rdma_destroy_id(rdma->cm_id);
}
static void rping_free_qp(struct rping_cb *cb)
{
ibv_destroy_qp(cb->qp);
ibv_destroy_cq(cb->cq);
ibv_destroy_comp_channel(cb->channel);
ibv_dealloc_pd(cb->pd);
}
RDMAAdapter::~RDMAAdapter() {
StopInternalThread();
CHECK(!ibv_destroy_cq(cq_)) << "Failed to destroy CQ";
CHECK(!ibv_destroy_comp_channel(channel_)) << "Failed to destroy channel";
CHECK(!ibv_dealloc_pd(pd_)) << "Failed to deallocate PD";
CHECK(!ibv_close_device(context_)) << "Failed to release context";
}
void xfer_rdma_destroy_ctx(struct xfer_context *ctx) {
rdma_destroy_qp(ctx->cm_id);
ibv_destroy_cq(ctx->cq);
ibv_destroy_comp_channel(ctx->ch);
ibv_dereg_mr(ctx->send_mr);
ibv_dereg_mr(ctx->recv_mr);
ibv_dealloc_pd(ctx->pd);
free(ctx);
}
CompletionChannel& CompletionChannel::operator=(CompletionChannel&& other) {
if (mChannel != nullptr && ibv_destroy_comp_channel(mChannel)) {
throw std::system_error(errno, std::generic_category());
}
mChannel = other.mChannel;
other.mChannel = nullptr;
return *this;
}
void network_release() {
ibv_dereg_mr(mr_data);
rdma_destroy_qp(cm_id);
ibv_destroy_cq(cq);
ibv_destroy_comp_channel(comp_chan);
rdma_destroy_id(cm_id);
rdma_destroy_event_channel(cm_channel);
}
static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f)
{
struct rdmaio_data *rd = td->io_ops->data;
struct ibv_send_wr *bad_wr;
/* unregister rdma buffer */
/*
* Client sends notification to the server side
*/
/* refer to: http://linux.die.net/man/7/rdma_cm */
if ((rd->is_client == 1) && ((rd->rdma_protocol == FIO_RDMA_MEM_WRITE)
|| (rd->rdma_protocol ==
FIO_RDMA_MEM_READ))) {
if (ibv_post_send(rd->qp, &rd->sq_wr, &bad_wr) != 0) {
log_err("fio: ibv_post_send fail");
return 1;
}
dprint(FD_IO, "fio: close information sent success\n");
rdma_poll_wait(td, IBV_WC_SEND);
}
if (rd->is_client == 1)
rdma_disconnect(rd->cm_id);
else {
rdma_disconnect(rd->child_cm_id);
#if 0
rdma_disconnect(rd->cm_id);
#endif
}
#if 0
if (get_next_channel_event(td, rd->cm_channel, RDMA_CM_EVENT_DISCONNECTED) != 0) {
log_err("fio: wait for RDMA_CM_EVENT_DISCONNECTED\n");
return 1;
}
#endif
ibv_destroy_cq(rd->cq);
ibv_destroy_qp(rd->qp);
if (rd->is_client == 1)
rdma_destroy_id(rd->cm_id);
else {
rdma_destroy_id(rd->child_cm_id);
rdma_destroy_id(rd->cm_id);
}
ibv_destroy_comp_channel(rd->channel);
ibv_dealloc_pd(rd->pd);
return 0;
}
static int rping_setup_qp(struct rping_cb *cb, struct rdma_cm_id *cm_id)
{
int ret;
cb->pd = ibv_alloc_pd(cm_id->verbs);
if (!cb->pd) {
fprintf(stderr, "ibv_alloc_pd failed\n");
return errno;
}
DEBUG_LOG("created pd %p\n", cb->pd);
cb->channel = ibv_create_comp_channel(cm_id->verbs);
if (!cb->channel) {
fprintf(stderr, "ibv_create_comp_channel failed\n");
ret = errno;
goto err1;
}
DEBUG_LOG("created channel %p\n", cb->channel);
cb->cq = ibv_create_cq(cm_id->verbs, RPING_SQ_DEPTH * 2, cb,
cb->channel, 0);
if (!cb->cq) {
fprintf(stderr, "ibv_create_cq failed\n");
ret = errno;
goto err2;
}
DEBUG_LOG("created cq %p\n", cb->cq);
ret = ibv_req_notify_cq(cb->cq, 0);
if (ret) {
fprintf(stderr, "ibv_create_cq failed\n");
ret = errno;
goto err3;
}
ret = rping_create_qp(cb);
if (ret) {
perror("rdma_create_qp");
goto err3;
}
DEBUG_LOG("created qp %p\n", cb->qp);
return 0;
err3:
ibv_destroy_cq(cb->cq);
err2:
ibv_destroy_comp_channel(cb->channel);
err1:
ibv_dealloc_pd(cb->pd);
return ret;
}
/** ========================================================================= */
static void destroy_sa_qp(struct oib_port *port)
{
int i;
// if the user just unregistered trap messages those messages may still
// be on this list, wait 5 seconds for the thread to handle the response.
for (i = 0; i < 5000; i++) {
if (!LIST_EMPTY(&port->pending_reg_msg_head)) {
usleep(1000);
}
else {
DBGPRINT("destroy_sa_qp: wait %d ms for LIST_EMPTY\n", i);
break;
}
}
stop_ud_cq_monitor(port);
join_port_thread(port);
/* Free any remaining unregistration messages */
if (!LIST_EMPTY(&port->pending_reg_msg_head)) {
OUTPUT_ERROR("Ignoring Pending Notice un-registation requests\n");
oib_sa_remove_all_pending_reg_msgs(port);
}
if (port->sa_ah)
ibv_destroy_ah(port->sa_ah);
if (port->sa_qp)
ibv_destroy_qp(port->sa_qp);
for (i = 0; i<port->num_userspace_recv_buf; i++)
if (port->recv_bufs)
ibv_dereg_mr(port->recv_bufs[i].mr);
if (port->sa_qp_pd)
ibv_dealloc_pd(port->sa_qp_pd);
if (port->sa_qp_cq)
ibv_destroy_cq(port->sa_qp_cq);
if (port->recv_bufs) {
free(port->recv_bufs);
port->recv_bufs = NULL;
}
if (port->sa_qp_comp_channel)
ibv_destroy_comp_channel(port->sa_qp_comp_channel);
}
static void
rdmasniff_cleanup(pcap_t *handle)
{
struct pcap_rdmasniff *priv = handle->priv;
ibv_dereg_mr(priv->mr);
ibv_destroy_flow(priv->flow);
ibv_destroy_qp(priv->qp);
ibv_destroy_cq(priv->cq);
ibv_dealloc_pd(priv->pd);
ibv_destroy_comp_channel(priv->channel);
ibv_close_device(priv->context);
free(priv->oneshot_buffer);
pcap_cleanup_live_common(handle);
}
static int ibw_conn_priv_destruct(struct ibw_conn_priv *pconn)
{
DEBUG(DEBUG_DEBUG, ("ibw_conn_priv_destruct(%p, cmid: %p)\n",
pconn, pconn->cm_id));
/* pconn->wr_index is freed by talloc */
/* pconn->wr_index[i] are freed by talloc */
/*
* tevent_fd must be removed before the fd is closed
*/
TALLOC_FREE(pconn->verbs_channel_event);
/* destroy verbs */
if (pconn->cm_id!=NULL && pconn->cm_id->qp!=NULL) {
rdma_destroy_qp(pconn->cm_id);
pconn->cm_id->qp = NULL;
}
if (pconn->cq!=NULL) {
ibv_destroy_cq(pconn->cq);
pconn->cq = NULL;
}
if (pconn->verbs_channel!=NULL) {
ibv_destroy_comp_channel(pconn->verbs_channel);
pconn->verbs_channel = NULL;
}
/* free memory regions */
ibw_free_mr(&pconn->buf_send, &pconn->mr_send);
ibw_free_mr(&pconn->buf_recv, &pconn->mr_recv);
if (pconn->pd) {
ibv_dealloc_pd(pconn->pd);
pconn->pd = NULL;
DEBUG(DEBUG_DEBUG, ("pconn=%p pd deallocated\n", pconn));
}
if (pconn->cm_id) {
rdma_destroy_id(pconn->cm_id);
pconn->cm_id = NULL;
DEBUG(DEBUG_DEBUG, ("pconn=%p cm_id destroyed\n", pconn));
}
return 0;
}
int
rd_close_2(DEVICE *dev)
{
if (dev->pd)
ibv_dealloc_pd(dev->pd);
if (dev->channel)
ibv_destroy_comp_channel(dev->channel);
#if 0
if (!Req.use_cm)
#endif
ib_close2(dev);
memset(dev, 0, sizeof(*dev));
return 0;
}
int
resource_destroy(resource_t *res)
{
int rc = 0;
// Delete QP
if (res->qp && ibv_destroy_qp(res->qp)){
fprintf(stderr, "failed to destroy QP\n");
rc = 1;
}
// Deregister MR
if(res->mr_list && res->mr_size > 0){
int i;
for(i=0; i<res->mr_size; i++){
ibv_dereg_mr(res->mr_list[i]);
}
free(res->mr_list);
}
if(res->buf){
free(res->buf);
}
// Delete CQ
if (res->scq && ibv_destroy_cq(res->scq)){
fprintf(stderr, "failed to destroy SCQ\n");
rc = 1;
}
if (res->rcq && ibv_destroy_cq(res->rcq)){
fprintf(stderr, "failed to destroy RCQ\n");
rc = 1;
}
if(res->comp_ch && ibv_destroy_comp_channel(res->comp_ch)){
fprintf(stderr, "failed to destroy Complete CH\n");
rc = 1;
}
// Deallocate PD
if (res->pd && ibv_dealloc_pd(res->pd)){
fprintf(stderr, "failed to deallocate PD\n");
rc = 1;
}
if (res->ib_ctx && ibv_close_device(res->ib_ctx)){
fprintf(stderr, "failed to close device context\n");
rc = 1;
}
return rc;
}
static int destroy_ctx_resources(struct pingpong_context *ctx) {
int test_result = 0;
if (ibv_destroy_qp(ctx->qp)) {
fprintf(stderr, "failed to destroy QP\n");
test_result = 1;
}
if (ibv_destroy_cq(ctx->cq)) {
fprintf(stderr, "failed to destroy CQ\n");
test_result = 1;
}
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "failed to deregister MR\n");
test_result = 1;
}
if (ctx->channel) {
if (ibv_destroy_comp_channel(ctx->channel)) {
fprintf(stderr, "failed to destroy channel \n");
test_result = 1;
}
}
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
test_result = 1;
}
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "failed to close device context\n");
test_result = 1;
}
free(ctx->buf);
free(ctx);
free(tstamp);
return test_result;
}
static UCS_CLASS_CLEANUP_FUNC(uct_ib_iface_t)
{
int ret;
ret = ibv_destroy_cq(self->recv_cq);
if (ret != 0) {
ucs_warn("ibv_destroy_cq(recv_cq) returned %d: %m", ret);
}
ret = ibv_destroy_cq(self->send_cq);
if (ret != 0) {
ucs_warn("ibv_destroy_cq(send_cq) returned %d: %m", ret);
}
ret = ibv_destroy_comp_channel(self->comp_channel);
if (ret != 0) {
ucs_warn("ibv_destroy_comp_channel(comp_channel) returned %d: %m", ret);
}
ucs_free(self->path_bits);
}
static int pp_close_ctx(void)
{
int i;
for (i = 0; i < ctx.num_clients; ++i) {
if (ibv_destroy_qp(ctx.send_qp[i])) {
fprintf(stderr, "Couldn't destroy INI QP[%d]\n", i);
return 1;
}
if (ibv_destroy_qp(ctx.recv_qp[i])) {
fprintf(stderr, "Couldn't destroy TGT QP[%d]\n", i);
return 1;
}
if (ctx.rem_dest[i].sockfd)
close(ctx.rem_dest[i].sockfd);
}
if (ibv_destroy_srq(ctx.srq)) {
fprintf(stderr, "Couldn't destroy SRQ\n");
return 1;
}
if (ctx.xrcd && ibv_close_xrcd(ctx.xrcd)) {
fprintf(stderr, "Couldn't close the XRC Domain\n");
return 1;
}
if (ctx.fd >= 0 && close(ctx.fd)) {
fprintf(stderr, "Couldn't close the file for the XRC Domain\n");
return 1;
}
if (ibv_destroy_cq(ctx.send_cq)) {
fprintf(stderr, "Couldn't destroy send CQ\n");
return 1;
}
if (ibv_destroy_cq(ctx.recv_cq)) {
fprintf(stderr, "Couldn't destroy recv CQ\n");
return 1;
}
if (ibv_dereg_mr(ctx.mr)) {
fprintf(stderr, "Couldn't deregister MR\n");
return 1;
}
if (ibv_dealloc_pd(ctx.pd)) {
fprintf(stderr, "Couldn't deallocate PD\n");
return 1;
}
if (ctx.channel) {
if (ibv_destroy_comp_channel(ctx.channel)) {
fprintf(stderr,
"Couldn't destroy completion channel\n");
return 1;
}
}
if (ibv_close_device(ctx.context)) {
fprintf(stderr, "Couldn't release context\n");
return 1;
}
free(ctx.buf);
free(ctx.rem_dest);
free(ctx.send_qp);
free(ctx.recv_qp);
return 0;
}
/*****************************************
* Function: resources_destroy
*****************************************/
static int resources_destroy(
struct resources *res)
{
int test_result = 0;
if (res->qp) {
if (ibv_destroy_qp(res->qp)) {
fprintf(stderr, "failed to destroy QP\n");
test_result = 1;
}
}
if (res->mr) {
if (ibv_dereg_mr(res->mr)) {
fprintf(stderr, "failed to deregister MR\n");
test_result = 1;
}
}
if (res->buf)
free(res->buf);
if (res->cq) {
if (ibv_destroy_cq(res->cq)) {
fprintf(stderr, "failed to destroy CQ\n");
test_result = 1;
}
}
if (res->comp_channel) {
if (ibv_destroy_comp_channel(res->comp_channel)) {
fprintf(stderr, "failed to destroy completion channel\n");
test_result = 1;
}
}
if (res->pd) {
if (ibv_dealloc_pd(res->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
test_result = 1;
}
}
if (res->ib_ctx) {
if (ibv_close_device(res->ib_ctx)) {
fprintf(stderr, "failed to close device context\n");
test_result = 1;
}
}
if (res->dev_list)
ibv_free_device_list(res->dev_list);
if (res->sock >= 0) {
if (close(res->sock)) {
fprintf(stderr, "failed to close socket\n");
test_result = 1;
}
}
return test_result;
}
CompletionChannel::~CompletionChannel() {
if (mChannel != nullptr && ibv_destroy_comp_channel(mChannel)) {
std::error_code ec(errno, std::generic_category());
LOG_ERROR("Failed to destroy completion channel [error = %1% %2%]", ec, ec.message());
}
}
static int
rdmasniff_activate(pcap_t *handle)
{
struct pcap_rdmasniff *priv = handle->priv;
struct ibv_qp_init_attr qp_init_attr;
struct ibv_qp_attr qp_attr;
struct ibv_flow_attr flow_attr;
struct ibv_port_attr port_attr;
int i;
priv->context = ibv_open_device(priv->rdma_device);
if (!priv->context) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to open device %s", handle->opt.device);
goto error;
}
priv->pd = ibv_alloc_pd(priv->context);
if (!priv->pd) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to alloc PD for device %s", handle->opt.device);
goto error;
}
priv->channel = ibv_create_comp_channel(priv->context);
if (!priv->channel) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to create comp channel for device %s", handle->opt.device);
goto error;
}
priv->cq = ibv_create_cq(priv->context, RDMASNIFF_NUM_RECEIVES,
NULL, priv->channel, 0);
if (!priv->cq) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to create CQ for device %s", handle->opt.device);
goto error;
}
ibv_req_notify_cq(priv->cq, 0);
memset(&qp_init_attr, 0, sizeof qp_init_attr);
qp_init_attr.send_cq = qp_init_attr.recv_cq = priv->cq;
qp_init_attr.cap.max_recv_wr = RDMASNIFF_NUM_RECEIVES;
qp_init_attr.cap.max_recv_sge = 1;
qp_init_attr.qp_type = IBV_QPT_RAW_PACKET;
priv->qp = ibv_create_qp(priv->pd, &qp_init_attr);
if (!priv->qp) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to create QP for device %s", handle->opt.device);
goto error;
}
memset(&qp_attr, 0, sizeof qp_attr);
qp_attr.qp_state = IBV_QPS_INIT;
qp_attr.port_num = priv->port_num;
if (ibv_modify_qp(priv->qp, &qp_attr, IBV_QP_STATE | IBV_QP_PORT)) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to modify QP to INIT for device %s", handle->opt.device);
goto error;
}
memset(&qp_attr, 0, sizeof qp_attr);
qp_attr.qp_state = IBV_QPS_RTR;
if (ibv_modify_qp(priv->qp, &qp_attr, IBV_QP_STATE)) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to modify QP to RTR for device %s", handle->opt.device);
goto error;
}
memset(&flow_attr, 0, sizeof flow_attr);
flow_attr.type = IBV_FLOW_ATTR_SNIFFER;
flow_attr.size = sizeof flow_attr;
flow_attr.port = priv->port_num;
priv->flow = ibv_create_flow(priv->qp, &flow_attr);
if (!priv->flow) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to create flow for device %s", handle->opt.device);
goto error;
}
handle->bufsize = RDMASNIFF_NUM_RECEIVES * RDMASNIFF_RECEIVE_SIZE;
handle->buffer = malloc(handle->bufsize);
if (!handle->buffer) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to allocate receive buffer for device %s", handle->opt.device);
goto error;
}
priv->oneshot_buffer = malloc(RDMASNIFF_RECEIVE_SIZE);
if (!priv->oneshot_buffer) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to allocate oneshot buffer for device %s", handle->opt.device);
goto error;
}
priv->mr = ibv_reg_mr(priv->pd, handle->buffer, handle->bufsize, IBV_ACCESS_LOCAL_WRITE);
if (!priv->mr) {
pcap_snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
"Failed to register MR for device %s", handle->opt.device);
goto error;
}
for (i = 0; i < RDMASNIFF_NUM_RECEIVES; ++i) {
rdmasniff_post_recv(handle, i);
}
if (!ibv_query_port(priv->context, priv->port_num, &port_attr) &&
port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) {
handle->linktype = DLT_INFINIBAND;
} else {
handle->linktype = DLT_EN10MB;
}
if (handle->snapshot <= 0 || handle->snapshot > RDMASNIFF_RECEIVE_SIZE)
handle->snapshot = RDMASNIFF_RECEIVE_SIZE;
handle->offset = 0;
handle->read_op = rdmasniff_read;
handle->stats_op = rdmasniff_stats;
handle->cleanup_op = rdmasniff_cleanup;
handle->setfilter_op = install_bpf_program;
handle->setdirection_op = NULL;
handle->set_datalink_op = NULL;
handle->getnonblock_op = pcap_getnonblock_fd;
handle->setnonblock_op = pcap_setnonblock_fd;
handle->oneshot_callback = rdmasniff_oneshot;
handle->selectable_fd = priv->channel->fd;
return 0;
error:
if (priv->mr) {
ibv_dereg_mr(priv->mr);
}
if (priv->flow) {
ibv_destroy_flow(priv->flow);
}
if (priv->qp) {
ibv_destroy_qp(priv->qp);
}
if (priv->cq) {
ibv_destroy_cq(priv->cq);
}
if (priv->channel) {
ibv_destroy_comp_channel(priv->channel);
}
if (priv->pd) {
ibv_dealloc_pd(priv->pd);
}
if (priv->context) {
ibv_close_device(priv->context);
}
if (priv->oneshot_buffer) {
free(priv->oneshot_buffer);
}
return PCAP_ERROR;
}
/**
* @param rx_headroom Headroom requested by the user.
* @param rx_priv_len Length of transport private data to reserve (0 if unused)
* @param rx_hdr_len Length of transport network header.
* @param mss Maximal segment size (transport limit).
*/
UCS_CLASS_INIT_FUNC(uct_ib_iface_t, uct_ib_iface_ops_t *ops, uct_md_h md,
uct_worker_h worker, const char *dev_name, unsigned rx_headroom,
unsigned rx_priv_len, unsigned rx_hdr_len, unsigned tx_cq_len,
size_t mss, uct_ib_iface_config_t *config)
{
uct_ib_device_t *dev = &ucs_derived_of(md, uct_ib_md_t)->dev;
ucs_status_t status;
uint8_t port_num;
UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &ops->super, md, worker,
&config->super UCS_STATS_ARG(dev->stats));
status = uct_ib_device_find_port(dev, dev_name, &port_num);
if (status != UCS_OK) {
goto err;
}
self->port_num = port_num;
self->sl = config->sl;
self->config.rx_payload_offset = sizeof(uct_ib_iface_recv_desc_t) +
ucs_max(sizeof(uct_am_recv_desc_t) + rx_headroom,
rx_priv_len + rx_hdr_len);
self->config.rx_hdr_offset = self->config.rx_payload_offset - rx_hdr_len;
self->config.rx_headroom_offset= self->config.rx_payload_offset - rx_headroom;
self->config.seg_size = ucs_min(mss, config->super.max_bcopy);
self->config.tx_max_poll = config->tx.max_poll;
self->config.rx_max_poll = config->rx.max_poll;
self->config.rx_max_batch = ucs_min(config->rx.max_batch,
config->rx.queue_len / 4);
self->ops = ops;
status = uct_ib_iface_init_pkey(self, config);
if (status != UCS_OK) {
goto err;
}
status = uct_ib_iface_init_gid(self, config);
if (status != UCS_OK) {
goto err;
}
status = uct_ib_iface_init_lmc(self, config);
if (status != UCS_OK) {
goto err;
}
self->comp_channel = ibv_create_comp_channel(dev->ibv_context);
if (self->comp_channel == NULL) {
ucs_error("Failed to create completion channel: %m");
status = UCS_ERR_IO_ERROR;
goto err_free_path_bits;
}
status = ucs_sys_fcntl_modfl(self->comp_channel->fd, O_NONBLOCK, 0);
if (status != UCS_OK) {
goto err_destroy_comp_channel;
}
/* TODO inline scatter for send SQ */
self->send_cq = ibv_create_cq(dev->ibv_context, tx_cq_len,
NULL, self->comp_channel, 0);
if (self->send_cq == NULL) {
ucs_error("Failed to create send cq: %m");
status = UCS_ERR_IO_ERROR;
goto err_destroy_comp_channel;
}
if (config->rx.inl > 32 /*UCT_IB_MLX5_CQE64_MAX_INL*/) {
ibv_exp_setenv(dev->ibv_context, "MLX5_CQE_SIZE", "128", 1);
}
self->recv_cq = ibv_create_cq(dev->ibv_context, config->rx.queue_len,
NULL, self->comp_channel, 0);
ibv_exp_setenv(dev->ibv_context, "MLX5_CQE_SIZE", "64", 1);
if (self->recv_cq == NULL) {
ucs_error("Failed to create recv cq: %m");
status = UCS_ERR_IO_ERROR;
goto err_destroy_send_cq;
}
if (!uct_ib_device_is_port_ib(dev, self->port_num)) {
ucs_error("Unsupported link layer");
status = UCS_ERR_UNSUPPORTED;
goto err_destroy_recv_cq;
}
/* Address scope and size */
self->addr_scope = uct_ib_address_scope(self->gid.global.subnet_prefix);
self->addr_size = uct_ib_address_size(self->addr_scope);
ucs_debug("created uct_ib_iface_t headroom_ofs %d payload_ofs %d hdr_ofs %d data_sz %d",
self->config.rx_headroom_offset, self->config.rx_payload_offset,
self->config.rx_hdr_offset, self->config.seg_size);
return UCS_OK;
err_destroy_recv_cq:
ibv_destroy_cq(self->recv_cq);
err_destroy_send_cq:
ibv_destroy_cq(self->send_cq);
err_destroy_comp_channel:
ibv_destroy_comp_channel(self->comp_channel);
err_free_path_bits:
ucs_free(self->path_bits);
err:
return status;
}
static int
mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel)
{
return ibv_destroy_comp_channel(channel);
}
struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
int rx_depth, int port, int use_event,
enum pp_wr_calc_op calc_op,
enum pp_wr_data_type calc_data_type,
char *calc_operands_str)
{
struct pingpong_context *ctx;
int rc;
ctx = malloc(sizeof *ctx);
if (!ctx)
return NULL;
memset(ctx, 0, sizeof *ctx);
ctx->size = size;
ctx->rx_depth = rx_depth;
ctx->calc_op.opcode = IBV_EXP_CALC_OP_NUMBER;
ctx->calc_op.data_type = IBV_EXP_CALC_DATA_TYPE_NUMBER;
ctx->calc_op.data_size = IBV_EXP_CALC_DATA_SIZE_NUMBER;
ctx->buf = memalign(page_size, size);
if (!ctx->buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
goto clean_ctx;
}
memset(ctx->buf, 0, size);
ctx->net_buf = memalign(page_size, size);
if (!ctx->net_buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
goto clean_buffer;
}
memset(ctx->net_buf, 0, size);
ctx->context = ibv_open_device(ib_dev);
if (!ctx->context) {
fprintf(stderr, "Couldn't get context for %s\n",
ibv_get_device_name(ib_dev));
goto clean_net_buf;
}
if (use_event) {
ctx->channel = ibv_create_comp_channel(ctx->context);
if (!ctx->channel) {
fprintf(stderr, "Couldn't create completion channel\n");
goto clean_device;
}
} else
ctx->channel = NULL;
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "Couldn't allocate PD\n");
goto clean_comp_channel;
}
ctx->mr = ibv_reg_mr(ctx->pd, ctx->net_buf, size, IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
fprintf(stderr, "Couldn't register MR\n");
goto clean_pd;
}
if (calc_op != PP_CALC_INVALID) {
int op_per_gather, num_op, max_num_op;
ctx->calc_op.opcode = IBV_EXP_CALC_OP_NUMBER;
ctx->calc_op.data_type = IBV_EXP_CALC_DATA_TYPE_NUMBER;
ctx->calc_op.data_size = IBV_EXP_CALC_DATA_SIZE_NUMBER;
num_op = pp_parse_calc_to_gather(calc_operands_str, calc_op, calc_data_type,
&ctx->calc_op, ctx->context, ctx->buf, ctx->net_buf);
if (num_op < 0) {
fprintf(stderr, "-E- failed parsing calc operators\n");
goto clean_mr;
}
rc = pp_query_calc_cap(ctx->context,
ctx->calc_op.opcode,
ctx->calc_op.data_type,
ctx->calc_op.data_size,
&op_per_gather, &max_num_op);
if (rc) {
fprintf(stderr, "-E- operation not supported on %s. valid ops are:\n",
ibv_get_device_name(ib_dev));
pp_print_dev_calc_ops(ctx->context);
goto clean_mr;
}
if (pp_prepare_sg_list(op_per_gather, num_op, ctx->mr->lkey, &ctx->calc_op, ctx->net_buf)) {
fprintf(stderr, "-failed to prepare the sg list\n");
goto clean_mr;
}
}
ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
ctx->channel, 0);
if (!ctx->cq) {
fprintf(stderr, "Couldn't create CQ\n");
goto clean_mr;
}
{
struct ibv_exp_qp_init_attr attr = {
.send_cq = ctx->cq,
.recv_cq = ctx->cq,
.cap = {
.max_send_wr = 16,
.max_recv_wr = rx_depth,
.max_send_sge = 16,
.max_recv_sge = 16
},
.qp_type = IBV_QPT_RC,
.pd = ctx->pd
};
attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS | IBV_EXP_QP_INIT_ATTR_PD;
attr.exp_create_flags = IBV_EXP_QP_CREATE_CROSS_CHANNEL;
ctx->qp = ibv_exp_create_qp(ctx->context, &attr);
if (!ctx->qp) {
fprintf(stderr, "Couldn't create QP\n");
goto clean_cq;
}
}
{
struct ibv_qp_attr attr = {
.qp_state = IBV_QPS_INIT,
.pkey_index = 0,
.port_num = port,
.qp_access_flags = 0
};
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "Failed to modify QP to INIT\n");
goto clean_qp;
}
}
ctx->mcq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
ctx->channel, 0);
if (!ctx->mcq) {
fprintf(stderr, "Couldn't create CQ for MQP\n");
goto clean_qp;
}
{
struct ibv_exp_qp_init_attr mattr = {
.send_cq = ctx->mcq,
.recv_cq = ctx->mcq,
.cap = {
.max_send_wr = 1,
.max_recv_wr = rx_depth,
.max_send_sge = 16,
.max_recv_sge = 16
},
.qp_type = IBV_QPT_RC,
.pd = ctx->pd
};
mattr.comp_mask |= IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS | IBV_EXP_QP_INIT_ATTR_PD;
mattr.exp_create_flags = IBV_EXP_QP_CREATE_CROSS_CHANNEL;
ctx->mqp = ibv_exp_create_qp(ctx->context, &mattr);
if (!ctx->qp) {
fprintf(stderr, "Couldn't create MQP\n");
goto clean_mcq;
}
}
{
struct ibv_qp_attr mattr = {
.qp_state = IBV_QPS_INIT,
.pkey_index = 0,
.port_num = port,
.qp_access_flags = 0
};
if (ibv_modify_qp(ctx->mqp, &mattr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "Failed to modify MQP to INIT\n");
goto clean_mqp;
}
}
return ctx;
clean_mqp:
ibv_destroy_qp(ctx->mqp);
clean_mcq:
ibv_destroy_cq(ctx->mcq);
clean_qp:
ibv_destroy_qp(ctx->qp);
clean_cq:
ibv_destroy_cq(ctx->cq);
clean_mr:
ibv_dereg_mr(ctx->mr);
clean_pd:
ibv_dealloc_pd(ctx->pd);
clean_comp_channel:
if (ctx->channel)
ibv_destroy_comp_channel(ctx->channel);
clean_device:
ibv_close_device(ctx->context);
clean_net_buf:
free(ctx->net_buf);
clean_buffer:
free(ctx->buf);
clean_ctx:
free(ctx);
return NULL;
}
int pp_close_ctx(struct pingpong_context *ctx)
{
if (ibv_destroy_qp(ctx->qp)) {
fprintf(stderr, "Couldn't destroy QP\n");
return 1;
}
if (ibv_destroy_qp(ctx->mqp)) {
fprintf(stderr, "Couldn't destroy MQP\n");
return 1;
}
if (ibv_destroy_cq(ctx->cq)) {
fprintf(stderr, "Couldn't destroy CQ\n");
return 1;
}
if (ibv_destroy_cq(ctx->mcq)) {
fprintf(stderr, "Couldn't destroy MCQ\n");
return 1;
}
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "Couldn't deregister MR\n");
return 1;
}
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "Couldn't deallocate PD\n");
return 1;
}
if (ctx->channel) {
if (ibv_destroy_comp_channel(ctx->channel)) {
fprintf(stderr, "Couldn't destroy completion channel\n");
return 1;
}
}
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "Couldn't release context\n");
return 1;
}
free(ctx->buf);
free(ctx->net_buf);
free(ctx);
return 0;
}
static int pp_post_recv(struct pingpong_context *ctx, int n)
{
int rc;
struct ibv_sge list = {
.addr = (uintptr_t) ctx->net_buf,
.length = ctx->size,
.lkey = ctx->mr->lkey
};
struct ibv_recv_wr wr = {
.wr_id = PP_RECV_WRID,
.sg_list = &list,
.num_sge = 1,
};
struct ibv_recv_wr *bad_wr;
int i;
for (i = 0; i < n; ++i) {
rc = ibv_post_recv(ctx->qp, &wr, &bad_wr);
if (rc)
return rc;
}
return i;
}
static int pp_post_send(struct pingpong_context *ctx)
{
int ret;
struct ibv_sge list = {
.addr = (uintptr_t) ctx->net_buf,
.length = ctx->size,
.lkey = ctx->mr->lkey
};
struct ibv_exp_send_wr wr = {
.wr_id = PP_SEND_WRID,
.sg_list = &list,
.num_sge = 1,
.exp_opcode = IBV_EXP_WR_SEND,
.exp_send_flags = IBV_EXP_SEND_SIGNALED,
};
struct ibv_exp_send_wr *bad_wr;
/* If this is a calc operation - set the required params in the wr */
if (ctx->calc_op.opcode != IBV_EXP_CALC_OP_NUMBER) {
wr.exp_opcode = IBV_EXP_WR_SEND;
wr.exp_send_flags |= IBV_EXP_SEND_WITH_CALC;
wr.sg_list = ctx->calc_op.gather_list;
wr.num_sge = ctx->calc_op.gather_list_size;
wr.op.calc.calc_op = ctx->calc_op.opcode;
wr.op.calc.data_type = ctx->calc_op.data_type;
wr.op.calc.data_size = ctx->calc_op.data_size;
}
ret = ibv_exp_post_send(ctx->qp, &wr, &bad_wr);
return ret;
}
int pp_post_ext_wqe(struct pingpong_context *ctx, enum ibv_exp_wr_opcode op)
{
int ret;
struct ibv_exp_send_wr wr = {
.wr_id = PP_CQE_WAIT,
.sg_list = NULL,
.num_sge = 0,
.exp_opcode = op,
.exp_send_flags = IBV_EXP_SEND_SIGNALED,
};
struct ibv_exp_send_wr *bad_wr;
switch (op) {
case IBV_EXP_WR_RECV_ENABLE:
case IBV_EXP_WR_SEND_ENABLE:
wr.task.wqe_enable.qp = ctx->qp;
wr.task.wqe_enable.wqe_count = 0;
wr.exp_send_flags |= IBV_EXP_SEND_WAIT_EN_LAST;
break;
case IBV_EXP_WR_CQE_WAIT:
wr.task.cqe_wait.cq = ctx->cq;
wr.task.cqe_wait.cq_count = 1;
wr.exp_send_flags |= IBV_EXP_SEND_WAIT_EN_LAST;
break;
default:
fprintf(stderr, "-E- unsupported m_wqe opcode %d\n", op);
return -1;
}
ret = ibv_exp_post_send(ctx->mqp, &wr, &bad_wr);
return ret;
}
int pp_poll_mcq(struct ibv_cq *cq, int num_cqe)
{
int ne;
int i;
struct ibv_wc wc[2];
if (num_cqe > 2) {
fprintf(stderr, "-E- max num cqe exceeded\n");
return -1;
}
do {
ne = ibv_poll_cq(cq, num_cqe, wc);
if (ne < 0) {
fprintf(stderr, "poll CQ failed %d\n", ne);
return 1;
}
} while (ne < 1);
for (i = 0; i < ne; ++i) {
if (wc[i].status != IBV_WC_SUCCESS) {
fprintf(stderr, "Failed %s status %s (%d)\n",
wr_id_str[(int)wc[i].wr_id],
ibv_wc_status_str(wc[i].status),
wc[i].status);
return 1;
}
if ((int) wc[i].wr_id != PP_CQE_WAIT) {
fprintf(stderr, "invalid wr_id %" PRIx64 "\n", wc[i].wr_id);
return -1;
}
}
return 0;
}
static int pp_calc_verify(struct pingpong_context *ctx,
enum pp_wr_data_type calc_data_type,
enum pp_wr_calc_op calc_opcode)
{
uint64_t *op1 = &(ctx->last_result);
uint64_t *op2 = (uint64_t *)ctx->buf + 2;
uint64_t *res = (uint64_t *)ctx->buf;
return !EXEC_VERIFY(calc_data_type, calc_opcode, 1, op1, op2, res);
}
static int pp_update_last_result(struct pingpong_context *ctx,
enum pp_wr_data_type calc_data_type,
enum pp_wr_calc_op calc_opcode)
{
/* EXEC_VERIFY derefence result parameter */
uint64_t *dummy;
uint64_t *op1 = (uint64_t *)ctx->buf;
uint64_t *op2 = (uint64_t *)ctx->buf + 2;
uint64_t res = (uint64_t)EXEC_VERIFY(calc_data_type, calc_opcode, 0, op1, op2, dummy);
ctx->last_result = res;
return 0;
}
static void usage(const char *argv0)
{
printf("Usage:\n");
printf(" %s start a server and wait for connection\n", argv0);
printf(" %s <host> connect to server at <host>\n", argv0);
printf("\n");
printf("Options:\n");
printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
printf(" -s, --size=<size> size of message to exchange (default 4096 minimum 16)\n");
printf(" -m, --mtu=<size> path MTU (default 1024)\n");
printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
printf(" -l, --sl=<sl> service level value\n");
printf(" -e, --events sleep on CQ events (default poll)\n");
printf(" -c, --calc=<operation> calc operation\n");
printf(" -t, --op_type=<type> calc operands type\n");
printf(" -o, --operands=<o1,o2,...> comma separated list of operands\n");
printf(" -w, --wait_cq=cqn wait for entries on cq\n");
printf(" -v, --verbose print verbose information\n");
printf(" -V, --verify verify calc operations\n");
}
static int fio_rdmaio_setup_qp(struct thread_data *td)
{
struct rdmaio_data *rd = td->io_ops->data;
struct ibv_qp_init_attr init_attr;
int qp_depth = td->o.iodepth * 2; /* 2 times of io depth */
if (rd->is_client == 0)
rd->pd = ibv_alloc_pd(rd->child_cm_id->verbs);
else
rd->pd = ibv_alloc_pd(rd->cm_id->verbs);
if (rd->pd == NULL) {
log_err("fio: ibv_alloc_pd fail\n");
return 1;
}
if (rd->is_client == 0)
rd->channel = ibv_create_comp_channel(rd->child_cm_id->verbs);
else
rd->channel = ibv_create_comp_channel(rd->cm_id->verbs);
if (rd->channel == NULL) {
log_err("fio: ibv_create_comp_channel fail\n");
goto err1;
}
if (qp_depth < 16)
qp_depth = 16;
if (rd->is_client == 0)
rd->cq = ibv_create_cq(rd->child_cm_id->verbs,
qp_depth, rd, rd->channel, 0);
else
rd->cq = ibv_create_cq(rd->cm_id->verbs,
qp_depth, rd, rd->channel, 0);
if (rd->cq == NULL) {
log_err("fio: ibv_create_cq failed\n");
goto err2;
}
if (ibv_req_notify_cq(rd->cq, 0) != 0) {
log_err("fio: ibv_create_cq failed\n");
goto err3;
}
/* create queue pair */
memset(&init_attr, 0, sizeof(init_attr));
init_attr.cap.max_send_wr = qp_depth;
init_attr.cap.max_recv_wr = qp_depth;
init_attr.cap.max_recv_sge = 1;
init_attr.cap.max_send_sge = 1;
init_attr.qp_type = IBV_QPT_RC;
init_attr.send_cq = rd->cq;
init_attr.recv_cq = rd->cq;
if (rd->is_client == 0) {
if (rdma_create_qp(rd->child_cm_id, rd->pd, &init_attr) != 0) {
log_err("fio: rdma_create_qp failed\n");
goto err3;
}
rd->qp = rd->child_cm_id->qp;
} else {
if (rdma_create_qp(rd->cm_id, rd->pd, &init_attr) != 0) {
log_err("fio: rdma_create_qp failed\n");
goto err3;
}
rd->qp = rd->cm_id->qp;
}
return 0;
err3:
ibv_destroy_cq(rd->cq);
err2:
ibv_destroy_comp_channel(rd->channel);
err1:
ibv_dealloc_pd(rd->pd);
return 1;
}
int
resource_create(resource_t *res, int ib_port, int myrank)
{
struct ibv_device **dev_list = NULL;
struct ibv_qp_init_attr qp_init_attr;
struct ibv_device *ib_dev = NULL;
char *dev_name = NULL;
size_t size;
int i;
int mr_flags = 0;
int cq_size = 0;
int dev_numm;
int rc = 0;
/* Init structure */
memset(res, 0, sizeof(resource_t));
/* Get the device list */
dev_list = ibv_get_device_list(&dev_numm);
if(!dev_list) {
fprintf(stderr, "[%d] failed to get IB devices list\n", myrank);
return 1;
}
// if no device
if(!dev_numm) {
fprintf(stderr, "[%d] No IB device is found\n", myrank);
rc = 1;
goto err_exit;
}
DEBUG { printf("[%d] found %d IB device(s)\n", myrank, dev_numm); }
/* Open the requested device */
for(i = 0; i < dev_numm; i ++){
dev_name = strdup(ibv_get_device_name(dev_list[i]));
DEBUG { printf("[%d] IB device name: %s\n", myrank, dev_name); }
ib_dev = dev_list[i];
break;
}
if (!ib_dev){
fprintf(stderr, "[%d] IB device %s wasn't found\n", myrank, dev_name);
rc = 1;
goto err_exit;
}
res->ib_ctx = ibv_open_device(ib_dev);
DEBUG { printf("[%d] IB context = %lx\n", myrank, (uintptr_t)res->ib_ctx); }
if(!res->ib_ctx){
fprintf(stderr, "[%d] failed to open device %s\n", myrank, dev_name);
rc = 1;
goto err_exit;
}
// free device list
ibv_free_device_list(dev_list);
dev_list = NULL;
ib_dev = NULL;
// query prot properties
if(ibv_query_port(res->ib_ctx, ib_port, &res->port_attr)){
fprintf(stderr, "[%d] ibv_query_port on port %u failed\n", myrank, ib_port);
rc = 1;
goto err_exit;
}
/* Create a PD */
res->pd = ibv_alloc_pd(res->ib_ctx);
if (!res->pd){
fprintf(stderr, "[%d] ibv_alloc_pd failed\n", myrank);
rc = 1;
goto err_exit;
}
/* Create send/recv CQ
* inputs:
* device handle
* CQ capacity
* Output:
* CQ handle
*/
res->scq = ibv_create_cq(res->ib_ctx, MAX_CQ_CAPACITY, NULL, NULL, 0);
res->rcq = ibv_create_cq(res->ib_ctx, MAX_CQ_CAPACITY, NULL, NULL, 0);
if (!res->scq){
fprintf(stderr, "[%d] failed to create SCQ with %u entries\n", myrank, cq_size);
rc = 1;
goto err_exit;
}
if (!res->rcq){
fprintf(stderr, "[%d] failed to create SCQ with %u entries\n", myrank, cq_size);
rc = 1;
goto err_exit;
}
/* Allocate fix buffer */
size = MAX_FIX_BUF_SIZE;
res->buf_size = size;
res->buf = (char *)malloc(size * sizeof(char));
if (!res->buf ){
fprintf(stderr, "[%d] failed to malloc %Zu bytes to memory buffer\n", myrank, size);
rc = 1;
goto err_exit;
}
memset(res->buf, 0 , size);
/* Memory Region
* inputs:
* device handle
* PD
* Virtual Addr(addr of MR)
* Access Ctrl: LocalWrite, RemoteRead, RemoteWrite, RemoteAtomicOp, MemWindowBinding
* outputs:
* MR handle
* L_Key
* R_Key
*/
res->mr_list = malloc(sizeof(struct ibv_mr*) * MAX_MR_NUM);
res->mr_size = 1;
mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ |
IBV_ACCESS_REMOTE_WRITE ;
res->mr_list[0] = ibv_reg_mr(res->pd, res->buf, size, mr_flags);
if (!res->mr_list[0]){
fprintf(stderr, "[%d] ibv_reg_mr failed with mr_flags=0x%x\n", myrank, mr_flags);
rc = 1;
goto err_exit;
}
DEBUG { printf("[%d] fixed MR was registered with addr=%p, lkey=0x%x, rkey=0x%x, flags=0x%x\n", myrank, res->buf, res->mr_list[0]->lkey, res->mr_list[0]->rkey, mr_flags); }
/* Create QP */
// inputs:
// PD
// CQs for SQ,RQ
// capacity of SQ,RQ
// Outputs:
// QP handle
memset(&qp_init_attr, 0, sizeof(qp_init_attr));
qp_init_attr.qp_type = IBV_QPT_RC;
qp_init_attr.sq_sig_all = 1;
qp_init_attr.send_cq = res->scq;
qp_init_attr.recv_cq = res->rcq;
// max SR/RR num in SQ/RQ
qp_init_attr.cap.max_send_wr = MAX_SQ_CAPACITY ;
qp_init_attr.cap.max_recv_wr = MAX_RQ_CAPACITY;
// max SGE num
qp_init_attr.cap.max_send_sge = MAX_SGE_CAPACITY;
qp_init_attr.cap.max_recv_sge = MAX_SGE_CAPACITY;
qp_init_attr.cap.max_inline_data = 256;
res->qp = ibv_create_qp(res->pd, &qp_init_attr);
if (!res->qp){
fprintf(stderr, "failed to create QP\n");
rc = 1;
goto err_exit;
}
DEBUG { printf("[%d] QP was created, QP number=0x%x\n", myrank, res->qp->qp_num); }
/* EXIT */
err_exit:
if(rc){
/* Error encountered, cleanup */
if(res->qp){
ibv_destroy_qp(res->qp);
res->qp = NULL;
}
if(res->mr_list && res->mr_size > 0){
int i;
for(i=0; i<res->mr_size; i++){
ibv_dereg_mr(res->mr_list[i]);
res->mr_list[i] = NULL;
}
free(res->mr_list);
}
if(res->buf){
free(res->buf);
res->buf = NULL;
}
if(res->scq){
ibv_destroy_cq(res->scq);
res->scq = NULL;
}
if(res->rcq){
ibv_destroy_cq(res->rcq);
res->rcq = NULL;
}
if(res->comp_ch){
ibv_destroy_comp_channel(res->comp_ch);
res->comp_ch = NULL;
}
if(res->pd){
ibv_dealloc_pd(res->pd);
res->pd = NULL;
}
if (res->ib_ctx) {
ibv_close_device(res->ib_ctx);
res->ib_ctx = NULL;
}
if (dev_list) {
ibv_free_device_list(dev_list);
dev_list = NULL;
}
}
return rc;
}
int rdma_backend_init(RdmaBackendDev *backend_dev, PCIDevice *pdev,
RdmaDeviceResources *rdma_dev_res,
const char *backend_device_name, uint8_t port_num,
struct ibv_device_attr *dev_attr, CharBackend *mad_chr_be)
{
int i;
int ret = 0;
int num_ibv_devices;
struct ibv_device **dev_list;
memset(backend_dev, 0, sizeof(*backend_dev));
backend_dev->dev = pdev;
backend_dev->port_num = port_num;
backend_dev->rdma_dev_res = rdma_dev_res;
rdma_backend_register_comp_handler(dummy_comp_handler);
dev_list = ibv_get_device_list(&num_ibv_devices);
if (!dev_list) {
rdma_error_report("Failed to get IB devices list");
return -EIO;
}
if (num_ibv_devices == 0) {
rdma_error_report("No IB devices were found");
ret = -ENXIO;
goto out_free_dev_list;
}
if (backend_device_name) {
for (i = 0; dev_list[i]; ++i) {
if (!strcmp(ibv_get_device_name(dev_list[i]),
backend_device_name)) {
break;
}
}
backend_dev->ib_dev = dev_list[i];
if (!backend_dev->ib_dev) {
rdma_error_report("Failed to find IB device %s",
backend_device_name);
ret = -EIO;
goto out_free_dev_list;
}
} else {
backend_dev->ib_dev = *dev_list;
}
rdma_info_report("uverb device %s", backend_dev->ib_dev->dev_name);
backend_dev->context = ibv_open_device(backend_dev->ib_dev);
if (!backend_dev->context) {
rdma_error_report("Failed to open IB device %s",
ibv_get_device_name(backend_dev->ib_dev));
ret = -EIO;
goto out;
}
backend_dev->channel = ibv_create_comp_channel(backend_dev->context);
if (!backend_dev->channel) {
rdma_error_report("Failed to create IB communication channel");
ret = -EIO;
goto out_close_device;
}
ret = init_device_caps(backend_dev, dev_attr);
if (ret) {
rdma_error_report("Failed to initialize device capabilities");
ret = -EIO;
goto out_destroy_comm_channel;
}
ret = mad_init(backend_dev, mad_chr_be);
if (ret) {
rdma_error_report("Failed to initialize mad");
ret = -EIO;
goto out_destroy_comm_channel;
}
backend_dev->comp_thread.run = false;
backend_dev->comp_thread.is_running = false;
ah_cache_init();
goto out_free_dev_list;
out_destroy_comm_channel:
ibv_destroy_comp_channel(backend_dev->channel);
out_close_device:
ibv_close_device(backend_dev->context);
out_free_dev_list:
ibv_free_device_list(dev_list);
out:
return ret;
}
/**
* @param rx_headroom Headroom requested by the user.
* @param rx_priv_len Length of transport private data to reserve (0 if unused)
* @param rx_hdr_len Length of transport network header.
* @param mss Maximal segment size (transport limit).
*/
UCS_CLASS_INIT_FUNC(uct_ib_iface_t, uct_ib_iface_ops_t *ops, uct_md_h md,
uct_worker_h worker, const uct_iface_params_t *params,
unsigned rx_priv_len, unsigned rx_hdr_len, unsigned tx_cq_len,
size_t mss, const uct_ib_iface_config_t *config)
{
uct_ib_device_t *dev = &ucs_derived_of(md, uct_ib_md_t)->dev;
ucs_status_t status;
uint8_t port_num;
UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &ops->super, md, worker,
&config->super UCS_STATS_ARG(dev->stats));
status = uct_ib_device_find_port(dev, params->dev_name, &port_num);
if (status != UCS_OK) {
goto err;
}
self->ops = ops;
self->config.rx_payload_offset = sizeof(uct_ib_iface_recv_desc_t) +
ucs_max(sizeof(uct_am_recv_desc_t) +
params->rx_headroom,
rx_priv_len + rx_hdr_len);
self->config.rx_hdr_offset = self->config.rx_payload_offset - rx_hdr_len;
self->config.rx_headroom_offset= self->config.rx_payload_offset -
params->rx_headroom;
self->config.seg_size = ucs_min(mss, config->super.max_bcopy);
self->config.tx_max_poll = config->tx.max_poll;
self->config.rx_max_poll = config->rx.max_poll;
self->config.rx_max_batch = ucs_min(config->rx.max_batch,
config->rx.queue_len / 4);
self->config.port_num = port_num;
self->config.sl = config->sl;
self->config.gid_index = config->gid_index;
status = uct_ib_iface_init_pkey(self, config);
if (status != UCS_OK) {
goto err;
}
status = uct_ib_device_query_gid(dev, self->config.port_num,
self->config.gid_index, &self->gid);
if (status != UCS_OK) {
goto err;
}
status = uct_ib_iface_init_lmc(self, config);
if (status != UCS_OK) {
goto err;
}
self->comp_channel = ibv_create_comp_channel(dev->ibv_context);
if (self->comp_channel == NULL) {
ucs_error("ibv_create_comp_channel() failed: %m");
status = UCS_ERR_IO_ERROR;
goto err_free_path_bits;
}
status = ucs_sys_fcntl_modfl(self->comp_channel->fd, O_NONBLOCK, 0);
if (status != UCS_OK) {
goto err_destroy_comp_channel;
}
status = uct_ib_iface_create_cq(self, tx_cq_len, 0, &self->send_cq);
if (status != UCS_OK) {
goto err_destroy_comp_channel;
}
status = uct_ib_iface_create_cq(self, config->rx.queue_len, config->rx.inl,
&self->recv_cq);
if (status != UCS_OK) {
goto err_destroy_send_cq;
}
/* Address scope and size */
if (config->addr_type == UCT_IB_IFACE_ADDRESS_TYPE_AUTO) {
if (IBV_PORT_IS_LINK_LAYER_ETHERNET(uct_ib_iface_port_attr(self))) {
self->addr_type = UCT_IB_ADDRESS_TYPE_ETH;
} else {
self->addr_type = uct_ib_address_scope(self->gid.global.subnet_prefix);
}
} else {
ucs_assert(config->addr_type < UCT_IB_ADDRESS_TYPE_LAST);
self->addr_type = config->addr_type;
}
self->addr_size = uct_ib_address_size(self->addr_type);
ucs_debug("created uct_ib_iface_t headroom_ofs %d payload_ofs %d hdr_ofs %d data_sz %d",
self->config.rx_headroom_offset, self->config.rx_payload_offset,
self->config.rx_hdr_offset, self->config.seg_size);
return UCS_OK;
err_destroy_send_cq:
ibv_destroy_cq(self->send_cq);
err_destroy_comp_channel:
ibv_destroy_comp_channel(self->comp_channel);
err_free_path_bits:
ucs_free(self->path_bits);
err:
return status;
}
