static void build_verbs(IbvConnection *conn, struct ibv_context *verbs)
{
conn->ibvctx = verbs;
TEST_Z(conn->pd = ibv_alloc_pd(conn->ibvctx));
TEST_Z(conn->comp_channel = ibv_create_comp_channel(conn->ibvctx));
TEST_Z(conn->cq = ibv_create_cq(conn->ibvctx, 10, NULL, conn->comp_channel, 0)); /* cqe=10 is arbitrary */
TEST_NZ(ibv_req_notify_cq(conn->cq, 0));
TEST_NZ(pthread_create(&conn->cq_poller_thread, NULL, poll_cq, conn));
}
ibv_pd *connection_handler::get_pd(ibv_context *context, boost::system::error_code & ec)
{
hpx::lcos::local::spinlock::scoped_lock l(pd_map_mtx_);
typedef pd_map_type::iterator iterator;
iterator it = pd_map_.find(context);
if(it == pd_map_.end())
{
ibv_pd *pd = ibv_alloc_pd(context);
if(pd == 0)
{
int verrno = errno;
boost::system::error_code err(verrno, boost::system::system_category());
HPX_IBVERBS_THROWS_IF(
ec
, err
);
return 0;
}
memory_pool_.register_chunk(pd);
pd_map_.insert(std::make_pair(context, pd));
mr_map_.insert(std::make_pair(pd, mr_cache_type()));
return pd;
}
return it->second;
}
static PdPtr make_pd(CtxPtr ctx) {
auto ptr = ibv_alloc_pd(ctx.get());
if(!ptr) {
throw std::runtime_error("cannot allocate pd");
}
return PdPtr(ptr, ibv_dealloc_pd);
}
static int init_node(struct cmatest_node *node)
{
struct ibv_qp_init_attr init_qp_attr;
int cqe, ret;
int i;
struct ibv_cq **cqs[] = {&node->cq[SEND_CQ_INDEX],
&node->cq[RECV_CQ_INDEX]};
node->pd = ibv_alloc_pd(node->cma_id->verbs);
if (!node->pd) {
ret = -ENOMEM;
printf("cmatose: unable to allocate PD\n");
goto out;
}
cqe = message_count ? message_count : 1;
for (i = 0; i < sizeof(cqs)/sizeof(cqs[0]); i++) {
if (set_ts) {
struct ibv_exp_cq_init_attr cq_init_attr;
memset(&cq_init_attr, 0, sizeof(cq_init_attr));
cq_init_attr.flags = IBV_EXP_CQ_TIMESTAMP;
cq_init_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_FLAGS;
*cqs[i] = (struct ibv_cq *)ibv_exp_create_cq(
node->cma_id->verbs, cqe, node,
NULL, 0, &cq_init_attr);
} else {
*cqs[i] = ibv_create_cq(node->cma_id->verbs, cqe, node,
0, 0);
}
}
if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) {
ret = -ENOMEM;
printf("cmatose: unable to create CQ\n");
goto out;
}
memset(&init_qp_attr, 0, sizeof init_qp_attr);
init_qp_attr.cap.max_send_wr = cqe;
init_qp_attr.cap.max_recv_wr = cqe;
init_qp_attr.cap.max_send_sge = 1;
init_qp_attr.cap.max_recv_sge = 1;
init_qp_attr.qp_context = node;
init_qp_attr.sq_sig_all = 1;
init_qp_attr.qp_type = IBV_QPT_RC;
init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX];
init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX];
ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
if (ret) {
perror("cmatose: unable to create QP");
goto out;
}
ret = create_message(node);
if (ret) {
printf("cmatose: failed to create messages: %d\n", ret);
goto out;
}
out:
return ret;
}
ProtectionDomain::ProtectionDomain(ibv_context* context)
: mDomain(ibv_alloc_pd(context)) {
if (mDomain == nullptr) {
throw std::system_error(errno, std::generic_category());
}
LOG_TRACE("Allocated protection domain");
}
int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd)
{
pd->ibpd = ibv_alloc_pd(backend_dev->context);
if (!pd->ibpd) {
rdma_error_report("ibv_alloc_pd fail, errno=%d", errno);
return -EIO;
}
return 0;
}
int opal_common_verbs_qp_test(struct ibv_context *device_context, int flags)
{
int rc = OPAL_SUCCESS;
struct ibv_pd *pd = NULL;
struct ibv_cq *cq = NULL;
/* Bozo check */
if (NULL == device_context ||
(0 == (flags & (OPAL_COMMON_VERBS_FLAGS_RC | OPAL_COMMON_VERBS_FLAGS_UD)))) {
return OPAL_ERR_BAD_PARAM;
}
/* Try to make both the PD and CQ */
pd = ibv_alloc_pd(device_context);
if (NULL == pd) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
cq = ibv_create_cq(device_context, 2, NULL, NULL, 0);
if (NULL == cq) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
goto out;
}
/* Now try to make the QP(s) of the desired type(s) */
if (flags & OPAL_COMMON_VERBS_FLAGS_RC &&
!make_qp(pd, cq, IBV_QPT_RC)) {
rc = OPAL_ERR_NOT_SUPPORTED;
goto out;
}
if (flags & OPAL_COMMON_VERBS_FLAGS_NOT_RC &&
make_qp(pd, cq, IBV_QPT_RC)) {
rc = OPAL_ERR_TYPE_MISMATCH;
goto out;
}
if (flags & OPAL_COMMON_VERBS_FLAGS_UD &&
!make_qp(pd, cq, IBV_QPT_UD)) {
rc = OPAL_ERR_NOT_SUPPORTED;
goto out;
}
out:
/* Free the PD and/or CQ */
if (NULL != pd) {
ibv_dealloc_pd(pd);
}
if (NULL != cq) {
ibv_destroy_cq(cq);
}
return rc;
}
static int
fi_ibv_domain(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context)
{
struct fi_ibv_domain *_domain;
struct fi_info *fi;
int ret;
fi = fi_ibv_get_verbs_info(info->domain_attr->name);
if (!fi)
return -FI_EINVAL;
ret = fi_ibv_check_domain_attr(info->domain_attr, fi);
if (ret)
return ret;
_domain = calloc(1, sizeof *_domain);
if (!_domain)
return -FI_ENOMEM;
_domain->info = fi_dupinfo(info);
if (!_domain->info)
goto err1;
_domain->rdm = FI_IBV_EP_TYPE_IS_RDM(info);
ret = fi_ibv_open_device_by_name(_domain, info->domain_attr->name);
if (ret)
goto err2;
_domain->pd = ibv_alloc_pd(_domain->verbs);
if (!_domain->pd) {
ret = -errno;
goto err2;
}
_domain->domain_fid.fid.fclass = FI_CLASS_DOMAIN;
_domain->domain_fid.fid.context = context;
_domain->domain_fid.fid.ops = &fi_ibv_fid_ops;
_domain->domain_fid.ops = _domain->rdm ? &fi_ibv_rdm_domain_ops :
&fi_ibv_domain_ops;
_domain->domain_fid.mr = &fi_ibv_domain_mr_ops;
_domain->fab = container_of(fabric, struct fi_ibv_fabric, fabric_fid);
*domain = &_domain->domain_fid;
return 0;
err2:
fi_freeinfo(_domain->info);
err1:
free(_domain);
return ret;
}
static int rping_setup_qp(struct rping_cb *cb, struct rdma_cm_id *cm_id)
{
int ret;
cb->pd = ibv_alloc_pd(cm_id->verbs);
if (!cb->pd) {
fprintf(stderr, "ibv_alloc_pd failed\n");
return errno;
}
DEBUG_LOG("created pd %p\n", cb->pd);
cb->channel = ibv_create_comp_channel(cm_id->verbs);
if (!cb->channel) {
fprintf(stderr, "ibv_create_comp_channel failed\n");
ret = errno;
goto err1;
}
DEBUG_LOG("created channel %p\n", cb->channel);
cb->cq = ibv_create_cq(cm_id->verbs, RPING_SQ_DEPTH * 2, cb,
cb->channel, 0);
if (!cb->cq) {
fprintf(stderr, "ibv_create_cq failed\n");
ret = errno;
goto err2;
}
DEBUG_LOG("created cq %p\n", cb->cq);
ret = ibv_req_notify_cq(cb->cq, 0);
if (ret) {
fprintf(stderr, "ibv_create_cq failed\n");
ret = errno;
goto err3;
}
ret = rping_create_qp(cb);
if (ret) {
perror("rdma_create_qp");
goto err3;
}
DEBUG_LOG("created qp %p\n", cb->qp);
return 0;
err3:
ibv_destroy_cq(cb->cq);
err2:
ibv_destroy_comp_channel(cb->channel);
err1:
ibv_dealloc_pd(cb->pd);
return ret;
}
static
struct ibv_pd *psofed_open_pd(struct ibv_context *ctx)
{
/* allocate a protection domain to be associated with QP */
struct ibv_pd *pd;
pd = ibv_alloc_pd(ctx);
if (!pd) {
psofed_err_errno("ibv_alloc_pd() failed", errno);
}
return pd;
}
static inline int fi_ibv_get_qp_cap(struct ibv_context *ctx,
struct fi_info *info)
{
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_qp *qp;
struct ibv_qp_init_attr init_attr;
int ret = 0;
pd = ibv_alloc_pd(ctx);
if (!pd) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_alloc_pd", errno);
return -errno;
}
cq = ibv_create_cq(ctx, 1, NULL, NULL, 0);
if (!cq) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_cq", errno);
ret = -errno;
goto err1;
}
memset(&init_attr, 0, sizeof init_attr);
init_attr.send_cq = cq;
init_attr.recv_cq = cq;
init_attr.cap.max_send_wr = verbs_default_tx_size;
init_attr.cap.max_recv_wr = verbs_default_rx_size;
init_attr.cap.max_send_sge = verbs_default_tx_iov_limit;
init_attr.cap.max_recv_sge = verbs_default_rx_iov_limit;
init_attr.cap.max_inline_data = verbs_default_inline_size;
init_attr.qp_type = IBV_QPT_RC;
qp = ibv_create_qp(pd, &init_attr);
if (!qp) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_qp", errno);
ret = -errno;
goto err2;
}
info->tx_attr->inject_size = init_attr.cap.max_inline_data;
ibv_destroy_qp(qp);
err2:
ibv_destroy_cq(cq);
err1:
ibv_dealloc_pd(pd);
return ret;
}
static ucs_status_t uct_ib_mlx5_check_dc(uct_ib_device_t *dev)
{
ucs_status_t status = UCS_OK;
struct ibv_context *ctx = dev->ibv_context;
struct ibv_qp_init_attr_ex qp_attr = {};
struct mlx5dv_qp_init_attr dv_attr = {};
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_qp *qp;
pd = ibv_alloc_pd(ctx);
if (pd == NULL) {
ucs_error("ibv_alloc_pd() failed: %m");
return UCS_ERR_IO_ERROR;
}
cq = ibv_create_cq(ctx, 1, NULL, NULL, 0);
if (cq == NULL) {
ucs_error("ibv_create_cq() failed: %m");
status = UCS_ERR_IO_ERROR;
goto err_cq;
}
qp_attr.send_cq = cq;
qp_attr.recv_cq = cq;
qp_attr.cap.max_send_wr = 1;
qp_attr.cap.max_send_sge = 1;
qp_attr.qp_type = IBV_QPT_DRIVER;
qp_attr.comp_mask = IBV_QP_INIT_ATTR_PD;
qp_attr.pd = pd;
dv_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_DC;
dv_attr.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI;
/* create DCI qp successful means DC is supported */
qp = mlx5dv_create_qp(ctx, &qp_attr, &dv_attr);
if (qp) {
ibv_destroy_qp(qp);
dev->flags |= UCT_IB_DEVICE_FLAG_DC;
}
ibv_destroy_cq(cq);
err_cq:
ibv_dealloc_pd(pd);
return status;
}
/// Initialize the InfiniBand verbs context.
void init_context(struct ibv_context* context)
{
context_ = context;
L_(debug) << "create verbs objects";
pd_ = ibv_alloc_pd(context);
if (!pd_)
throw InfinibandException("ibv_alloc_pd failed");
cq_ = ibv_create_cq(context, num_cqe_, nullptr, nullptr, 0);
if (!cq_)
throw InfinibandException("ibv_create_cq failed");
if (ibv_req_notify_cq(cq_, 0))
throw InfinibandException("ibv_req_notify_cq failed");
}
static int init_node(struct cmatest_node *node)
{
struct ibv_qp_init_attr init_qp_attr;
int cqe, ret;
node->pd = ibv_alloc_pd(node->cma_id->verbs);
if (!node->pd) {
ret = -ENOMEM;
printf("cmatose: unable to allocate PD\n");
goto out;
}
cqe = message_count ? message_count : 1;
node->cq[SEND_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0);
node->cq[RECV_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0);
if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) {
ret = -ENOMEM;
printf("cmatose: unable to create CQ\n");
goto out;
}
memset(&init_qp_attr, 0, sizeof init_qp_attr);
init_qp_attr.cap.max_send_wr = cqe;
init_qp_attr.cap.max_recv_wr = cqe;
init_qp_attr.cap.max_send_sge = 1;
init_qp_attr.cap.max_recv_sge = 1;
init_qp_attr.qp_context = node;
init_qp_attr.sq_sig_all = 1;
init_qp_attr.qp_type = IBV_QPT_RC;
init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX];
init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX];
ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
if (ret) {
perror("cmatose: unable to create QP");
goto out;
}
ret = create_message(node);
if (ret) {
printf("cmatose: failed to create messages: %d\n", ret);
goto out;
}
out:
return ret;
}
void build_context(struct ibv_context *verbs)
{
if (s_ctx) {
if (s_ctx->ctx != verbs) {
die("cannot handle events in more than one context.");
}
return;
}
s_ctx = (rdma_ctx_t *)malloc(sizeof(rdma_ctx_t));
s_ctx->ctx = verbs;
TEST_Z(s_ctx->pd = ibv_alloc_pd(s_ctx->ctx));
TEST_Z(s_ctx->comp_channel = ibv_create_comp_channel(s_ctx->ctx));
TEST_Z(s_ctx->cq = ibv_create_cq(s_ctx->ctx, 10, NULL, s_ctx->comp_channel, 0)); /* cqe=10 is arbitrary */
TEST_NZ(ibv_req_notify_cq(s_ctx->cq, 0));
}
static int init_node(struct cmatest_node *node)
{
struct ibv_qp_init_attr init_qp_attr;
int cqe, ret;
node->pd = ibv_alloc_pd(node->cma_id->verbs);
if (!node->pd) {
ret = -ENOMEM;
printf("rxe_send_mc: unable to allocate PD\n");
goto out;
}
cqe = message_buffer ? message_buffer * 2 : 2;
node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);
if (!node->cq) {
ret = -ENOMEM;
printf("rxe_send_mc: unable to create CQ\n");
goto out;
}
memset(&init_qp_attr, 0, sizeof init_qp_attr);
init_qp_attr.cap.max_send_wr = message_buffer ? message_buffer : 1;
init_qp_attr.cap.max_recv_wr = message_buffer ? message_buffer : 1;
init_qp_attr.cap.max_send_sge = 1;
init_qp_attr.cap.max_recv_sge = 1;
init_qp_attr.qp_context = node;
init_qp_attr.sq_sig_all = 1; //singal all
init_qp_attr.qp_type = IBV_QPT_UD;
init_qp_attr.send_cq = node->cq;
init_qp_attr.recv_cq = node->cq;
ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
if (ret) {
perror("rxe_send_mc: unable to create QP");
goto out;
}
ret = create_message(node);
if (ret) {
printf("rxe_send_mc: failed to create messages: %d\n", ret);
goto out;
}
out:
return ret;
}
static void build_context(struct ibv_context *verbs)
{
if (s_ctx) {
if (s_ctx->ctx != verbs)
die("cannot handle events in more than one context.");
return;
}
s_ctx = (struct context *)malloc(sizeof(struct context));
s_ctx->ctx = verbs;
TEST_Z(s_ctx->pd = ibv_alloc_pd(s_ctx->ctx));
TEST_Z(s_ctx->comp_channel = ibv_create_comp_channel(s_ctx->ctx));
TEST_Z(s_ctx->cq = ibv_create_cq(s_ctx->ctx, 10, NULL, s_ctx->comp_channel, 0)); /* cqe=10 is arbitrary */
TEST_NZ(ibv_req_notify_cq(s_ctx->cq, 0));
// TEST_NZ(pthread_create(&s_ctx->cq_poller_thread, NULL, poll_cq, NULL));
}
struct ibv_pd_1_0 *__ibv_alloc_pd_1_0(struct ibv_context_1_0 *context)
{ fprintf(stderr, "%s:%s:%d \n", __func__, __FILE__, __LINE__);
struct ibv_pd *real_pd;
struct ibv_pd_1_0 *pd;
pd = malloc(sizeof *pd);
if (!pd)
return NULL;
real_pd = ibv_alloc_pd(context->real_context);
if (!real_pd) {
free(pd);
return NULL;
}
pd->context = context;
pd->real_pd = real_pd;
return pd;
}
struct ibv_pd_1_0 *__ibv_alloc_pd_1_0(struct ibv_context_1_0 *context)
{
struct ibv_pd *real_pd;
struct ibv_pd_1_0 *pd;
pd = malloc(sizeof *pd);
if (!pd)
return NULL;
real_pd = ibv_alloc_pd(context->real_context);
if (!real_pd) {
free(pd);
return NULL;
}
pd->context = context;
pd->real_pd = real_pd;
return pd;
}
void Connector::build_context(struct ibv_context* verb_)
{
if (s_ctx_ && s_ctx_->ctx_ != verb_) {
log_(ERROR, "cannot handle events in more than one context.")
exit(EXIT_FAILURE);
}
s_ctx_ = (struct context*)malloc(sizeof(struct context) );
s_ctx_->ctx_ = verb_;
TEST_Z(s_ctx_->pd_ = ibv_alloc_pd(s_ctx_->ctx_) );
TEST_Z(s_ctx_->comp_channel_ = ibv_create_comp_channel(s_ctx_->ctx_) );
TEST_Z(s_ctx_->cq_ = ibv_create_cq(s_ctx_->ctx_, MAX_QP__CQ_LENGTH, NULL, s_ctx_->comp_channel_, 0) );
TEST_NZ(ibv_req_notify_cq(s_ctx_->cq_, 0) )
// TODO
// TEST_NZ(pthread_create(pthread_v.back(), NULL, &Connector::bst_poll_cq, (void*)(this) ) )
pthread_v.push_back(new pthread_t() );
wrap_Connector* wrap_ = new wrap_Connector(this, s_ctx_);
TEST_NZ(pthread_create(pthread_v.back(), NULL, call_poll_cq_w_wrap, wrap_) )
}
int open_hca(void)
{
struct ibv_device **dev_list=NULL;
struct ibv_context *cxt = NULL;
int rc;
int num_hcas;
dev_list = ibv_get_device_list(&num_hcas);
// Assume that the first device has an ACTIVE port
// if it does not we do not handle this situation for now
hca.ib_dev = dev_list[0];
hca.context = ibv_open_device(hca.ib_dev);
if(!hca.context) {
fprintf(stderr,"Couldn't get context %s\n",
ibv_get_device_name(hca.ib_dev));
return 1;
}
hca.pd = ibv_alloc_pd(hca.context);
assert(hca.pd != NULL);
if(!hca.pd) {
fprintf(stderr,"Couldn't get pd %s\n",
ibv_get_device_name(hca.ib_dev));
return 1;
}
return 0;
}
/**
* DPDK callback to register a PCI device.
*
* This function creates an Ethernet device for each port of a given
* PCI device.
*
* @param[in] pci_drv
* PCI driver structure (mlx5_driver).
* @param[in] pci_dev
* PCI device information.
*
* @return
* 0 on success, negative errno value on failure.
*/
static int
mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
{
struct ibv_device **list;
struct ibv_device *ibv_dev;
int err = 0;
struct ibv_context *attr_ctx = NULL;
struct ibv_device_attr device_attr;
unsigned int vf;
int idx;
int i;
(void)pci_drv;
assert(pci_drv == &mlx5_driver.pci_drv);
/* Get mlx5_dev[] index. */
idx = mlx5_dev_idx(&pci_dev->addr);
if (idx == -1) {
ERROR("this driver cannot support any more adapters");
return -ENOMEM;
}
DEBUG("using driver device index %d", idx);
/* Save PCI address. */
mlx5_dev[idx].pci_addr = pci_dev->addr;
list = ibv_get_device_list(&i);
if (list == NULL) {
assert(errno);
if (errno == ENOSYS) {
WARN("cannot list devices, is ib_uverbs loaded?");
return 0;
}
return -errno;
}
assert(i >= 0);
/*
* For each listed device, check related sysfs entry against
* the provided PCI ID.
*/
while (i != 0) {
struct rte_pci_addr pci_addr;
--i;
DEBUG("checking device \"%s\"", list[i]->name);
if (mlx5_ibv_device_to_pci_addr(list[i], &pci_addr))
continue;
if ((pci_dev->addr.domain != pci_addr.domain) ||
(pci_dev->addr.bus != pci_addr.bus) ||
(pci_dev->addr.devid != pci_addr.devid) ||
(pci_dev->addr.function != pci_addr.function))
continue;
vf = ((pci_dev->id.device_id ==
PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) ||
(pci_dev->id.device_id ==
PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF));
INFO("PCI information matches, using device \"%s\" (VF: %s)",
list[i]->name, (vf ? "true" : "false"));
attr_ctx = ibv_open_device(list[i]);
err = errno;
break;
}
if (attr_ctx == NULL) {
ibv_free_device_list(list);
switch (err) {
case 0:
WARN("cannot access device, is mlx5_ib loaded?");
return 0;
case EINVAL:
WARN("cannot use device, are drivers up to date?");
return 0;
}
assert(err > 0);
return -err;
}
ibv_dev = list[i];
DEBUG("device opened");
if (ibv_query_device(attr_ctx, &device_attr))
goto error;
INFO("%u port(s) detected", device_attr.phys_port_cnt);
for (i = 0; i < device_attr.phys_port_cnt; i++) {
uint32_t port = i + 1; /* ports are indexed from one */
uint32_t test = (1 << i);
struct ibv_context *ctx = NULL;
struct ibv_port_attr port_attr;
struct ibv_pd *pd = NULL;
struct priv *priv = NULL;
struct rte_eth_dev *eth_dev;
#ifdef HAVE_EXP_QUERY_DEVICE
struct ibv_exp_device_attr exp_device_attr;
#endif /* HAVE_EXP_QUERY_DEVICE */
struct ether_addr mac;
#ifdef HAVE_EXP_QUERY_DEVICE
exp_device_attr.comp_mask =
IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS |
IBV_EXP_DEVICE_ATTR_RX_HASH;
#endif /* HAVE_EXP_QUERY_DEVICE */
DEBUG("using port %u (%08" PRIx32 ")", port, test);
ctx = ibv_open_device(ibv_dev);
if (ctx == NULL)
goto port_error;
/* Check port status. */
err = ibv_query_port(ctx, port, &port_attr);
if (err) {
ERROR("port query failed: %s", strerror(err));
goto port_error;
}
if (port_attr.state != IBV_PORT_ACTIVE)
DEBUG("port %d is not active: \"%s\" (%d)",
port, ibv_port_state_str(port_attr.state),
port_attr.state);
/* Allocate protection domain. */
pd = ibv_alloc_pd(ctx);
if (pd == NULL) {
ERROR("PD allocation failure");
err = ENOMEM;
goto port_error;
}
mlx5_dev[idx].ports |= test;
/* from rte_ethdev.c */
priv = rte_zmalloc("ethdev private structure",
sizeof(*priv),
RTE_CACHE_LINE_SIZE);
if (priv == NULL) {
ERROR("priv allocation failure");
err = ENOMEM;
goto port_error;
}
priv->ctx = ctx;
priv->device_attr = device_attr;
priv->port = port;
priv->pd = pd;
priv->mtu = ETHER_MTU;
#ifdef HAVE_EXP_QUERY_DEVICE
if (ibv_exp_query_device(ctx, &exp_device_attr)) {
ERROR("ibv_exp_query_device() failed");
goto port_error;
}
priv->hw_csum =
((exp_device_attr.exp_device_cap_flags &
IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) &&
(exp_device_attr.exp_device_cap_flags &
IBV_EXP_DEVICE_RX_CSUM_IP_PKT));
DEBUG("checksum offloading is %ssupported",
(priv->hw_csum ? "" : "not "));
priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags &
IBV_EXP_DEVICE_VXLAN_SUPPORT);
DEBUG("L2 tunnel checksum offloads are %ssupported",
(priv->hw_csum_l2tun ? "" : "not "));
priv->ind_table_max_size = exp_device_attr.rx_hash_caps.max_rwq_indirection_table_size;
DEBUG("maximum RX indirection table size is %u",
priv->ind_table_max_size);
#else /* HAVE_EXP_QUERY_DEVICE */
priv->ind_table_max_size = RSS_INDIRECTION_TABLE_SIZE;
#endif /* HAVE_EXP_QUERY_DEVICE */
priv->vf = vf;
/* Allocate and register default RSS hash keys. */
priv->rss_conf = rte_calloc(__func__, hash_rxq_init_n,
sizeof((*priv->rss_conf)[0]), 0);
if (priv->rss_conf == NULL) {
err = ENOMEM;
goto port_error;
}
err = rss_hash_rss_conf_new_key(priv,
rss_hash_default_key,
rss_hash_default_key_len,
ETH_RSS_PROTO_MASK);
if (err)
goto port_error;
/* Configure the first MAC address by default. */
if (priv_get_mac(priv, &mac.addr_bytes)) {
ERROR("cannot get MAC address, is mlx5_en loaded?"
" (errno: %s)", strerror(errno));
goto port_error;
}
INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
priv->port,
mac.addr_bytes[0], mac.addr_bytes[1],
mac.addr_bytes[2], mac.addr_bytes[3],
mac.addr_bytes[4], mac.addr_bytes[5]);
/* Register MAC and broadcast addresses. */
claim_zero(priv_mac_addr_add(priv, 0,
(const uint8_t (*)[ETHER_ADDR_LEN])
mac.addr_bytes));
claim_zero(priv_mac_addr_add(priv, (RTE_DIM(priv->mac) - 1),
&(const uint8_t [ETHER_ADDR_LEN])
{ "\xff\xff\xff\xff\xff\xff" }));
#ifndef NDEBUG
{
char ifname[IF_NAMESIZE];
if (priv_get_ifname(priv, &ifname) == 0)
DEBUG("port %u ifname is \"%s\"",
priv->port, ifname);
else
DEBUG("port %u ifname is unknown", priv->port);
}
#endif
/* Get actual MTU if possible. */
priv_get_mtu(priv, &priv->mtu);
DEBUG("port %u MTU is %u", priv->port, priv->mtu);
/* from rte_ethdev.c */
{
char name[RTE_ETH_NAME_MAX_LEN];
snprintf(name, sizeof(name), "%s port %u",
ibv_get_device_name(ibv_dev), port);
eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_PCI);
}
if (eth_dev == NULL) {
ERROR("can not allocate rte ethdev");
err = ENOMEM;
goto port_error;
}
eth_dev->data->dev_private = priv;
eth_dev->pci_dev = pci_dev;
eth_dev->driver = &mlx5_driver;
eth_dev->data->rx_mbuf_alloc_failed = 0;
eth_dev->data->mtu = ETHER_MTU;
priv->dev = eth_dev;
eth_dev->dev_ops = &mlx5_dev_ops;
eth_dev->data->mac_addrs = priv->mac;
TAILQ_INIT(ð_dev->link_intr_cbs);
/* Bring Ethernet device up. */
DEBUG("forcing Ethernet interface up");
priv_set_flags(priv, ~IFF_UP, IFF_UP);
continue;
port_error:
rte_free(priv->rss_conf);
rte_free(priv);
if (pd)
claim_zero(ibv_dealloc_pd(pd));
if (ctx)
claim_zero(ibv_close_device(ctx));
break;
}
/***************************************************************************//**
* Description
* Init rdma global resources
*
******************************************************************************/
static struct thread_context*
init_rdma_thread_resources() {
struct thread_context *ctx = calloc(1, sizeof(struct thread_context));
ctx->qp_hash = hashtable_create(1024);
int num_device;
if ( !(ctx->device_ctx_list = rdma_get_devices(&num_device)) ) {
perror("rdma_get_devices()");
return NULL;
}
ctx->device_ctx = *ctx->device_ctx_list;
if (verbose) {
printf("Get device: %d\n", num_device);
}
if ( !(ctx->pd = ibv_alloc_pd(ctx->device_ctx)) ) {
perror("ibv_alloc_pd()");
return NULL;
}
if ( !(ctx->comp_channel = ibv_create_comp_channel(ctx->device_ctx)) ) {
perror("ibv_create_comp_channel()");
return NULL;
}
struct ibv_srq_init_attr srq_init_attr;
srq_init_attr.srq_context = NULL;
srq_init_attr.attr.max_sge = 16;
srq_init_attr.attr.max_wr = srq_size;
srq_init_attr.attr.srq_limit = srq_size; /* RDMA TODO: what is srq_limit? */
if ( !(ctx->srq = ibv_create_srq(ctx->pd, &srq_init_attr)) ) {
perror("ibv_create_srq()");
return NULL;
}
if ( !(ctx->send_cq = ibv_create_cq(ctx->device_ctx,
cq_size, NULL, ctx->comp_channel, 0)) ) {
perror("ibv_create_cq()");
return NULL;
}
if (0 != ibv_req_notify_cq(ctx->send_cq, 0)) {
perror("ibv_reg_notify_cq()");
return NULL;
}
if ( !(ctx->recv_cq = ibv_create_cq(ctx->device_ctx,
cq_size, NULL, ctx->comp_channel, 0)) ) {
perror("ibv_create_cq()");
return NULL;
}
if (0 != ibv_req_notify_cq(ctx->recv_cq, 0)) {
perror("ibv_reg_notify_cq()");
return NULL;
}
ctx->rsize = BUFF_SIZE;
ctx->rbuf_list = calloc(buff_per_thread, sizeof(char *));
ctx->rmr_list = calloc(buff_per_thread, sizeof(struct ibv_mr*));
ctx->poll_wc = calloc(poll_wc_size, sizeof(struct ibv_wc));
int i = 0;
for (i = 0; i < buff_per_thread; ++i) {
ctx->rbuf_list[i] = malloc(ctx->rsize);
if (ctx->rbuf_list[i] == 0) {
break;
}
}
if (i != buff_per_thread) {
int j = 0;
for (j = 0; j < i; ++j) {
free(ctx->rbuf_list[j]);
}
free(ctx->rbuf_list);
ctx->rbuf_list = 0;
}
if (!ctx->rmr_list || !ctx->rbuf_list) {
fprintf(stderr, "out of ctxmory in init_rdma_thread_resources()\n");
return NULL;
}
struct ibv_recv_wr *bad = NULL;
struct ibv_sge sge;
struct ibv_recv_wr rwr;
for (i = 0; i < buff_per_thread; ++i) {
ctx->rmr_list[i] = ibv_reg_mr(ctx->pd, ctx->rbuf_list[i], ctx->rsize, IBV_ACCESS_LOCAL_WRITE);
sge.addr = (uintptr_t)ctx->rbuf_list[i];
sge.length = ctx->rsize;
sge.lkey = ctx->rmr_list[i]->lkey;
rwr.wr_id = (uintptr_t)ctx->rmr_list[i];
rwr.next = NULL;
rwr.sg_list = &sge;
rwr.num_sge = 1;
if (0 != ibv_post_srq_recv(ctx->srq, &rwr, &bad)) {
perror("ibv_post_srq_recv()");
return NULL;
}
}
return ctx;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
verbs_runtime_query(mca_base_module_t **module,
int *priority,
const char *hint)
{
int rc = OSHMEM_SUCCESS;
openib_device_t my_device;
openib_device_t *device = &my_device;
int num_devs = 0;
int i = 0;
*priority = 0;
*module = NULL;
memset(device, 0, sizeof(*device));
#ifdef HAVE_IBV_GET_DEVICE_LIST
device->ib_devs = ibv_get_device_list(&num_devs);
#else
#error unsupported ibv_get_device_list in infiniband/verbs.h
#endif
if (num_devs == 0 || !device->ib_devs) {
return OSHMEM_ERR_NOT_SUPPORTED;
}
/* Open device */
if (NULL != mca_sshmem_verbs_component.hca_name) {
for (i = 0; i < num_devs; i++) {
if (0 == strcmp(mca_sshmem_verbs_component.hca_name, ibv_get_device_name(device->ib_devs[i]))) {
device->ib_dev = device->ib_devs[i];
break;
}
}
} else {
device->ib_dev = device->ib_devs[0];
}
if (NULL == device->ib_dev) {
rc = OSHMEM_ERR_NOT_FOUND;
goto out;
}
if (NULL == (device->ib_dev_context = ibv_open_device(device->ib_dev))) {
rc = OSHMEM_ERR_RESOURCE_BUSY;
goto out;
}
/* Obtain device attributes */
if (ibv_query_device(device->ib_dev_context, &device->ib_dev_attr)) {
rc = OSHMEM_ERR_RESOURCE_BUSY;
goto out;
}
/* Allocate the protection domain for the device */
device->ib_pd = ibv_alloc_pd(device->ib_dev_context);
if (NULL == device->ib_pd) {
rc = OSHMEM_ERR_RESOURCE_BUSY;
goto out;
}
/* Allocate memory */
if (!rc) {
void *addr = NULL;
size_t size = getpagesize();
struct ibv_mr *ib_mr = NULL;
uint64_t access_flag = IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ;
uint64_t exp_access_flag = 0;
OBJ_CONSTRUCT(&device->ib_mr_array, opal_value_array_t);
opal_value_array_init(&device->ib_mr_array, sizeof(struct ibv_mr *));
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
exp_access_flag = IBV_EXP_ACCESS_ALLOCATE_MR |
IBV_EXP_ACCESS_SHARED_MR_USER_READ |
IBV_EXP_ACCESS_SHARED_MR_USER_WRITE;
#endif /* MPAGE_ENABLE */
struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag|exp_access_flag, 0};
ib_mr = ibv_exp_reg_mr(&in);
if (NULL == ib_mr) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
} else {
device->ib_mr_shared = ib_mr;
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
}
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
if (!rc) {
struct ibv_exp_reg_shared_mr_in in_smr;
access_flag = IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ|
IBV_EXP_ACCESS_NO_RDMA;
addr = (void *)mca_sshmem_base_start_address;
mca_sshmem_verbs_fill_shared_mr(&in_smr, device->ib_pd, device->ib_mr_shared->handle, addr, access_flag);
ib_mr = ibv_exp_reg_shared_mr(&in_smr);
if (NULL == ib_mr) {
mca_sshmem_verbs_component.has_shared_mr = 0;
} else {
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
mca_sshmem_verbs_component.has_shared_mr = 1;
}
}
#endif /* MPAGE_ENABLE */
}
/* all is well - rainbows and butterflies */
if (!rc) {
*priority = mca_sshmem_verbs_component.priority;
*module = (mca_base_module_t *)&mca_sshmem_verbs_module.super;
}
out:
if (device) {
if (opal_value_array_get_size(&device->ib_mr_array)) {
struct ibv_mr** array;
struct ibv_mr* ib_mr = NULL;
array = OPAL_VALUE_ARRAY_GET_BASE(&device->ib_mr_array, struct ibv_mr *);
while (opal_value_array_get_size(&device->ib_mr_array) > 0) {
ib_mr = array[0];
ibv_dereg_mr(ib_mr);
opal_value_array_remove_item(&device->ib_mr_array, 0);
}
if (device->ib_mr_shared) {
device->ib_mr_shared = NULL;
}
OBJ_DESTRUCT(&device->ib_mr_array);
}
if (device->ib_pd) {
ibv_dealloc_pd(device->ib_pd);
device->ib_pd = NULL;
}
if(device->ib_dev_context) {
ibv_close_device(device->ib_dev_context);
device->ib_dev_context = NULL;
}
if(device->ib_devs) {
ibv_free_device_list(device->ib_devs);
device->ib_devs = NULL;
}
}
return rc;
}
static inline int fi_ibv_get_qp_cap(struct ibv_context *ctx,
struct ibv_device_attr *device_attr,
struct fi_info *info)
{
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_qp *qp;
struct ibv_qp_init_attr init_attr;
int ret = 0;
pd = ibv_alloc_pd(ctx);
if (!pd) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_alloc_pd", errno);
return -errno;
}
cq = ibv_create_cq(ctx, 1, NULL, NULL, 0);
if (!cq) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_cq", errno);
ret = -errno;
goto err1;
}
/* TODO: serialize access to string buffers */
fi_read_file(FI_CONF_DIR, "def_tx_ctx_size",
def_tx_ctx_size, sizeof def_tx_ctx_size);
fi_read_file(FI_CONF_DIR, "def_rx_ctx_size",
def_rx_ctx_size, sizeof def_rx_ctx_size);
fi_read_file(FI_CONF_DIR, "def_tx_iov_limit",
def_tx_iov_limit, sizeof def_tx_iov_limit);
fi_read_file(FI_CONF_DIR, "def_rx_iov_limit",
def_rx_iov_limit, sizeof def_rx_iov_limit);
fi_read_file(FI_CONF_DIR, "def_inject_size",
def_inject_size, sizeof def_inject_size);
memset(&init_attr, 0, sizeof init_attr);
init_attr.send_cq = cq;
init_attr.recv_cq = cq;
init_attr.cap.max_send_wr = MIN(atoi(def_tx_ctx_size), device_attr->max_qp_wr);
init_attr.cap.max_recv_wr = MIN(atoi(def_rx_ctx_size), device_attr->max_qp_wr);
init_attr.cap.max_send_sge = MIN(atoi(def_tx_iov_limit), device_attr->max_sge);
init_attr.cap.max_recv_sge = MIN(atoi(def_rx_iov_limit), device_attr->max_sge);
init_attr.cap.max_inline_data = atoi(def_inject_size);
init_attr.qp_type = IBV_QPT_RC;
qp = ibv_create_qp(pd, &init_attr);
if (!qp) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_qp", errno);
ret = -errno;
goto err2;
}
info->tx_attr->inject_size = init_attr.cap.max_inline_data;
info->tx_attr->iov_limit = init_attr.cap.max_send_sge;
info->tx_attr->size = init_attr.cap.max_send_wr;
info->rx_attr->iov_limit = init_attr.cap.max_recv_sge;
/*
* On some HW ibv_create_qp can increase max_recv_wr value more than
* it really supports. So, alignment with device capability is needed.
*/
info->rx_attr->size = MIN(init_attr.cap.max_recv_wr,
device_attr->max_qp_wr);
ibv_destroy_qp(qp);
err2:
ibv_destroy_cq(cq);
err1:
ibv_dealloc_pd(pd);
return ret;
}
static inline int mca_oob_ud_device_setup (mca_oob_ud_device_t *device,
struct ibv_device *ib_device)
{
int rc, port_num;
struct ibv_device_attr dev_attr;
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup attempting to setup ib device %p",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (void *) ib_device));
device->ib_context = ibv_open_device (ib_device);
if (NULL == device->ib_context) {
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup error opening device. errno = %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), errno));
return ORTE_ERROR;
}
rc = ibv_query_device (device->ib_context, &dev_attr);
if (0 != rc) {
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup error querying device. errno = %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), errno));
return ORTE_ERROR;
}
device->ib_channel = ibv_create_comp_channel (device->ib_context);
if (NULL == device->ib_channel) {
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup error completing completion channel."
"errno = %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), errno));
return ORTE_ERROR;
}
device->ib_pd = ibv_alloc_pd (device->ib_context);
if (NULL == device->ib_pd) {
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup error allocating protection domain."
"errno = %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), errno));
return ORTE_ERROR;
}
for (port_num = 1 ; port_num <= dev_attr.phys_port_cnt ; ++port_num) {
mca_oob_ud_port_t *port = OBJ_NEW(mca_oob_ud_port_t);
if (NULL == port) {
opal_output (0, "oob:ud:device_setup malloc failure. errno = %d", errno);
return ORTE_ERR_OUT_OF_RESOURCE;
}
port->device = device;
port->port_num = port_num;
rc = mca_oob_ud_port_setup (port);
if (ORTE_SUCCESS != rc) {
OBJ_RELEASE(port);
continue;
}
opal_list_append (&device->ports, (opal_list_item_t *) port);
break;
}
if (0 == opal_list_get_size(&device->ports)) {
OPAL_OUTPUT_VERBOSE((5, mca_oob_base_output, "%s oob:ud:device_setup could not init device. no usable "
"ports present", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
return ORTE_ERROR;
}
return ORTE_SUCCESS;
}
static int ibw_setup_cq_qp(struct ibw_conn *conn)
{
struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
struct ibv_qp_init_attr init_attr;
struct ibv_qp_attr attr;
int rc;
DEBUG(DEBUG_DEBUG, ("ibw_setup_cq_qp(cmid: %p)\n", pconn->cm_id));
/* init verbs */
pconn->verbs_channel = ibv_create_comp_channel(pconn->cm_id->verbs);
if (!pconn->verbs_channel) {
sprintf(ibw_lasterr, "ibv_create_comp_channel failed %d\n", errno);
return -1;
}
DEBUG(DEBUG_DEBUG, ("created channel %p\n", pconn->verbs_channel));
pconn->verbs_channel_event = tevent_add_fd(pctx->ectx, NULL, /* not pconn or conn */
pconn->verbs_channel->fd, TEVENT_FD_READ, ibw_event_handler_verbs, conn);
pconn->pd = ibv_alloc_pd(pconn->cm_id->verbs);
if (!pconn->pd) {
sprintf(ibw_lasterr, "ibv_alloc_pd failed %d\n", errno);
return -1;
}
DEBUG(DEBUG_DEBUG, ("created pd %p\n", pconn->pd));
/* init mr */
if (ibw_init_memory(conn))
return -1;
/* init cq */
pconn->cq = ibv_create_cq(pconn->cm_id->verbs,
pctx->opts.max_recv_wr + pctx->opts.max_send_wr,
conn, pconn->verbs_channel, 0);
if (pconn->cq==NULL) {
sprintf(ibw_lasterr, "ibv_create_cq failed\n");
return -1;
}
rc = ibv_req_notify_cq(pconn->cq, 0);
if (rc) {
sprintf(ibw_lasterr, "ibv_req_notify_cq failed with %d\n", rc);
return rc;
}
/* init qp */
memset(&init_attr, 0, sizeof(init_attr));
init_attr.cap.max_send_wr = pctx->opts.max_send_wr;
init_attr.cap.max_recv_wr = pctx->opts.max_recv_wr;
init_attr.cap.max_recv_sge = 1;
init_attr.cap.max_send_sge = 1;
init_attr.qp_type = IBV_QPT_RC;
init_attr.send_cq = pconn->cq;
init_attr.recv_cq = pconn->cq;
rc = rdma_create_qp(pconn->cm_id, pconn->pd, &init_attr);
if (rc) {
sprintf(ibw_lasterr, "rdma_create_qp failed with %d\n", rc);
return rc;
}
/* elase result is in pconn->cm_id->qp */
rc = ibv_query_qp(pconn->cm_id->qp, &attr, IBV_QP_PATH_MTU, &init_attr);
if (rc) {
sprintf(ibw_lasterr, "ibv_query_qp failed with %d\n", rc);
return rc;
}
return ibw_fill_cq(conn);
}
int main(int argc, char *argv[]) {
struct pdata rep_pdata;
struct rdma_event_channel *cm_channel;
struct rdma_cm_id *listen_id;
struct rdma_cm_id *cm_id;
struct rdma_cm_event *event;
struct rdma_conn_param conn_param = { };
struct ibv_pd *pd;
struct ibv_comp_channel *comp_chan;
struct ibv_cq *cq;
struct ibv_cq *evt_cq;
struct ibv_mr *mr;
struct ibv_qp_init_attr qp_attr = { };
struct ibv_sge sge;
struct ibv_send_wr send_wr = { };
struct ibv_send_wr *bad_send_wr;
struct ibv_recv_wr recv_wr = { };
struct ibv_recv_wr *bad_recv_wr;
struct ibv_wc wc;
void *cq_context;
struct sockaddr_in sin;
uint32_t *buf;
int err;
/* Set up RDMA CM structures */
cm_channel = rdma_create_event_channel();
if (!cm_channel)
return 1;
err = rdma_create_id(cm_channel, &listen_id, NULL, RDMA_PS_TCP);
if (err)
return err;
sin.sin_family = AF_INET;
sin.sin_port = htons(20079);
sin.sin_addr.s_addr = INADDR_ANY;
/* Bind to local port and listen for connection request */
err = rdma_bind_addr(listen_id, (struct sockaddr *) &sin);
if (err)
return 1;
err = rdma_listen(listen_id, 1);
if (err)
return 1;
err = rdma_get_cm_event(cm_channel, &event);
if (err)
return err;
printf("after get_cm_event\n");
if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST)
return 1;
cm_id = event->id;
rdma_ack_cm_event(event);
/* Create verbs objects now that we know which device to use */
pd = ibv_alloc_pd(cm_id->verbs);
if (!pd)
return 1;
comp_chan = ibv_create_comp_channel(cm_id->verbs);
if (!comp_chan)
return 1;
cq = ibv_create_cq(cm_id->verbs, 2, NULL, comp_chan, 0);
if (!cq)
return 1;
if (ibv_req_notify_cq(cq, 0))
return 1;
buf = calloc(2, sizeof(uint32_t));
if (!buf)
return 1;
mr = ibv_reg_mr(pd, buf, 2 * sizeof(uint32_t),
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ
| IBV_ACCESS_REMOTE_WRITE);
if (!mr)
return 1;
qp_attr.cap.max_send_wr = 1;
qp_attr.cap.max_send_sge = 1;
qp_attr.cap.max_recv_wr = 1;
qp_attr.cap.max_recv_sge = 1;
qp_attr.send_cq = cq;
qp_attr.recv_cq = cq;
qp_attr.qp_type = IBV_QPT_RC;
err = rdma_create_qp(cm_id, pd, &qp_attr);
if (err)
return err;
/* Post receive before accepting connection */
sge.addr = (uintptr_t) buf + sizeof(uint32_t);
sge.length = sizeof(uint32_t);
sge.lkey = mr->lkey;
recv_wr.sg_list = &sge;
recv_wr.num_sge = 1;
if (ibv_post_recv(cm_id->qp, &recv_wr, &bad_recv_wr))
return 1;
rep_pdata.buf_va = htonll((uintptr_t) buf);
rep_pdata.buf_rkey = htonl(mr->rkey);
conn_param.responder_resources = 1;
conn_param.private_data = &rep_pdata;
conn_param.private_data_len = sizeof rep_pdata;
/* Accept connection */
printf("before accept\n");
err = rdma_accept(cm_id, &conn_param);
if (err)
return 1;
printf("after accept\n");
err = rdma_get_cm_event(cm_channel, &event);
if (err)
return err;
if (event->event != RDMA_CM_EVENT_ESTABLISHED)
return 1;
rdma_ack_cm_event(event);
/* Wait for receive completion */
if (ibv_get_cq_event(comp_chan, &evt_cq, &cq_context))
return 1;
if (ibv_req_notify_cq(cq, 0))
return 1;
if (ibv_poll_cq(cq, 1, &wc) < 1)
return 1;
if (wc.status != IBV_WC_SUCCESS)
return 1;
/* Add two integers and send reply back */
buf[0] = htonl(ntohl(buf[0]) + ntohl(buf[1]));
sge.addr = (uintptr_t) buf;
sge.length = sizeof(uint32_t);
sge.lkey = mr->lkey;
send_wr.opcode = IBV_WR_SEND;
send_wr.send_flags = IBV_SEND_SIGNALED;
send_wr.sg_list = &sge;
send_wr.num_sge = 1;
if (ibv_post_send(cm_id->qp, &send_wr, &bad_send_wr))
return 1;
/* Wait for send completion */
if (ibv_get_cq_event(comp_chan, &evt_cq, &cq_context))
return 1;
if (ibv_poll_cq(cq, 1, &wc) < 1)
return 1;
if (wc.status != IBV_WC_SUCCESS)
return 1;
printf("before ack cq 2\n");
ibv_ack_cq_events(cq, 2);
return 0;
}
ibv_pd* alloc_protection_domain(ibv_context* context) {
ibv_pd* pd = ibv_alloc_pd(context);
CHECK(pd) << "Failed to allocate protection domain";
return pd;
}
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
int tx_depth, int port,struct user_parameters *user_parm) {
struct pingpong_context *ctx;
struct ibv_device_attr device_attr;
ctx = malloc(sizeof *ctx);
if (!ctx)
return NULL;
ctx->size = size;
ctx->tx_depth = tx_depth;
/* in case of UD need space for the GRH */
if (user_parm->connection_type==UD) {
ctx->buf = memalign(page_size, ( size + 40 ) * 2);
if (!ctx->buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return NULL;
}
memset(ctx->buf, 0, ( size + 40 ) * 2);
} else {
ctx->buf = memalign(page_size, size * 2);
if (!ctx->buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return NULL;
}
memset(ctx->buf, 0, size * 2);
}
ctx->post_buf = (char*)ctx->buf + (size - 1);
ctx->poll_buf = (char*)ctx->buf + (2 * size - 1);
ctx->context = ibv_open_device(ib_dev);
if (!ctx->context) {
fprintf(stderr, "Couldn't get context for %s\n",
ibv_get_device_name(ib_dev));
return NULL;
}
if (user_parm->mtu == 0) {/*user did not ask for specific mtu */
if (ibv_query_device(ctx->context, &device_attr)) {
fprintf(stderr, "Failed to query device props");
return NULL;
}
if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {
user_parm->mtu = 1024;
} else {
user_parm->mtu = 2048;
}
}
if (user_parm->use_event) {
ctx->channel = ibv_create_comp_channel(ctx->context);
if (!ctx->channel) {
fprintf(stderr, "Couldn't create completion channel\n");
return NULL;
}
} else
ctx->channel = NULL;
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "Couldn't allocate PD\n");
return NULL;
}
if (user_parm->connection_type==UD) {
ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, (size + 40 ) * 2,
IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
fprintf(stderr, "Couldn't allocate MR\n");
return NULL;
}
} else {
ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size * 2,
IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
fprintf(stderr, "Couldn't allocate MR\n");
return NULL;
}
}
ctx->scq = ibv_create_cq(ctx->context, tx_depth, NULL, ctx->channel, 0);
if (!ctx->scq) {
fprintf(stderr, "Couldn't create CQ\n");
return NULL;
}
ctx->rcq = ibv_create_cq(ctx->context, tx_depth, NULL, ctx->channel, 0);
if (!ctx->rcq) {
fprintf(stderr, "Couldn't create Recieve CQ\n");
return NULL;
}
{
struct ibv_qp_init_attr attr;
memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
attr.send_cq = ctx->scq;
attr.recv_cq = ctx->rcq;
attr.cap.max_send_wr = tx_depth;
/* Work around: driver doesnt support
* recv_wr = 0 */
attr.cap.max_recv_wr = tx_depth;
attr.cap.max_send_sge = 1;
attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = user_parm->inline_size;
switch (user_parm->connection_type) {
case RC :
attr.qp_type = IBV_QPT_RC;
break;
case UC :
attr.qp_type = IBV_QPT_UC;
break;
case UD :
attr.qp_type = IBV_QPT_UD;
break;
default:
fprintf(stderr, "Unknown connection type %d \n",user_parm->connection_type);
return NULL;
}
attr.sq_sig_all = 0;
ctx->qp = ibv_create_qp(ctx->pd, &attr);
if (!ctx->qp) {
fprintf(stderr, "Couldn't create QP\n");
return NULL;
}
}
{
struct ibv_qp_attr attr;
memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
attr.qp_state = IBV_QPS_INIT;
attr.pkey_index = 0;
attr.port_num = port;
if (user_parm->connection_type==UD) {
attr.qkey = 0x11111111;
} else {
attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE;
}
if (user_parm->connection_type==UD) {
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_QKEY)) {
fprintf(stderr, "Failed to modify UD QP to INIT\n");
return NULL;
}
if (user_parm->use_mcg) {
union ibv_gid gid;
uint8_t mcg_gid[16] = MCG_GID;
/* use the local QP number as part of the mcg */
mcg_gid[11] = (user_parm->servername) ? 0 : 1;
*(uint32_t *)(&mcg_gid[12]) = ctx->qp->qp_num;
memcpy(gid.raw, mcg_gid, 16);
if (ibv_attach_mcast(ctx->qp, &gid, MCG_LID)) {
fprintf(stderr, "Couldn't attach QP to mcg\n");
return NULL;
}
}
} else if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "Failed to modify QP to INIT\n");
return NULL;
}
}
//send
ctx->wr.wr_id = PINGPONG_SEND_WRID;
ctx->wr.sg_list = &ctx->list;
ctx->wr.num_sge = 1;
ctx->wr.opcode = IBV_WR_SEND;
ctx->wr.next = NULL;
//recieve
ctx->rwr.wr_id = PINGPONG_RECV_WRID;
ctx->rwr.sg_list = &ctx->recv_list;
ctx->rwr.num_sge = 1;
ctx->rwr.next = NULL;
return ctx;
}