static int rxe_query_pkey(struct ib_device *device,
u8 port_num, u16 index, u16 *pkey)
{
struct rxe_dev *rxe = to_rdev(device);
struct rxe_port *port;
if (unlikely(port_num != 1)) {
dev_warn(device->dma_device, "invalid port_num = %d\n",
port_num);
goto err1;
}
port = &rxe->port;
if (unlikely(index >= port->attr.pkey_tbl_len)) {
dev_warn(device->dma_device, "invalid index = %d\n",
index);
goto err1;
}
*pkey = port->pkey_tbl[index];
return 0;
err1:
return -EINVAL;
}
/* called by the query qp verb */
int rxe_qp_to_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
*attr = qp->attr;
attr->rq_psn = qp->resp.psn;
attr->sq_psn = qp->req.psn;
attr->cap.max_send_wr = qp->sq.max_wr;
attr->cap.max_send_sge = qp->sq.max_sge;
attr->cap.max_inline_data = qp->sq.max_inline;
if (!qp->srq) {
attr->cap.max_recv_wr = qp->rq.max_wr;
attr->cap.max_recv_sge = qp->rq.max_sge;
}
rxe_av_to_attr(rxe, &qp->pri_av, &attr->ah_attr);
rxe_av_to_attr(rxe, &qp->alt_av, &attr->alt_ah_attr);
if (qp->req.state == QP_STATE_DRAIN) {
attr->sq_draining = 1;
/* applications that get this state
* typically spin on it. yield the
* processor */
cond_resched();
} else {
attr->sq_draining = 0;
}
pr_debug("attr->sq_draining = %d\n", attr->sq_draining);
return 0;
}
static struct ib_cq *rxe_create_cq(struct ib_device *dev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *context,
struct ib_udata *udata)
{
int err;
struct rxe_dev *rxe = to_rdev(dev);
struct rxe_cq *cq;
if (attr->flags)
return ERR_PTR(-EINVAL);
err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector, udata);
if (err)
goto err1;
cq = rxe_alloc(&rxe->cq_pool);
if (!cq) {
err = -ENOMEM;
goto err1;
}
err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector,
context, udata);
if (err)
goto err2;
return &cq->ibcq;
err2:
rxe_drop_ref(cq);
err1:
return ERR_PTR(err);
}
static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
return rxe_add_to_pool(&rxe->pd_pool, &pd->pelem);
}
static int rxe_query_port(struct ib_device *dev,
u8 port_num, struct ib_port_attr *attr)
{
struct rxe_dev *rxe = to_rdev(dev);
struct rxe_port *port;
int rc;
port = &rxe->port;
/* *attr being zeroed by the caller, avoid zeroing it here */
*attr = port->attr;
mutex_lock(&rxe->usdev_lock);
rc = ib_get_eth_speed(dev, port_num, &attr->active_speed,
&attr->active_width);
if (attr->state == IB_PORT_ACTIVE)
attr->phys_state = RDMA_LINK_PHYS_STATE_LINK_UP;
else if (dev_get_flags(rxe->ndev) & IFF_UP)
attr->phys_state = RDMA_LINK_PHYS_STATE_POLLING;
else
attr->phys_state = RDMA_LINK_PHYS_STATE_DISABLED;
mutex_unlock(&rxe->usdev_lock);
return rc;
}
static int rxe_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
{
struct rxe_dev *rxe = to_rdev(uctx->device);
struct rxe_ucontext *uc = to_ruc(uctx);
return rxe_add_to_pool(&rxe->uc_pool, &uc->pelem);
}
static int rxe_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
{
int err;
struct rxe_cq *cq = to_rcq(ibcq);
struct rxe_dev *rxe = to_rdev(ibcq->device);
struct rxe_resize_cq_resp __user *uresp = NULL;
if (udata) {
if (udata->outlen < sizeof(*uresp))
return -EINVAL;
uresp = udata->outbuf;
}
err = rxe_cq_chk_attr(rxe, cq, cqe, 0);
if (err)
goto err1;
err = rxe_cq_resize_queue(cq, cqe, uresp, udata);
if (err)
goto err1;
return 0;
err1:
return err;
}
static enum rdma_link_layer rxe_get_link_layer(struct ib_device *dev,
u8 port_num)
{
struct rxe_dev *rxe = to_rdev(dev);
return rxe_link_layer(rxe, port_num);
}
static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
{
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
struct rxe_mem *mr;
int err;
mr = rxe_alloc(&rxe->mr_pool);
if (!mr) {
err = -ENOMEM;
goto err1;
}
rxe_add_index(mr);
rxe_add_ref(pd);
err = rxe_mem_init_dma(pd, access, mr);
if (err)
goto err2;
return &mr->ibmr;
err2:
rxe_drop_ref(pd);
rxe_drop_index(mr);
rxe_drop_ref(mr);
err1:
return ERR_PTR(err);
}
static int rxe_detach_mcast(struct ib_qp *ibqp, union ib_gid *mgid, u16 mlid)
{
struct rxe_dev *rxe = to_rdev(ibqp->device);
struct rxe_qp *qp = to_rqp(ibqp);
return rxe_mcast_drop_grp_elem(rxe, qp, mgid);
}
static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata)
{
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
struct rxe_mem *mr;
int err;
if (mr_type != IB_MR_TYPE_MEM_REG)
return ERR_PTR(-EINVAL);
mr = rxe_alloc(&rxe->mr_pool);
if (!mr) {
err = -ENOMEM;
goto err1;
}
rxe_add_index(mr);
rxe_add_ref(pd);
err = rxe_mem_init_fast(pd, max_num_sg, mr);
if (err)
goto err2;
return &mr->ibmr;
err2:
rxe_drop_ref(pd);
rxe_drop_index(mr);
rxe_drop_ref(mr);
err1:
return ERR_PTR(err);
}
static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
u64 start,
u64 length,
u64 iova,
int access, struct ib_udata *udata)
{
int err;
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
struct rxe_mem *mr;
mr = rxe_alloc(&rxe->mr_pool);
if (!mr) {
err = -ENOMEM;
goto err2;
}
rxe_add_index(mr);
rxe_add_ref(pd);
err = rxe_mem_init_user(pd, start, length, iova,
access, udata, mr);
if (err)
goto err3;
return &mr->ibmr;
err3:
rxe_drop_ref(pd);
rxe_drop_index(mr);
rxe_drop_ref(mr);
err2:
return ERR_PTR(err);
}
static int rxe_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
enum ib_srq_attr_mask mask,
struct ib_udata *udata)
{
int err;
struct rxe_srq *srq = to_rsrq(ibsrq);
struct rxe_dev *rxe = to_rdev(ibsrq->device);
struct rxe_modify_srq_cmd ucmd = {};
if (udata) {
if (udata->inlen < sizeof(ucmd))
return -EINVAL;
err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
if (err)
return err;
}
err = rxe_srq_chk_attr(rxe, srq, attr, mask);
if (err)
goto err1;
err = rxe_srq_from_attr(rxe, srq, attr, mask, &ucmd, udata);
if (err)
goto err1;
return 0;
err1:
return err;
}
static struct ib_ah *rxe_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
{
int err;
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
struct rxe_ah *ah;
err = rxe_av_chk_attr(rxe, attr);
if (err)
goto err1;
ah = rxe_alloc(&rxe->ah_pool);
if (!ah) {
err = -ENOMEM;
goto err1;
}
rxe_add_ref(pd);
ah->pd = pd;
err = rxe_init_av(rxe, attr, &ah->av);
if (err)
goto err2;
return &ah->ibah;
err2:
rxe_drop_ref(pd);
rxe_drop_ref(ah);
err1:
return ERR_PTR(err);
}
/* called by the destroy qp verb */
void rxe_qp_destroy(struct rxe_qp *qp)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
qp->valid = 0;
qp->qp_timeout_jiffies = 0;
rxe_cleanup_task(&qp->resp.task);
del_timer_sync(&qp->retrans_timer);
del_timer_sync(&qp->rnr_nak_timer);
rxe_cleanup_task(&qp->req.task);
if (qp_type(qp) == IB_QPT_RC)
rxe_cleanup_task(&qp->comp.task);
/* flush out any receive wr's or pending requests */
__rxe_do_task(&qp->req.task);
if (qp->sq.queue) {
__rxe_do_task(&qp->comp.task);
__rxe_do_task(&qp->req.task);
}
/* drain the output queue */
while (!list_empty(&qp->arbiter_list))
__rxe_do_task(&rxe->arbiter.task);
}
static int rxe_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
{
struct rxe_dev *rxe = to_rdev(ibah->device);
struct rxe_ah *ah = to_rah(ibah);
rxe_av_to_attr(rxe, &ah->av, attr);
return 0;
}
static struct ib_ucontext *rxe_alloc_ucontext(struct ib_device *dev,
struct ib_udata *udata)
{
struct rxe_dev *rxe = to_rdev(dev);
struct rxe_ucontext *uc;
uc = rxe_alloc(&rxe->uc_pool);
return uc ? &uc->ibuc : ERR_PTR(-ENOMEM);
}
static inline int get_mtu(struct rxe_qp *qp)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC))
return qp->mtu;
return rxe->port.mtu_cap;
}
static int rxe_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
{
struct rxe_dev *rxe = to_rdev(ibah->device);
struct rxe_ah *ah = to_rah(ibah);
memset(attr, 0, sizeof(*attr));
attr->type = ibah->type;
rxe_av_to_attr(rxe, &ah->av, attr);
return 0;
}
static struct ib_pd *rxe_alloc_pd(struct ib_device *dev,
struct ib_ucontext *context,
struct ib_udata *udata)
{
struct rxe_dev *rxe = to_rdev(dev);
struct rxe_pd *pd;
pd = rxe_alloc(&rxe->pd_pool);
return pd ? &pd->ibpd : ERR_PTR(-ENOMEM);
}
static struct net_device *rxe_get_netdev(struct ib_device *device,
u8 port_num)
{
struct rxe_dev *rxe = to_rdev(device);
if (rxe->ndev) {
dev_hold(rxe->ndev);
return rxe->ndev;
}
return NULL;
}
static int rxe_query_device(struct ib_device *dev,
struct ib_device_attr *attr,
struct ib_udata *uhw)
{
struct rxe_dev *rxe = to_rdev(dev);
if (uhw->inlen || uhw->outlen)
return -EINVAL;
*attr = rxe->attr;
return 0;
}
static int rxe_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
{
int err;
struct rxe_dev *rxe = to_rdev(ibah->device);
struct rxe_ah *ah = to_rah(ibah);
err = rxe_av_chk_attr(rxe, attr);
if (err)
return err;
rxe_init_av(attr, &ah->av);
return 0;
}
void free_rd_atomic_resource(struct rxe_qp *qp, struct resp_res *res)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
if (res->type == RXE_ATOMIC_MASK) {
rxe_drop_ref(qp);
kfree_skb(res->atomic.skb);
atomic_dec(&rxe->resp_skb_out);
} else if (res->type == RXE_READ_MASK) {
if (res->read.mr)
rxe_drop_ref(res->read.mr);
}
res->type = 0;
}
static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr,
int data_len)
{
int err;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
err = copy_data(rxe, qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
data_addr, data_len, to_mem_obj, NULL);
if (unlikely(err))
return (err == -ENOSPC) ? RESPST_ERR_LENGTH
: RESPST_ERR_MALFORMED_WQE;
return RESPST_NONE;
}
static struct ib_qp *rxe_create_qp(struct ib_pd *ibpd,
struct ib_qp_init_attr *init,
struct ib_udata *udata)
{
int err;
struct rxe_dev *rxe = to_rdev(ibpd->device);
struct rxe_pd *pd = to_rpd(ibpd);
struct rxe_qp *qp;
struct rxe_create_qp_resp __user *uresp = NULL;
if (udata) {
if (udata->outlen < sizeof(*uresp))
return ERR_PTR(-EINVAL);
uresp = udata->outbuf;
}
err = rxe_qp_chk_init(rxe, init);
if (err)
goto err1;
qp = rxe_alloc(&rxe->qp_pool);
if (!qp) {
err = -ENOMEM;
goto err1;
}
if (udata) {
if (udata->inlen) {
err = -EINVAL;
goto err2;
}
qp->is_user = 1;
}
rxe_add_index(qp);
err = rxe_qp_from_init(rxe, qp, pd, init, uresp, ibpd, udata);
if (err)
goto err3;
return &qp->ibqp;
err3:
rxe_drop_index(qp);
err2:
rxe_drop_ref(qp);
err1:
return ERR_PTR(err);
}
static int rxe_modify_device(struct ib_device *dev,
int mask, struct ib_device_modify *attr)
{
struct rxe_dev *rxe = to_rdev(dev);
if (mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID)
rxe->attr.sys_image_guid = cpu_to_be64(attr->sys_image_guid);
if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
memcpy(rxe->ib_dev.node_desc,
attr->node_desc, sizeof(rxe->ib_dev.node_desc));
}
return 0;
}
static int rxe_modify_port(struct ib_device *dev,
u8 port_num, int mask, struct ib_port_modify *attr)
{
struct rxe_dev *rxe = to_rdev(dev);
struct rxe_port *port;
port = &rxe->port;
port->attr.port_cap_flags |= attr->set_port_cap_mask;
port->attr.port_cap_flags &= ~attr->clr_port_cap_mask;
if (mask & IB_PORT_RESET_QKEY_CNTR)
port->attr.qkey_viol_cntr = 0;
return 0;
}
static int rxe_attach_mcast(struct ib_qp *ibqp, union ib_gid *mgid, u16 mlid)
{
int err;
struct rxe_dev *rxe = to_rdev(ibqp->device);
struct rxe_qp *qp = to_rqp(ibqp);
struct rxe_mc_grp *grp;
/* takes a ref on grp if successful */
err = rxe_mcast_get_grp(rxe, mgid, &grp);
if (err)
return err;
err = rxe_mcast_add_grp_elem(rxe, qp, grp);
rxe_drop_ref(grp);
return err;
}
static inline enum comp_state do_atomic(struct rxe_qp *qp,
struct rxe_pkt_info *pkt,
struct rxe_send_wqe *wqe)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
int ret;
u64 atomic_orig = atmack_orig(pkt);
ret = copy_data(rxe, qp->pd, IB_ACCESS_LOCAL_WRITE,
&wqe->dma, &atomic_orig,
sizeof(u64), to_mem_obj, NULL);
if (ret)
return COMPST_ERROR;
else
return COMPST_COMP_ACK;
}