/** * rvt_query_port: Passes the query port call to the driver * @ibdev: Verbs IB dev * @port_num: port number, 1 based from ib core * @props: structure to hold returned properties * * Return: 0 on success */ static int rvt_query_port(struct ib_device *ibdev, u8 port_num, struct ib_port_attr *props) { struct rvt_dev_info *rdi = ib_to_rvt(ibdev); struct rvt_ibport *rvp; int port_index = ibport_num_to_idx(ibdev, port_num); if (port_index < 0) return -EINVAL; rvp = rdi->ports[port_index]; /* props being zeroed by the caller, avoid zeroing it here */ props->sm_lid = rvp->sm_lid; props->sm_sl = rvp->sm_sl; props->port_cap_flags = rvp->port_cap_flags; props->max_msg_sz = 0x80000000; props->pkey_tbl_len = rvt_get_npkeys(rdi); props->bad_pkey_cntr = rvp->pkey_violations; props->qkey_viol_cntr = rvp->qkey_violations; props->subnet_timeout = rvp->subnet_timeout; props->init_type_reply = 0; /* Populate the remaining ib_port_attr elements */ return rdi->driver_f.query_port_state(rdi, port_num, props); }
/** * rvt_destroy_qp - destroy a queue pair * @ibqp: the queue pair to destroy * * Note that this can be called while the QP is actively sending or * receiving! * * Return: 0 on success. */ int rvt_destroy_qp(struct ib_qp *ibqp) { struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); spin_lock_irq(&qp->r_lock); spin_lock(&qp->s_hlock); spin_lock(&qp->s_lock); rvt_reset_qp(rdi, qp, ibqp->qp_type); spin_unlock(&qp->s_lock); spin_unlock(&qp->s_hlock); spin_unlock_irq(&qp->r_lock); /* qpn is now available for use again */ rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); spin_lock(&rdi->n_qps_lock); rdi->n_qps_allocated--; if (qp->ibqp.qp_type == IB_QPT_RC) { rdi->n_rc_qps--; rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL; } spin_unlock(&rdi->n_qps_lock); if (qp->ip) kref_put(&qp->ip->ref, rvt_release_mmap_info); else vfree(qp->r_rq.wq); vfree(qp->s_wq); rdi->driver_f.qp_priv_free(rdi, qp); kfree(qp); return 0; }
/** * rvt_modify_port * @ibdev: Verbs IB dev * @port_num: Port number, 1 based from ib core * @port_modify_mask: How to change the port * @props: Structure to fill in * * Return: 0 on success */ static int rvt_modify_port(struct ib_device *ibdev, u8 port_num, int port_modify_mask, struct ib_port_modify *props) { struct rvt_dev_info *rdi = ib_to_rvt(ibdev); struct rvt_ibport *rvp; int ret = 0; int port_index = ibport_num_to_idx(ibdev, port_num); if (port_index < 0) return -EINVAL; rvp = rdi->ports[port_index]; if (port_modify_mask & IB_PORT_OPA_MASK_CHG) { rvp->port_cap3_flags |= props->set_port_cap_mask; rvp->port_cap3_flags &= ~props->clr_port_cap_mask; } else { rvp->port_cap_flags |= props->set_port_cap_mask; rvp->port_cap_flags &= ~props->clr_port_cap_mask; } if (props->set_port_cap_mask || props->clr_port_cap_mask) rdi->driver_f.cap_mask_chg(rdi, port_num); if (port_modify_mask & IB_PORT_SHUTDOWN) ret = rdi->driver_f.shut_down_port(rdi, port_num); if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR) rvp->qkey_violations = 0; return ret; }
/** * rvt_dealloc_pd - Free PD * @ibpd: Free up PD * * Return: always 0 */ int rvt_dealloc_pd(struct ib_pd *ibpd) { struct rvt_pd *pd = ibpd_to_rvtpd(ibpd); struct rvt_dev_info *dev = ib_to_rvt(ibpd->device); spin_lock(&dev->n_pds_lock); dev->n_pds_allocated--; spin_unlock(&dev->n_pds_lock); kfree(pd); return 0; }
static int rvt_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *uhw) { struct rvt_dev_info *rdi = ib_to_rvt(ibdev); if (uhw->inlen || uhw->outlen) return -EINVAL; /* * Return rvt_dev_info.dparms.props contents */ *props = rdi->dparms.props; return 0; }
/** * rvt_query_qp - query an ipbq * @ibqp: IB qp to query * @attr: attr struct to fill in * @attr_mask: attr mask ignored * @init_attr: struct to fill in * * Return: always 0 */ int rvt_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_qp_init_attr *init_attr) { struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); attr->qp_state = qp->state; attr->cur_qp_state = attr->qp_state; attr->path_mtu = qp->path_mtu; attr->path_mig_state = qp->s_mig_state; attr->qkey = qp->qkey; attr->rq_psn = qp->r_psn & rdi->dparms.psn_mask; attr->sq_psn = qp->s_next_psn & rdi->dparms.psn_mask; attr->dest_qp_num = qp->remote_qpn; attr->qp_access_flags = qp->qp_access_flags; attr->cap.max_send_wr = qp->s_size - 1; attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1; attr->cap.max_send_sge = qp->s_max_sge; attr->cap.max_recv_sge = qp->r_rq.max_sge; attr->cap.max_inline_data = 0; attr->ah_attr = qp->remote_ah_attr; attr->alt_ah_attr = qp->alt_ah_attr; attr->pkey_index = qp->s_pkey_index; attr->alt_pkey_index = qp->s_alt_pkey_index; attr->en_sqd_async_notify = 0; attr->sq_draining = qp->s_draining; attr->max_rd_atomic = qp->s_max_rd_atomic; attr->max_dest_rd_atomic = qp->r_max_rd_atomic; attr->min_rnr_timer = qp->r_min_rnr_timer; attr->port_num = qp->port_num; attr->timeout = qp->timeout; attr->retry_cnt = qp->s_retry_cnt; attr->rnr_retry = qp->s_rnr_retry_cnt; attr->alt_port_num = qp->alt_ah_attr.port_num; attr->alt_timeout = qp->alt_timeout; init_attr->event_handler = qp->ibqp.event_handler; init_attr->qp_context = qp->ibqp.qp_context; init_attr->send_cq = qp->ibqp.send_cq; init_attr->recv_cq = qp->ibqp.recv_cq; init_attr->srq = qp->ibqp.srq; init_attr->cap = attr->cap; if (qp->s_flags & RVT_S_SIGNAL_REQ_WR) init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; else init_attr->sq_sig_type = IB_SIGNAL_ALL_WR; init_attr->qp_type = qp->ibqp.qp_type; init_attr->port_num = qp->port_num; return 0; }
/** * rvt_destroy_srq - destory an srq * @ibsrq: srq object to destroy * * Return always 0 */ int rvt_destroy_srq(struct ib_srq *ibsrq) { struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); spin_lock(&dev->n_srqs_lock); dev->n_srqs_allocated--; spin_unlock(&dev->n_srqs_lock); if (srq->ip) kref_put(&srq->ip->ref, rvt_release_mmap_info); else vfree(srq->rq.wq); kfree(srq); return 0; }
/** * rvt_post_send - post a send on a QP * @ibqp: the QP to post the send on * @wr: the list of work requests to post * @bad_wr: the first bad WR is put here * * This may be called from interrupt context. * * Return: 0 on success else errno */ int rvt_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, struct ib_send_wr **bad_wr) { struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); unsigned long flags = 0; int call_send; unsigned nreq = 0; int err = 0; spin_lock_irqsave(&qp->s_hlock, flags); /* * Ensure QP state is such that we can send. If not bail out early, * there is no need to do this every time we post a send. */ if (unlikely(!(ib_rvt_state_ops[qp->state] & RVT_POST_SEND_OK))) { spin_unlock_irqrestore(&qp->s_hlock, flags); return -EINVAL; } /* * If the send queue is empty, and we only have a single WR then just go * ahead and kick the send engine into gear. Otherwise we will always * just schedule the send to happen later. */ call_send = qp->s_head == ACCESS_ONCE(qp->s_last) && !wr->next; for (; wr; wr = wr->next) { err = rvt_post_one_wr(qp, wr, &call_send); if (unlikely(err)) { *bad_wr = wr; goto bail; } nreq++; } bail: spin_unlock_irqrestore(&qp->s_hlock, flags); if (nreq) { if (call_send) rdi->driver_f.schedule_send_no_lock(qp); else rdi->driver_f.do_send(qp); } return err; }
/** * rvt_alloc_pd - allocate a protection domain * @ibdev: ib device * @context: optional user context * @udata: optional user data * * Allocate and keep track of a PD. * * Return: 0 on success */ struct ib_pd *rvt_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata) { struct rvt_dev_info *dev = ib_to_rvt(ibdev); struct rvt_pd *pd; struct ib_pd *ret; pd = kmalloc(sizeof(*pd), GFP_KERNEL); if (!pd) { ret = ERR_PTR(-ENOMEM); goto bail; } /* * While we could continue allocating protecetion domains, being * constrained only by system resources. The IBTA spec defines that * there is a max_pd limit that can be set and we need to check for * that. */ spin_lock(&dev->n_pds_lock); if (dev->n_pds_allocated == dev->dparms.props.max_pd) { spin_unlock(&dev->n_pds_lock); kfree(pd); ret = ERR_PTR(-ENOMEM); goto bail; } dev->n_pds_allocated++; spin_unlock(&dev->n_pds_lock); /* ib_alloc_pd() will initialize pd->ibpd. */ pd->user = udata ? 1 : 0; ret = &pd->ibpd; bail: return ret; }
/** * rvt_query_pkey - Return a pkey from the table at a given index * @ibdev: Verbs IB dev * @port_num: Port number, 1 based from ib core * @index: Index into pkey table * @pkey: returned pkey from the port pkey table * * Return: 0 on failure pkey otherwise */ static int rvt_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index, u16 *pkey) { /* * Driver will be responsible for keeping rvt_dev_info.pkey_table up to * date. This function will just return that value. There is no need to * lock, if a stale value is read and sent to the user so be it there is * no way to protect against that anyway. */ struct rvt_dev_info *rdi = ib_to_rvt(ibdev); int port_index; port_index = ibport_num_to_idx(ibdev, port_num); if (port_index < 0) return -EINVAL; if (index >= rvt_get_npkeys(rdi)) return -EINVAL; *pkey = rvt_get_pkey(rdi, port_index, index); return 0; }
/** * rvt_query_gid - Return a gid from the table * @ibdev: Verbs IB dev * @port_num: Port number, 1 based from ib core * @guid_index: Index in table * @gid: Gid to return * * Return: 0 on success */ static int rvt_query_gid(struct ib_device *ibdev, u8 port_num, int guid_index, union ib_gid *gid) { struct rvt_dev_info *rdi; struct rvt_ibport *rvp; int port_index; /* * Driver is responsible for updating the guid table. Which will be used * to craft the return value. This will work similar to how query_pkey() * is being done. */ port_index = ibport_num_to_idx(ibdev, port_num); if (port_index < 0) return -EINVAL; rdi = ib_to_rvt(ibdev); rvp = rdi->ports[port_index]; gid->global.subnet_prefix = rvp->gid_prefix; return rdi->driver_f.get_guid_be(rdi, rvp, guid_index, &gid->global.interface_id); }
/** * rvt_modify_srq - modify a shared receive queue * @ibsrq: the SRQ to modify * @attr: the new attributes of the SRQ * @attr_mask: indicates which attributes to modify * @udata: user data for libibverbs.so * * Return: 0 on success */ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) { struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); struct rvt_rwq *wq; int ret = 0; if (attr_mask & IB_SRQ_MAX_WR) { struct rvt_rwq *owq; struct rvt_rwqe *p; u32 sz, size, n, head, tail; /* Check that the requested sizes are below the limits. */ if ((attr->max_wr > dev->dparms.props.max_srq_wr) || ((attr_mask & IB_SRQ_LIMIT) ? attr->srq_limit : srq->limit) > attr->max_wr) return -EINVAL; sz = sizeof(struct rvt_rwqe) + srq->rq.max_sge * sizeof(struct ib_sge); size = attr->max_wr + 1; wq = udata ? vmalloc_user(sizeof(struct rvt_rwq) + size * sz) : vzalloc_node(sizeof(struct rvt_rwq) + size * sz, dev->dparms.node); if (!wq) return -ENOMEM; /* Check that we can write the offset to mmap. */ if (udata && udata->inlen >= sizeof(__u64)) { __u64 offset_addr; __u64 offset = 0; ret = ib_copy_from_udata(&offset_addr, udata, sizeof(offset_addr)); if (ret) goto bail_free; udata->outbuf = (void __user *) (unsigned long)offset_addr; ret = ib_copy_to_udata(udata, &offset, sizeof(offset)); if (ret) goto bail_free; } spin_lock_irq(&srq->rq.lock); /* * validate head and tail pointer values and compute * the number of remaining WQEs. */ owq = srq->rq.wq; head = owq->head; tail = owq->tail; if (head >= srq->rq.size || tail >= srq->rq.size) { ret = -EINVAL; goto bail_unlock; } n = head; if (n < tail) n += srq->rq.size - tail; else n -= tail; if (size <= n) { ret = -EINVAL; goto bail_unlock; } n = 0; p = wq->wq; while (tail != head) { struct rvt_rwqe *wqe; int i; wqe = rvt_get_rwqe_ptr(&srq->rq, tail); p->wr_id = wqe->wr_id; p->num_sge = wqe->num_sge; for (i = 0; i < wqe->num_sge; i++) p->sg_list[i] = wqe->sg_list[i]; n++; p = (struct rvt_rwqe *)((char *)p + sz); if (++tail >= srq->rq.size) tail = 0; } srq->rq.wq = wq; srq->rq.size = size; wq->head = n; wq->tail = 0; if (attr_mask & IB_SRQ_LIMIT) srq->limit = attr->srq_limit; spin_unlock_irq(&srq->rq.lock); vfree(owq); if (srq->ip) { struct rvt_mmap_info *ip = srq->ip; struct rvt_dev_info *dev = ib_to_rvt(srq->ibsrq.device); u32 s = sizeof(struct rvt_rwq) + size * sz; rvt_update_mmap_info(dev, ip, s, wq); /* * Return the offset to mmap. * See rvt_mmap() for details. */ if (udata && udata->inlen >= sizeof(__u64)) { ret = ib_copy_to_udata(udata, &ip->offset, sizeof(ip->offset)); if (ret) return ret; } /* * Put user mapping info onto the pending list * unless it already is on the list. */ spin_lock_irq(&dev->pending_lock); if (list_empty(&ip->pending_mmaps)) list_add(&ip->pending_mmaps, &dev->pending_mmaps); spin_unlock_irq(&dev->pending_lock); } } else if (attr_mask & IB_SRQ_LIMIT) { spin_lock_irq(&srq->rq.lock); if (attr->srq_limit >= srq->rq.size) ret = -EINVAL; else srq->limit = attr->srq_limit; spin_unlock_irq(&srq->rq.lock); } return ret; bail_unlock: spin_unlock_irq(&srq->rq.lock); bail_free: vfree(wq); return ret; }
/** * rvt_create_srq - create a shared receive queue * @ibpd: the protection domain of the SRQ to create * @srq_init_attr: the attributes of the SRQ * @udata: data from libibverbs when creating a user SRQ * * Return: Allocated srq object */ struct ib_srq *rvt_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *srq_init_attr, struct ib_udata *udata) { struct rvt_dev_info *dev = ib_to_rvt(ibpd->device); struct rvt_ucontext *ucontext = rdma_udata_to_drv_context( udata, struct rvt_ucontext, ibucontext); struct rvt_srq *srq; u32 sz; struct ib_srq *ret; if (srq_init_attr->srq_type != IB_SRQT_BASIC) return ERR_PTR(-EOPNOTSUPP); if (srq_init_attr->attr.max_sge == 0 || srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge || srq_init_attr->attr.max_wr == 0 || srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr) return ERR_PTR(-EINVAL); srq = kzalloc_node(sizeof(*srq), GFP_KERNEL, dev->dparms.node); if (!srq) return ERR_PTR(-ENOMEM); /* * Need to use vmalloc() if we want to support large #s of entries. */ srq->rq.size = srq_init_attr->attr.max_wr + 1; srq->rq.max_sge = srq_init_attr->attr.max_sge; sz = sizeof(struct ib_sge) * srq->rq.max_sge + sizeof(struct rvt_rwqe); srq->rq.wq = udata ? vmalloc_user(sizeof(struct rvt_rwq) + srq->rq.size * sz) : vzalloc_node(sizeof(struct rvt_rwq) + srq->rq.size * sz, dev->dparms.node); if (!srq->rq.wq) { ret = ERR_PTR(-ENOMEM); goto bail_srq; } /* * Return the address of the RWQ as the offset to mmap. * See rvt_mmap() for details. */ if (udata && udata->outlen >= sizeof(__u64)) { int err; u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz; srq->ip = rvt_create_mmap_info(dev, s, &ucontext->ibucontext, srq->rq.wq); if (!srq->ip) { ret = ERR_PTR(-ENOMEM); goto bail_wq; } err = ib_copy_to_udata(udata, &srq->ip->offset, sizeof(srq->ip->offset)); if (err) { ret = ERR_PTR(err); goto bail_ip; } } /* * ib_create_srq() will initialize srq->ibsrq. */ spin_lock_init(&srq->rq.lock); srq->limit = srq_init_attr->attr.srq_limit; spin_lock(&dev->n_srqs_lock); if (dev->n_srqs_allocated == dev->dparms.props.max_srq) { spin_unlock(&dev->n_srqs_lock); ret = ERR_PTR(-ENOMEM); goto bail_ip; } dev->n_srqs_allocated++; spin_unlock(&dev->n_srqs_lock); if (srq->ip) { spin_lock_irq(&dev->pending_lock); list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps); spin_unlock_irq(&dev->pending_lock); } return &srq->ibsrq; bail_ip: kfree(srq->ip); bail_wq: vfree(srq->rq.wq); bail_srq: kfree(srq); return ret; }
/** * rvt_create_cq - create a completion queue * @ibdev: the device this completion queue is attached to * @attr: creation attributes * @context: unused by the QLogic_IB driver * @udata: user data for libibverbs.so * * Called by ib_create_cq() in the generic verbs code. * * Return: pointer to the completion queue or negative errno values * for failure. */ struct ib_cq *rvt_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_ucontext *context, struct ib_udata *udata) { struct rvt_dev_info *rdi = ib_to_rvt(ibdev); struct rvt_cq *cq; struct rvt_cq_wc *wc; struct ib_cq *ret; u32 sz; unsigned int entries = attr->cqe; if (attr->flags) return ERR_PTR(-EINVAL); if (entries < 1 || entries > rdi->dparms.props.max_cqe) return ERR_PTR(-EINVAL); /* Allocate the completion queue structure. */ cq = kzalloc(sizeof(*cq), GFP_KERNEL); if (!cq) return ERR_PTR(-ENOMEM); /* * Allocate the completion queue entries and head/tail pointers. * This is allocated separately so that it can be resized and * also mapped into user space. * We need to use vmalloc() in order to support mmap and large * numbers of entries. */ sz = sizeof(*wc); if (udata && udata->outlen >= sizeof(__u64)) sz += sizeof(struct ib_uverbs_wc) * (entries + 1); else sz += sizeof(struct ib_wc) * (entries + 1); wc = vmalloc_user(sz); if (!wc) { ret = ERR_PTR(-ENOMEM); goto bail_cq; } /* * Return the address of the WC as the offset to mmap. * See rvt_mmap() for details. */ if (udata && udata->outlen >= sizeof(__u64)) { int err; cq->ip = rvt_create_mmap_info(rdi, sz, context, wc); if (!cq->ip) { ret = ERR_PTR(-ENOMEM); goto bail_wc; } err = ib_copy_to_udata(udata, &cq->ip->offset, sizeof(cq->ip->offset)); if (err) { ret = ERR_PTR(err); goto bail_ip; } } spin_lock(&rdi->n_cqs_lock); if (rdi->n_cqs_allocated == rdi->dparms.props.max_cq) { spin_unlock(&rdi->n_cqs_lock); ret = ERR_PTR(-ENOMEM); goto bail_ip; } rdi->n_cqs_allocated++; spin_unlock(&rdi->n_cqs_lock); if (cq->ip) { spin_lock_irq(&rdi->pending_lock); list_add(&cq->ip->pending_mmaps, &rdi->pending_mmaps); spin_unlock_irq(&rdi->pending_lock); } /* * ib_create_cq() will initialize cq->ibcq except for cq->ibcq.cqe. * The number of entries should be >= the number requested or return * an error. */ cq->rdi = rdi; cq->ibcq.cqe = entries; cq->notify = RVT_CQ_NONE; spin_lock_init(&cq->lock); init_kthread_work(&cq->comptask, send_complete); cq->queue = wc; ret = &cq->ibcq; goto done; bail_ip: kfree(cq->ip); bail_wc: vfree(wc); bail_cq: kfree(cq); done: return ret; }
/** * qib_modify_qp - modify the attributes of a queue pair * @ibqp: the queue pair who's attributes we're modifying * @attr: the new attributes * @attr_mask: the mask of attributes to modify * @udata: user data for libibverbs.so * * Return: 0 on success, otherwise returns an errno. */ int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); enum ib_qp_state cur_state, new_state; struct ib_event ev; int lastwqe = 0; int mig = 0; int pmtu = 0; /* for gcc warning only */ enum rdma_link_layer link; link = rdma_port_get_link_layer(ibqp->device, qp->port_num); spin_lock_irq(&qp->r_lock); spin_lock(&qp->s_hlock); spin_lock(&qp->s_lock); cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask, link)) goto inval; if (rdi->driver_f.check_modify_qp && rdi->driver_f.check_modify_qp(qp, attr, attr_mask, udata)) goto inval; if (attr_mask & IB_QP_AV) { if (attr->ah_attr.dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) goto inval; if (rvt_check_ah(qp->ibqp.device, &attr->ah_attr)) goto inval; } if (attr_mask & IB_QP_ALT_PATH) { if (attr->alt_ah_attr.dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) goto inval; if (rvt_check_ah(qp->ibqp.device, &attr->alt_ah_attr)) goto inval; if (attr->alt_pkey_index >= rvt_get_npkeys(rdi)) goto inval; } if (attr_mask & IB_QP_PKEY_INDEX) if (attr->pkey_index >= rvt_get_npkeys(rdi)) goto inval; if (attr_mask & IB_QP_MIN_RNR_TIMER) if (attr->min_rnr_timer > 31) goto inval; if (attr_mask & IB_QP_PORT) if (qp->ibqp.qp_type == IB_QPT_SMI || qp->ibqp.qp_type == IB_QPT_GSI || attr->port_num == 0 || attr->port_num > ibqp->device->phys_port_cnt) goto inval; if (attr_mask & IB_QP_DEST_QPN) if (attr->dest_qp_num > RVT_QPN_MASK) goto inval; if (attr_mask & IB_QP_RETRY_CNT) if (attr->retry_cnt > 7) goto inval; if (attr_mask & IB_QP_RNR_RETRY) if (attr->rnr_retry > 7) goto inval; /* * Don't allow invalid path_mtu values. OK to set greater * than the active mtu (or even the max_cap, if we have tuned * that to a small mtu. We'll set qp->path_mtu * to the lesser of requested attribute mtu and active, * for packetizing messages. * Note that the QP port has to be set in INIT and MTU in RTR. */ if (attr_mask & IB_QP_PATH_MTU) { pmtu = rdi->driver_f.get_pmtu_from_attr(rdi, qp, attr); if (pmtu < 0) goto inval; } if (attr_mask & IB_QP_PATH_MIG_STATE) { if (attr->path_mig_state == IB_MIG_REARM) { if (qp->s_mig_state == IB_MIG_ARMED) goto inval; if (new_state != IB_QPS_RTS) goto inval; } else if (attr->path_mig_state == IB_MIG_MIGRATED) { if (qp->s_mig_state == IB_MIG_REARM) goto inval; if (new_state != IB_QPS_RTS && new_state != IB_QPS_SQD) goto inval; if (qp->s_mig_state == IB_MIG_ARMED) mig = 1; } else { goto inval; } } if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) if (attr->max_dest_rd_atomic > rdi->dparms.max_rdma_atomic) goto inval; switch (new_state) { case IB_QPS_RESET: if (qp->state != IB_QPS_RESET) rvt_reset_qp(rdi, qp, ibqp->qp_type); break; case IB_QPS_RTR: /* Allow event to re-trigger if QP set to RTR more than once */ qp->r_flags &= ~RVT_R_COMM_EST; qp->state = new_state; break; case IB_QPS_SQD: qp->s_draining = qp->s_last != qp->s_cur; qp->state = new_state; break; case IB_QPS_SQE: if (qp->ibqp.qp_type == IB_QPT_RC) goto inval; qp->state = new_state; break; case IB_QPS_ERR: lastwqe = rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); break; default: qp->state = new_state; break; } if (attr_mask & IB_QP_PKEY_INDEX) qp->s_pkey_index = attr->pkey_index; if (attr_mask & IB_QP_PORT) qp->port_num = attr->port_num; if (attr_mask & IB_QP_DEST_QPN) qp->remote_qpn = attr->dest_qp_num; if (attr_mask & IB_QP_SQ_PSN) { qp->s_next_psn = attr->sq_psn & rdi->dparms.psn_modify_mask; qp->s_psn = qp->s_next_psn; qp->s_sending_psn = qp->s_next_psn; qp->s_last_psn = qp->s_next_psn - 1; qp->s_sending_hpsn = qp->s_last_psn; } if (attr_mask & IB_QP_RQ_PSN) qp->r_psn = attr->rq_psn & rdi->dparms.psn_modify_mask; if (attr_mask & IB_QP_ACCESS_FLAGS) qp->qp_access_flags = attr->qp_access_flags; if (attr_mask & IB_QP_AV) { qp->remote_ah_attr = attr->ah_attr; qp->s_srate = attr->ah_attr.static_rate; qp->srate_mbps = ib_rate_to_mbps(qp->s_srate); } if (attr_mask & IB_QP_ALT_PATH) { qp->alt_ah_attr = attr->alt_ah_attr; qp->s_alt_pkey_index = attr->alt_pkey_index; } if (attr_mask & IB_QP_PATH_MIG_STATE) { qp->s_mig_state = attr->path_mig_state; if (mig) { qp->remote_ah_attr = qp->alt_ah_attr; qp->port_num = qp->alt_ah_attr.port_num; qp->s_pkey_index = qp->s_alt_pkey_index; } } if (attr_mask & IB_QP_PATH_MTU) { qp->pmtu = rdi->driver_f.mtu_from_qp(rdi, qp, pmtu); qp->path_mtu = rdi->driver_f.mtu_to_path_mtu(qp->pmtu); qp->log_pmtu = ilog2(qp->pmtu); } if (attr_mask & IB_QP_RETRY_CNT) { qp->s_retry_cnt = attr->retry_cnt; qp->s_retry = attr->retry_cnt; } if (attr_mask & IB_QP_RNR_RETRY) { qp->s_rnr_retry_cnt = attr->rnr_retry; qp->s_rnr_retry = attr->rnr_retry; } if (attr_mask & IB_QP_MIN_RNR_TIMER) qp->r_min_rnr_timer = attr->min_rnr_timer; if (attr_mask & IB_QP_TIMEOUT) { qp->timeout = attr->timeout; qp->timeout_jiffies = usecs_to_jiffies((4096UL * (1UL << qp->timeout)) / 1000UL); } if (attr_mask & IB_QP_QKEY) qp->qkey = attr->qkey; if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) qp->r_max_rd_atomic = attr->max_dest_rd_atomic; if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) qp->s_max_rd_atomic = attr->max_rd_atomic; if (rdi->driver_f.modify_qp) rdi->driver_f.modify_qp(qp, attr, attr_mask, udata); spin_unlock(&qp->s_lock); spin_unlock(&qp->s_hlock); spin_unlock_irq(&qp->r_lock); if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) rvt_insert_qp(rdi, qp); if (lastwqe) { ev.device = qp->ibqp.device; ev.element.qp = &qp->ibqp; ev.event = IB_EVENT_QP_LAST_WQE_REACHED; qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); } if (mig) { ev.device = qp->ibqp.device; ev.element.qp = &qp->ibqp; ev.event = IB_EVENT_PATH_MIG; qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); } return 0; inval: spin_unlock(&qp->s_lock); spin_unlock(&qp->s_hlock); spin_unlock_irq(&qp->r_lock); return -EINVAL; }
/** * rvt_error_qp - put a QP into the error state * @qp: the QP to put into the error state * @err: the receive completion error to signal if a RWQE is active * * Flushes both send and receive work queues. * * Return: true if last WQE event should be generated. * The QP r_lock and s_lock should be held and interrupts disabled. * If we are already in error state, just return. */ int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err) { struct ib_wc wc; int ret = 0; struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET) goto bail; qp->state = IB_QPS_ERR; if (qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR)) { qp->s_flags &= ~(RVT_S_TIMER | RVT_S_WAIT_RNR); del_timer(&qp->s_timer); } if (qp->s_flags & RVT_S_ANY_WAIT_SEND) qp->s_flags &= ~RVT_S_ANY_WAIT_SEND; rdi->driver_f.notify_error_qp(qp); /* Schedule the sending tasklet to drain the send work queue. */ if (ACCESS_ONCE(qp->s_last) != qp->s_head) rdi->driver_f.schedule_send(qp); rvt_clear_mr_refs(qp, 0); memset(&wc, 0, sizeof(wc)); wc.qp = &qp->ibqp; wc.opcode = IB_WC_RECV; if (test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) { wc.wr_id = qp->r_wr_id; wc.status = err; rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); } wc.status = IB_WC_WR_FLUSH_ERR; if (qp->r_rq.wq) { struct rvt_rwq *wq; u32 head; u32 tail; spin_lock(&qp->r_rq.lock); /* sanity check pointers before trusting them */ wq = qp->r_rq.wq; head = wq->head; if (head >= qp->r_rq.size) head = 0; tail = wq->tail; if (tail >= qp->r_rq.size) tail = 0; while (tail != head) { wc.wr_id = rvt_get_rwqe_ptr(&qp->r_rq, tail)->wr_id; if (++tail >= qp->r_rq.size) tail = 0; rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1); } wq->tail = tail; spin_unlock(&qp->r_rq.lock); } else if (qp->ibqp.event_handler) { ret = 1; } bail: return ret; }
/** * rvt_create_qp - create a queue pair for a device * @ibpd: the protection domain who's device we create the queue pair for * @init_attr: the attributes of the queue pair * @udata: user data for libibverbs.so * * Queue pair creation is mostly an rvt issue. However, drivers have their own * unique idea of what queue pair numbers mean. For instance there is a reserved * range for PSM. * * Return: the queue pair on success, otherwise returns an errno. * * Called by the ib_create_qp() core verbs function. */ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { struct rvt_qp *qp; int err; struct rvt_swqe *swq = NULL; size_t sz; size_t sg_list_sz; struct ib_qp *ret = ERR_PTR(-ENOMEM); struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device); void *priv = NULL; gfp_t gfp; if (!rdi) return ERR_PTR(-EINVAL); if (init_attr->cap.max_send_sge > rdi->dparms.props.max_sge || init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr || init_attr->create_flags & ~(IB_QP_CREATE_USE_GFP_NOIO)) return ERR_PTR(-EINVAL); /* GFP_NOIO is applicable to RC QP's only */ if (init_attr->create_flags & IB_QP_CREATE_USE_GFP_NOIO && init_attr->qp_type != IB_QPT_RC) return ERR_PTR(-EINVAL); gfp = init_attr->create_flags & IB_QP_CREATE_USE_GFP_NOIO ? GFP_NOIO : GFP_KERNEL; /* Check receive queue parameters if no SRQ is specified. */ if (!init_attr->srq) { if (init_attr->cap.max_recv_sge > rdi->dparms.props.max_sge || init_attr->cap.max_recv_wr > rdi->dparms.props.max_qp_wr) return ERR_PTR(-EINVAL); if (init_attr->cap.max_send_sge + init_attr->cap.max_send_wr + init_attr->cap.max_recv_sge + init_attr->cap.max_recv_wr == 0) return ERR_PTR(-EINVAL); } switch (init_attr->qp_type) { case IB_QPT_SMI: case IB_QPT_GSI: if (init_attr->port_num == 0 || init_attr->port_num > ibpd->device->phys_port_cnt) return ERR_PTR(-EINVAL); case IB_QPT_UC: case IB_QPT_RC: case IB_QPT_UD: sz = sizeof(struct rvt_sge) * init_attr->cap.max_send_sge + sizeof(struct rvt_swqe); if (gfp == GFP_NOIO) swq = __vmalloc( (init_attr->cap.max_send_wr + 1) * sz, gfp, PAGE_KERNEL); else swq = vmalloc_node( (init_attr->cap.max_send_wr + 1) * sz, rdi->dparms.node); if (!swq) return ERR_PTR(-ENOMEM); sz = sizeof(*qp); sg_list_sz = 0; if (init_attr->srq) { struct rvt_srq *srq = ibsrq_to_rvtsrq(init_attr->srq); if (srq->rq.max_sge > 1) sg_list_sz = sizeof(*qp->r_sg_list) * (srq->rq.max_sge - 1); } else if (init_attr->cap.max_recv_sge > 1) sg_list_sz = sizeof(*qp->r_sg_list) * (init_attr->cap.max_recv_sge - 1); qp = kzalloc_node(sz + sg_list_sz, gfp, rdi->dparms.node); if (!qp) goto bail_swq; RCU_INIT_POINTER(qp->next, NULL); /* * Driver needs to set up it's private QP structure and do any * initialization that is needed. */ priv = rdi->driver_f.qp_priv_alloc(rdi, qp, gfp); if (!priv) goto bail_qp; qp->priv = priv; qp->timeout_jiffies = usecs_to_jiffies((4096UL * (1UL << qp->timeout)) / 1000UL); if (init_attr->srq) { sz = 0; } else { qp->r_rq.size = init_attr->cap.max_recv_wr + 1; qp->r_rq.max_sge = init_attr->cap.max_recv_sge; sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) + sizeof(struct rvt_rwqe); if (udata) qp->r_rq.wq = vmalloc_user( sizeof(struct rvt_rwq) + qp->r_rq.size * sz); else if (gfp == GFP_NOIO) qp->r_rq.wq = __vmalloc( sizeof(struct rvt_rwq) + qp->r_rq.size * sz, gfp, PAGE_KERNEL); else qp->r_rq.wq = vmalloc_node( sizeof(struct rvt_rwq) + qp->r_rq.size * sz, rdi->dparms.node); if (!qp->r_rq.wq) goto bail_driver_priv; } /* * ib_create_qp() will initialize qp->ibqp * except for qp->ibqp.qp_num. */ spin_lock_init(&qp->r_lock); spin_lock_init(&qp->s_hlock); spin_lock_init(&qp->s_lock); spin_lock_init(&qp->r_rq.lock); atomic_set(&qp->refcount, 0); init_waitqueue_head(&qp->wait); init_timer(&qp->s_timer); qp->s_timer.data = (unsigned long)qp; INIT_LIST_HEAD(&qp->rspwait); qp->state = IB_QPS_RESET; qp->s_wq = swq; qp->s_size = init_attr->cap.max_send_wr + 1; qp->s_avail = init_attr->cap.max_send_wr; qp->s_max_sge = init_attr->cap.max_send_sge; if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR) qp->s_flags = RVT_S_SIGNAL_REQ_WR; err = alloc_qpn(rdi, &rdi->qp_dev->qpn_table, init_attr->qp_type, init_attr->port_num, gfp); if (err < 0) { ret = ERR_PTR(err); goto bail_rq_wq; } qp->ibqp.qp_num = err; qp->port_num = init_attr->port_num; rvt_reset_qp(rdi, qp, init_attr->qp_type); break; default: /* Don't support raw QPs */ return ERR_PTR(-EINVAL); } init_attr->cap.max_inline_data = 0; /* * Return the address of the RWQ as the offset to mmap. * See rvt_mmap() for details. */ if (udata && udata->outlen >= sizeof(__u64)) { if (!qp->r_rq.wq) { __u64 offset = 0; err = ib_copy_to_udata(udata, &offset, sizeof(offset)); if (err) { ret = ERR_PTR(err); goto bail_qpn; } } else { u32 s = sizeof(struct rvt_rwq) + qp->r_rq.size * sz; qp->ip = rvt_create_mmap_info(rdi, s, ibpd->uobject->context, qp->r_rq.wq); if (!qp->ip) { ret = ERR_PTR(-ENOMEM); goto bail_qpn; } err = ib_copy_to_udata(udata, &qp->ip->offset, sizeof(qp->ip->offset)); if (err) { ret = ERR_PTR(err); goto bail_ip; } } qp->pid = current->pid; } spin_lock(&rdi->n_qps_lock); if (rdi->n_qps_allocated == rdi->dparms.props.max_qp) { spin_unlock(&rdi->n_qps_lock); ret = ERR_PTR(-ENOMEM); goto bail_ip; } rdi->n_qps_allocated++; /* * Maintain a busy_jiffies variable that will be added to the timeout * period in mod_retry_timer and add_retry_timer. This busy jiffies * is scaled by the number of rc qps created for the device to reduce * the number of timeouts occurring when there is a large number of * qps. busy_jiffies is incremented every rc qp scaling interval. * The scaling interval is selected based on extensive performance * evaluation of targeted workloads. */ if (init_attr->qp_type == IB_QPT_RC) { rdi->n_rc_qps++; rdi->busy_jiffies = rdi->n_rc_qps / RC_QP_SCALING_INTERVAL; } spin_unlock(&rdi->n_qps_lock); if (qp->ip) { spin_lock_irq(&rdi->pending_lock); list_add(&qp->ip->pending_mmaps, &rdi->pending_mmaps); spin_unlock_irq(&rdi->pending_lock); } ret = &qp->ibqp; /* * We have our QP and its good, now keep track of what types of opcodes * can be processed on this QP. We do this by keeping track of what the * 3 high order bits of the opcode are. */ switch (init_attr->qp_type) { case IB_QPT_SMI: case IB_QPT_GSI: case IB_QPT_UD: qp->allowed_ops = IB_OPCODE_UD_SEND_ONLY & RVT_OPCODE_QP_MASK; break; case IB_QPT_RC: qp->allowed_ops = IB_OPCODE_RC_SEND_ONLY & RVT_OPCODE_QP_MASK; break; case IB_QPT_UC: qp->allowed_ops = IB_OPCODE_UC_SEND_ONLY & RVT_OPCODE_QP_MASK; break; default: ret = ERR_PTR(-EINVAL); goto bail_ip; } return ret; bail_ip: kref_put(&qp->ip->ref, rvt_release_mmap_info); bail_qpn: free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num); bail_rq_wq: vfree(qp->r_rq.wq); bail_driver_priv: rdi->driver_f.qp_priv_free(rdi, qp); bail_qp: kfree(qp); bail_swq: vfree(swq); return ret; }
/** * rvt_post_one_wr - post one RC, UC, or UD send work request * @qp: the QP to post on * @wr: the work request to send */ static int rvt_post_one_wr(struct rvt_qp *qp, struct ib_send_wr *wr, int *call_send) { struct rvt_swqe *wqe; u32 next; int i; int j; int acc; struct rvt_lkey_table *rkt; struct rvt_pd *pd; struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); u8 log_pmtu; int ret; /* IB spec says that num_sge == 0 is OK. */ if (unlikely(wr->num_sge > qp->s_max_sge)) return -EINVAL; /* * Don't allow RDMA reads or atomic operations on UC or * undefined operations. * Make sure buffer is large enough to hold the result for atomics. */ if (qp->ibqp.qp_type == IB_QPT_UC) { if ((unsigned)wr->opcode >= IB_WR_RDMA_READ) return -EINVAL; } else if (qp->ibqp.qp_type != IB_QPT_RC) { /* Check IB_QPT_SMI, IB_QPT_GSI, IB_QPT_UD opcode */ if (wr->opcode != IB_WR_SEND && wr->opcode != IB_WR_SEND_WITH_IMM) return -EINVAL; /* Check UD destination address PD */ if (qp->ibqp.pd != ud_wr(wr)->ah->pd) return -EINVAL; } else if ((unsigned)wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD) { return -EINVAL; } else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP && (wr->num_sge == 0 || wr->sg_list[0].length < sizeof(u64) || wr->sg_list[0].addr & (sizeof(u64) - 1))) { return -EINVAL; } else if (wr->opcode >= IB_WR_RDMA_READ && !qp->s_max_rd_atomic) { return -EINVAL; } /* check for avail */ if (unlikely(!qp->s_avail)) { qp->s_avail = qp_get_savail(qp); if (WARN_ON(qp->s_avail > (qp->s_size - 1))) rvt_pr_err(rdi, "More avail entries than QP RB size.\nQP: %u, size: %u, avail: %u\nhead: %u, tail: %u, cur: %u, acked: %u, last: %u", qp->ibqp.qp_num, qp->s_size, qp->s_avail, qp->s_head, qp->s_tail, qp->s_cur, qp->s_acked, qp->s_last); if (!qp->s_avail) return -ENOMEM; } next = qp->s_head + 1; if (next >= qp->s_size) next = 0; rkt = &rdi->lkey_table; pd = ibpd_to_rvtpd(qp->ibqp.pd); wqe = rvt_get_swqe_ptr(qp, qp->s_head); if (qp->ibqp.qp_type != IB_QPT_UC && qp->ibqp.qp_type != IB_QPT_RC) memcpy(&wqe->ud_wr, ud_wr(wr), sizeof(wqe->ud_wr)); else if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || wr->opcode == IB_WR_RDMA_WRITE || wr->opcode == IB_WR_RDMA_READ) memcpy(&wqe->rdma_wr, rdma_wr(wr), sizeof(wqe->rdma_wr)); else if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) memcpy(&wqe->atomic_wr, atomic_wr(wr), sizeof(wqe->atomic_wr)); else memcpy(&wqe->wr, wr, sizeof(wqe->wr)); wqe->length = 0; j = 0; if (wr->num_sge) { acc = wr->opcode >= IB_WR_RDMA_READ ? IB_ACCESS_LOCAL_WRITE : 0; for (i = 0; i < wr->num_sge; i++) { u32 length = wr->sg_list[i].length; int ok; if (length == 0) continue; ok = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j], &wr->sg_list[i], acc); if (!ok) { ret = -EINVAL; goto bail_inval_free; } wqe->length += length; j++; } wqe->wr.num_sge = j; } /* general part of wqe valid - allow for driver checks */ if (rdi->driver_f.check_send_wqe) { ret = rdi->driver_f.check_send_wqe(qp, wqe); if (ret < 0) goto bail_inval_free; if (ret) *call_send = ret; } log_pmtu = qp->log_pmtu; if (qp->ibqp.qp_type != IB_QPT_UC && qp->ibqp.qp_type != IB_QPT_RC) { struct rvt_ah *ah = ibah_to_rvtah(wqe->ud_wr.ah); log_pmtu = ah->log_pmtu; atomic_inc(&ibah_to_rvtah(ud_wr(wr)->ah)->refcount); } wqe->ssn = qp->s_ssn++; wqe->psn = qp->s_next_psn; wqe->lpsn = wqe->psn + (wqe->length ? ((wqe->length - 1) >> log_pmtu) : 0); qp->s_next_psn = wqe->lpsn + 1; trace_rvt_post_one_wr(qp, wqe); smp_wmb(); /* see request builders */ qp->s_avail--; qp->s_head = next; return 0; bail_inval_free: /* release mr holds */ while (j) { struct rvt_sge *sge = &wqe->sg_list[--j]; rvt_put_mr(sge->mr); } return ret; }