/** * ipath_destroy_srq - destroy a shared receive queue * @ibsrq: the SRQ to destroy */ int ipath_destroy_srq(struct ib_srq *ibsrq) { struct ipath_srq *srq = to_isrq(ibsrq); vfree(srq->rq.wq); kfree(srq); return 0; }
int qib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) { struct qib_srq *srq = to_isrq(ibsrq); attr->max_wr = srq->rq.size - 1; attr->max_sge = srq->rq.max_sge; attr->srq_limit = srq->limit; return 0; }
/** * ipath_destroy_srq - destroy a shared receive queue * @ibsrq: the SRQ to destroy */ int ipath_destroy_srq(struct ib_srq *ibsrq) { struct ipath_srq *srq = to_isrq(ibsrq); struct ipath_ibdev *dev = to_idev(ibsrq->device); dev->n_srqs_allocated--; vfree(srq->rq.wq); kfree(srq); return 0; }
/** * qib_destroy_srq - destroy a shared receive queue * @ibsrq: the SRQ to destroy */ int qib_destroy_srq(struct ib_srq *ibsrq) { struct qib_srq *srq = to_isrq(ibsrq); struct qib_ibdev *dev = to_idev(ibsrq->device); spin_lock(&dev->n_srqs_lock); dev->n_srqs_allocated--; spin_unlock(&dev->n_srqs_lock); if (srq->ip) kref_put(&srq->ip->ref, qib_release_mmap_info); else vfree(srq->rq.wq); kfree(srq); return 0; }
/** * qib_post_srq_receive - post a receive on a shared receive queue * @ibsrq: the SRQ to post the receive on * @wr: the list of work requests to post * @bad_wr: A pointer to the first WR to cause a problem is put here * * This may be called from interrupt context. */ int qib_post_srq_receive(struct ib_srq *ibsrq, struct ib_recv_wr *wr, struct ib_recv_wr **bad_wr) { struct qib_srq *srq = to_isrq(ibsrq); struct qib_rwq *wq; unsigned long flags; int ret; for (; wr; wr = wr->next) { struct qib_rwqe *wqe; u32 next; int i; if ((unsigned) wr->num_sge > srq->rq.max_sge) { *bad_wr = wr; ret = -EINVAL; goto bail; } spin_lock_irqsave(&srq->rq.lock, flags); wq = srq->rq.wq; next = wq->head + 1; if (next >= srq->rq.size) next = 0; if (next == wq->tail) { spin_unlock_irqrestore(&srq->rq.lock, flags); *bad_wr = wr; ret = -ENOMEM; goto bail; } wqe = get_rwqe_ptr(&srq->rq, wq->head); wqe->wr_id = wr->wr_id; wqe->num_sge = wr->num_sge; for (i = 0; i < wr->num_sge; i++) wqe->sg_list[i] = wr->sg_list[i]; /* Make sure queue entry is written before the head index. */ smp_wmb(); wq->head = next; spin_unlock_irqrestore(&srq->rq.lock, flags); } ret = 0; bail: return ret; }
/** * qib_modify_srq - modify a shared receive queue * @ibsrq: the SRQ to modify * @attr: the new attributes of the SRQ * @attr_mask: indicates which attributes to modify * @udata: user data for libibverbs.so */ int qib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) { struct qib_srq *srq = to_isrq(ibsrq); struct qib_rwq *wq; int ret = 0; if (attr_mask & IB_SRQ_MAX_WR) { struct qib_rwq *owq; struct qib_rwqe *p; u32 sz, size, n, head, tail; /* Check that the requested sizes are below the limits. */ if ((attr->max_wr > ib_qib_max_srq_wrs) || ((attr_mask & IB_SRQ_LIMIT) ? attr->srq_limit : srq->limit) > attr->max_wr) { ret = -EINVAL; goto bail; } sz = sizeof(struct qib_rwqe) + srq->rq.max_sge * sizeof(struct ib_sge); size = attr->max_wr + 1; wq = vmalloc_user(sizeof(struct qib_rwq) + size * sz); if (!wq) { ret = -ENOMEM; goto bail; } /* Check that we can write the offset to mmap. */ if (udata && udata->inlen >= sizeof(__u64)) { __u64 offset_addr; __u64 offset = 0; ret = ib_copy_from_udata(&offset_addr, udata, sizeof(offset_addr)); if (ret) goto bail_free; udata->outbuf = (void __user *) (unsigned long) offset_addr; ret = ib_copy_to_udata(udata, &offset, sizeof(offset)); if (ret) goto bail_free; } spin_lock_irq(&srq->rq.lock); /* * validate head and tail pointer values and compute * the number of remaining WQEs. */ owq = srq->rq.wq; head = owq->head; tail = owq->tail; if (head >= srq->rq.size || tail >= srq->rq.size) { ret = -EINVAL; goto bail_unlock; } n = head; if (n < tail) n += srq->rq.size - tail; else n -= tail; if (size <= n) { ret = -EINVAL; goto bail_unlock; } n = 0; p = wq->wq; while (tail != head) { struct qib_rwqe *wqe; int i; wqe = get_rwqe_ptr(&srq->rq, tail); p->wr_id = wqe->wr_id; p->num_sge = wqe->num_sge; for (i = 0; i < wqe->num_sge; i++) p->sg_list[i] = wqe->sg_list[i]; n++; p = (struct qib_rwqe *)((char *) p + sz); if (++tail >= srq->rq.size) tail = 0; } srq->rq.wq = wq; srq->rq.size = size; wq->head = n; wq->tail = 0; if (attr_mask & IB_SRQ_LIMIT) srq->limit = attr->srq_limit; spin_unlock_irq(&srq->rq.lock); vfree(owq); if (srq->ip) { struct qib_mmap_info *ip = srq->ip; struct qib_ibdev *dev = to_idev(srq->ibsrq.device); u32 s = sizeof(struct qib_rwq) + size * sz; qib_update_mmap_info(dev, ip, s, wq); /* * Return the offset to mmap. * See qib_mmap() for details. */ if (udata && udata->inlen >= sizeof(__u64)) { ret = ib_copy_to_udata(udata, &ip->offset, sizeof(ip->offset)); if (ret) goto bail; } /* * Put user mapping info onto the pending list * unless it already is on the list. */ spin_lock_irq(&dev->pending_lock); if (list_empty(&ip->pending_mmaps)) list_add(&ip->pending_mmaps, &dev->pending_mmaps); spin_unlock_irq(&dev->pending_lock); } } else if (attr_mask & IB_SRQ_LIMIT) { spin_lock_irq(&srq->rq.lock); if (attr->srq_limit >= srq->rq.size) ret = -EINVAL; else srq->limit = attr->srq_limit; spin_unlock_irq(&srq->rq.lock); } goto bail; bail_unlock: spin_unlock_irq(&srq->rq.lock); bail_free: vfree(wq); bail: return ret; }
struct ib_qp *ipath_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { struct ipath_qp *qp; int err; struct ipath_swqe *swq = NULL; struct ipath_ibdev *dev; size_t sz; size_t sg_list_sz; struct ib_qp *ret; if (init_attr->create_flags) { ret = ERR_PTR(-EINVAL); goto bail; } if (init_attr->cap.max_send_sge > ib_ipath_max_sges || init_attr->cap.max_send_wr > ib_ipath_max_qp_wrs) { ret = ERR_PTR(-EINVAL); goto bail; } /* */ if (!init_attr->srq) { if (init_attr->cap.max_recv_sge > ib_ipath_max_sges || init_attr->cap.max_recv_wr > ib_ipath_max_qp_wrs) { ret = ERR_PTR(-EINVAL); goto bail; } if (init_attr->cap.max_send_sge + init_attr->cap.max_send_wr + init_attr->cap.max_recv_sge + init_attr->cap.max_recv_wr == 0) { ret = ERR_PTR(-EINVAL); goto bail; } } switch (init_attr->qp_type) { case IB_QPT_UC: case IB_QPT_RC: case IB_QPT_UD: case IB_QPT_SMI: case IB_QPT_GSI: sz = sizeof(struct ipath_sge) * init_attr->cap.max_send_sge + sizeof(struct ipath_swqe); swq = vmalloc((init_attr->cap.max_send_wr + 1) * sz); if (swq == NULL) { ret = ERR_PTR(-ENOMEM); goto bail; } sz = sizeof(*qp); sg_list_sz = 0; if (init_attr->srq) { struct ipath_srq *srq = to_isrq(init_attr->srq); if (srq->rq.max_sge > 1) sg_list_sz = sizeof(*qp->r_sg_list) * (srq->rq.max_sge - 1); } else if (init_attr->cap.max_recv_sge > 1) sg_list_sz = sizeof(*qp->r_sg_list) * (init_attr->cap.max_recv_sge - 1); qp = kmalloc(sz + sg_list_sz, GFP_KERNEL); if (!qp) { ret = ERR_PTR(-ENOMEM); goto bail_swq; } if (sg_list_sz && (init_attr->qp_type == IB_QPT_UD || init_attr->qp_type == IB_QPT_SMI || init_attr->qp_type == IB_QPT_GSI)) { qp->r_ud_sg_list = kmalloc(sg_list_sz, GFP_KERNEL); if (!qp->r_ud_sg_list) { ret = ERR_PTR(-ENOMEM); goto bail_qp; } } else qp->r_ud_sg_list = NULL; if (init_attr->srq) { sz = 0; qp->r_rq.size = 0; qp->r_rq.max_sge = 0; qp->r_rq.wq = NULL; init_attr->cap.max_recv_wr = 0; init_attr->cap.max_recv_sge = 0; } else { qp->r_rq.size = init_attr->cap.max_recv_wr + 1; qp->r_rq.max_sge = init_attr->cap.max_recv_sge; sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) + sizeof(struct ipath_rwqe); qp->r_rq.wq = vmalloc_user(sizeof(struct ipath_rwq) + qp->r_rq.size * sz); if (!qp->r_rq.wq) { ret = ERR_PTR(-ENOMEM); goto bail_sg_list; } } /* */ spin_lock_init(&qp->s_lock); spin_lock_init(&qp->r_rq.lock); atomic_set(&qp->refcount, 0); init_waitqueue_head(&qp->wait); init_waitqueue_head(&qp->wait_dma); tasklet_init(&qp->s_task, ipath_do_send, (unsigned long)qp); INIT_LIST_HEAD(&qp->piowait); INIT_LIST_HEAD(&qp->timerwait); qp->state = IB_QPS_RESET; qp->s_wq = swq; qp->s_size = init_attr->cap.max_send_wr + 1; qp->s_max_sge = init_attr->cap.max_send_sge; if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR) qp->s_flags = IPATH_S_SIGNAL_REQ_WR; else qp->s_flags = 0; dev = to_idev(ibpd->device); err = ipath_alloc_qpn(&dev->qp_table, qp, init_attr->qp_type); if (err) { ret = ERR_PTR(err); vfree(qp->r_rq.wq); goto bail_sg_list; } qp->ip = NULL; qp->s_tx = NULL; ipath_reset_qp(qp, init_attr->qp_type); break; default: /* */ ret = ERR_PTR(-ENOSYS); goto bail; } init_attr->cap.max_inline_data = 0; /* */ if (udata && udata->outlen >= sizeof(__u64)) { if (!qp->r_rq.wq) { __u64 offset = 0; err = ib_copy_to_udata(udata, &offset, sizeof(offset)); if (err) { ret = ERR_PTR(err); goto bail_ip; } } else { u32 s = sizeof(struct ipath_rwq) + qp->r_rq.size * sz; qp->ip = ipath_create_mmap_info(dev, s, ibpd->uobject->context, qp->r_rq.wq); if (!qp->ip) { ret = ERR_PTR(-ENOMEM); goto bail_ip; } err = ib_copy_to_udata(udata, &(qp->ip->offset), sizeof(qp->ip->offset)); if (err) { ret = ERR_PTR(err); goto bail_ip; } } } spin_lock(&dev->n_qps_lock); if (dev->n_qps_allocated == ib_ipath_max_qps) { spin_unlock(&dev->n_qps_lock); ret = ERR_PTR(-ENOMEM); goto bail_ip; } dev->n_qps_allocated++; spin_unlock(&dev->n_qps_lock); if (qp->ip) { spin_lock_irq(&dev->pending_lock); list_add(&qp->ip->pending_mmaps, &dev->pending_mmaps); spin_unlock_irq(&dev->pending_lock); } ret = &qp->ibqp; goto bail; bail_ip: if (qp->ip) kref_put(&qp->ip->ref, ipath_release_mmap_info); else vfree(qp->r_rq.wq); ipath_free_qp(&dev->qp_table, qp); free_qpn(&dev->qp_table, qp->ibqp.qp_num); bail_sg_list: kfree(qp->r_ud_sg_list); bail_qp: kfree(qp); bail_swq: vfree(swq); bail: return ret; }
/** * qib_get_rwqe - copy the next RWQE into the QP's RWQE * @qp: the QP * @wr_id_only: update qp->r_wr_id only, not qp->r_sge * * Return -1 if there is a local error, 0 if no RWQE is available, * otherwise return 1. * * Can be called from interrupt level. */ int qib_get_rwqe(struct qib_qp *qp, int wr_id_only) { unsigned long flags; struct qib_rq *rq; struct qib_rwq *wq; struct qib_srq *srq; struct qib_rwqe *wqe; void (*handler)(struct ib_event *, void *); u32 tail; int ret; if (qp->ibqp.srq) { srq = to_isrq(qp->ibqp.srq); handler = srq->ibsrq.event_handler; rq = &srq->rq; } else { srq = NULL; handler = NULL; rq = &qp->r_rq; } spin_lock_irqsave(&rq->lock, flags); if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) { ret = 0; goto unlock; } wq = rq->wq; tail = wq->tail; /* Validate tail before using it since it is user writable. */ if (tail >= rq->size) tail = 0; if (unlikely(tail == wq->head)) { ret = 0; goto unlock; } /* Make sure entry is read after head index is read. */ smp_rmb(); wqe = get_rwqe_ptr(rq, tail); /* * Even though we update the tail index in memory, the verbs * consumer is not supposed to post more entries until a * completion is generated. */ if (++tail >= rq->size) tail = 0; wq->tail = tail; if (!wr_id_only && !qib_init_sge(qp, wqe)) { ret = -1; goto unlock; } qp->r_wr_id = wqe->wr_id; ret = 1; set_bit(QIB_R_WRID_VALID, &qp->r_aflags); if (handler) { u32 n; /* * Validate head pointer value and compute * the number of remaining WQEs. */ n = wq->head; if (n >= rq->size) n = 0; if (n < tail) n += rq->size - tail; else n -= tail; if (n < srq->limit) { struct ib_event ev; srq->limit = 0; spin_unlock_irqrestore(&rq->lock, flags); ev.device = qp->ibqp.device; ev.element.srq = qp->ibqp.srq; ev.event = IB_EVENT_SRQ_LIMIT_REACHED; handler(&ev, srq->ibsrq.srq_context); goto bail; } } unlock: spin_unlock_irqrestore(&rq->lock, flags); bail: return ret; }
/** * ipath_get_rwqe - copy the next RWQE into the QP's RWQE * @qp: the QP * @wr_id_only: update wr_id only, not SGEs * * Return 0 if no RWQE is available, otherwise return 1. * * Can be called from interrupt level. */ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only) { unsigned long flags; struct ipath_rq *rq; struct ipath_rwq *wq; struct ipath_srq *srq; struct ipath_rwqe *wqe; void (*handler)(struct ib_event *, void *); u32 tail; int ret; qp->r_sge.sg_list = qp->r_sg_list; if (qp->ibqp.srq) { srq = to_isrq(qp->ibqp.srq); handler = srq->ibsrq.event_handler; rq = &srq->rq; } else { srq = NULL; handler = NULL; rq = &qp->r_rq; } spin_lock_irqsave(&rq->lock, flags); if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) { ret = 0; goto unlock; } wq = rq->wq; tail = wq->tail; /* Validate tail before using it since it is user writable. */ if (tail >= rq->size) tail = 0; do { if (unlikely(tail == wq->head)) { ret = 0; goto unlock; } /* Make sure entry is read after head index is read. */ smp_rmb(); wqe = get_rwqe_ptr(rq, tail); if (++tail >= rq->size) tail = 0; } while (!wr_id_only && !ipath_init_sge(qp, wqe, &qp->r_len, &qp->r_sge)); qp->r_wr_id = wqe->wr_id; wq->tail = tail; ret = 1; set_bit(IPATH_R_WRID_VALID, &qp->r_aflags); if (handler) { u32 n; /* * validate head pointer value and compute * the number of remaining WQEs. */ n = wq->head; if (n >= rq->size) n = 0; if (n < tail) n += rq->size - tail; else n -= tail; if (n < srq->limit) { struct ib_event ev; srq->limit = 0; spin_unlock_irqrestore(&rq->lock, flags); ev.device = qp->ibqp.device; ev.element.srq = qp->ibqp.srq; ev.event = IB_EVENT_SRQ_LIMIT_REACHED; handler(&ev, srq->ibsrq.srq_context); goto bail; } } unlock: spin_unlock_irqrestore(&rq->lock, flags); bail: return ret; }
/** * ipath_post_srq_receive - post a receive on a shared receive queue * @ibsrq: the SRQ to post the receive on * @wr: the list of work requests to post * @bad_wr: the first WR to cause a problem is put here * * This may be called from interrupt context. */ int ipath_post_srq_receive(struct ib_srq *ibsrq, struct ib_recv_wr *wr, struct ib_recv_wr **bad_wr) { struct ipath_srq *srq = to_isrq(ibsrq); struct ipath_ibdev *dev = to_idev(ibsrq->device); unsigned long flags; int ret; for (; wr; wr = wr->next) { struct ipath_rwqe *wqe; u32 next; int i, j; if (wr->num_sge > srq->rq.max_sge) { *bad_wr = wr; ret = -ENOMEM; goto bail; } spin_lock_irqsave(&srq->rq.lock, flags); next = srq->rq.head + 1; if (next >= srq->rq.size) next = 0; if (next == srq->rq.tail) { spin_unlock_irqrestore(&srq->rq.lock, flags); *bad_wr = wr; ret = -ENOMEM; goto bail; } wqe = get_rwqe_ptr(&srq->rq, srq->rq.head); wqe->wr_id = wr->wr_id; wqe->sg_list[0].mr = NULL; wqe->sg_list[0].vaddr = NULL; wqe->sg_list[0].length = 0; wqe->sg_list[0].sge_length = 0; wqe->length = 0; for (i = 0, j = 0; i < wr->num_sge; i++) { /* Check LKEY */ if (to_ipd(srq->ibsrq.pd)->user && wr->sg_list[i].lkey == 0) { spin_unlock_irqrestore(&srq->rq.lock, flags); *bad_wr = wr; ret = -EINVAL; goto bail; } if (wr->sg_list[i].length == 0) continue; if (!ipath_lkey_ok(&dev->lk_table, &wqe->sg_list[j], &wr->sg_list[i], IB_ACCESS_LOCAL_WRITE)) { spin_unlock_irqrestore(&srq->rq.lock, flags); *bad_wr = wr; ret = -EINVAL; goto bail; } wqe->length += wr->sg_list[i].length; j++; } wqe->num_sge = j; srq->rq.head = next; spin_unlock_irqrestore(&srq->rq.lock, flags); } ret = 0; bail: return ret; }
/** * ipath_modify_srq - modify a shared receive queue * @ibsrq: the SRQ to modify * @attr: the new attributes of the SRQ * @attr_mask: indicates which attributes to modify */ int ipath_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask) { struct ipath_srq *srq = to_isrq(ibsrq); unsigned long flags; int ret; if (attr_mask & IB_SRQ_MAX_WR) if ((attr->max_wr > ib_ipath_max_srq_wrs) || (attr->max_sge > srq->rq.max_sge)) { ret = -EINVAL; goto bail; } if (attr_mask & IB_SRQ_LIMIT) if (attr->srq_limit >= srq->rq.size) { ret = -EINVAL; goto bail; } if (attr_mask & IB_SRQ_MAX_WR) { struct ipath_rwqe *wq, *p; u32 sz, size, n; sz = sizeof(struct ipath_rwqe) + attr->max_sge * sizeof(struct ipath_sge); size = attr->max_wr + 1; wq = vmalloc(size * sz); if (!wq) { ret = -ENOMEM; goto bail; } spin_lock_irqsave(&srq->rq.lock, flags); if (srq->rq.head < srq->rq.tail) n = srq->rq.size + srq->rq.head - srq->rq.tail; else n = srq->rq.head - srq->rq.tail; if (size <= n || size <= srq->limit) { spin_unlock_irqrestore(&srq->rq.lock, flags); vfree(wq); ret = -EINVAL; goto bail; } n = 0; p = wq; while (srq->rq.tail != srq->rq.head) { struct ipath_rwqe *wqe; int i; wqe = get_rwqe_ptr(&srq->rq, srq->rq.tail); p->wr_id = wqe->wr_id; p->length = wqe->length; p->num_sge = wqe->num_sge; for (i = 0; i < wqe->num_sge; i++) p->sg_list[i] = wqe->sg_list[i]; n++; p = (struct ipath_rwqe *)((char *) p + sz); if (++srq->rq.tail >= srq->rq.size) srq->rq.tail = 0; } vfree(srq->rq.wq); srq->rq.wq = wq; srq->rq.size = size; srq->rq.head = n; srq->rq.tail = 0; srq->rq.max_sge = attr->max_sge; spin_unlock_irqrestore(&srq->rq.lock, flags); } if (attr_mask & IB_SRQ_LIMIT) { spin_lock_irqsave(&srq->rq.lock, flags); srq->limit = attr->srq_limit; spin_unlock_irqrestore(&srq->rq.lock, flags); } ret = 0; bail: return ret; }
/** * ipath_get_rwqe - copy the next RWQE into the QP's RWQE * @qp: the QP * @wr_id_only: update wr_id only, not SGEs * * Return 0 if no RWQE is available, otherwise return 1. * * Called at interrupt level with the QP r_rq.lock held. */ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only) { struct ipath_rq *rq; struct ipath_srq *srq; struct ipath_rwqe *wqe; int ret; if (!qp->ibqp.srq) { rq = &qp->r_rq; if (unlikely(rq->tail == rq->head)) { ret = 0; goto bail; } wqe = get_rwqe_ptr(rq, rq->tail); qp->r_wr_id = wqe->wr_id; if (!wr_id_only) { qp->r_sge.sge = wqe->sg_list[0]; qp->r_sge.sg_list = wqe->sg_list + 1; qp->r_sge.num_sge = wqe->num_sge; qp->r_len = wqe->length; } if (++rq->tail >= rq->size) rq->tail = 0; ret = 1; goto bail; } srq = to_isrq(qp->ibqp.srq); rq = &srq->rq; spin_lock(&rq->lock); if (unlikely(rq->tail == rq->head)) { spin_unlock(&rq->lock); ret = 0; goto bail; } wqe = get_rwqe_ptr(rq, rq->tail); qp->r_wr_id = wqe->wr_id; if (!wr_id_only) { qp->r_sge.sge = wqe->sg_list[0]; qp->r_sge.sg_list = wqe->sg_list + 1; qp->r_sge.num_sge = wqe->num_sge; qp->r_len = wqe->length; } if (++rq->tail >= rq->size) rq->tail = 0; if (srq->ibsrq.event_handler) { struct ib_event ev; u32 n; if (rq->head < rq->tail) n = rq->size + rq->head - rq->tail; else n = rq->head - rq->tail; if (n < srq->limit) { srq->limit = 0; spin_unlock(&rq->lock); ev.device = qp->ibqp.device; ev.element.srq = qp->ibqp.srq; ev.event = IB_EVENT_SRQ_LIMIT_REACHED; srq->ibsrq.event_handler(&ev, srq->ibsrq.srq_context); } else spin_unlock(&rq->lock); } else spin_unlock(&rq->lock); ret = 1; bail: return ret; }
int qib_get_rwqe(struct qib_qp *qp, int wr_id_only) { unsigned long flags; struct qib_rq *rq; struct qib_rwq *wq; struct qib_srq *srq; struct qib_rwqe *wqe; void (*handler)(struct ib_event *, void *); u32 tail; int ret; if (qp->ibqp.srq) { srq = to_isrq(qp->ibqp.srq); handler = srq->ibsrq.event_handler; rq = &srq->rq; } else { srq = NULL; handler = NULL; rq = &qp->r_rq; } spin_lock_irqsave(&rq->lock, flags); if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) { ret = 0; goto unlock; } wq = rq->wq; tail = wq->tail; if (tail >= rq->size) tail = 0; if (unlikely(tail == wq->head)) { ret = 0; goto unlock; } smp_rmb(); wqe = get_rwqe_ptr(rq, tail); if (++tail >= rq->size) tail = 0; wq->tail = tail; if (!wr_id_only && !qib_init_sge(qp, wqe)) { ret = -1; goto unlock; } qp->r_wr_id = wqe->wr_id; ret = 1; set_bit(QIB_R_WRID_VALID, &qp->r_aflags); if (handler) { u32 n; n = wq->head; if (n >= rq->size) n = 0; if (n < tail) n += rq->size - tail; else n -= tail; if (n < srq->limit) { struct ib_event ev; srq->limit = 0; spin_unlock_irqrestore(&rq->lock, flags); ev.device = qp->ibqp.device; ev.element.srq = qp->ibqp.srq; ev.event = IB_EVENT_SRQ_LIMIT_REACHED; handler(&ev, srq->ibsrq.srq_context); goto bail; } } unlock: spin_unlock_irqrestore(&rq->lock, flags); bail: return ret; }
/** * ipath_get_rwqe - copy the next RWQE into the QP's RWQE * @qp: the QP * @wr_id_only: update wr_id only, not SGEs * * Return 0 if no RWQE is available, otherwise return 1. * * Can be called from interrupt level. */ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only) { unsigned long flags; struct ipath_rq *rq; struct ipath_rwq *wq; struct ipath_srq *srq; struct ipath_rwqe *wqe; void (*handler)(struct ib_event *, void *); u32 tail; int ret; if (qp->ibqp.srq) { srq = to_isrq(qp->ibqp.srq); handler = srq->ibsrq.event_handler; rq = &srq->rq; } else { srq = NULL; handler = NULL; rq = &qp->r_rq; } spin_lock_irqsave(&rq->lock, flags); wq = rq->wq; tail = wq->tail; /* Validate tail before using it since it is user writable. */ if (tail >= rq->size) tail = 0; do { if (unlikely(tail == wq->head)) { spin_unlock_irqrestore(&rq->lock, flags); ret = 0; goto bail; } wqe = get_rwqe_ptr(rq, tail); if (++tail >= rq->size) tail = 0; } while (!wr_id_only && !init_sge(qp, wqe)); qp->r_wr_id = wqe->wr_id; wq->tail = tail; ret = 1; qp->r_wrid_valid = 1; if (handler) { u32 n; /* * validate head pointer value and compute * the number of remaining WQEs. */ n = wq->head; if (n >= rq->size) n = 0; if (n < tail) n += rq->size - tail; else n -= tail; if (n < srq->limit) { struct ib_event ev; srq->limit = 0; spin_unlock_irqrestore(&rq->lock, flags); ev.device = qp->ibqp.device; ev.element.srq = qp->ibqp.srq; ev.event = IB_EVENT_SRQ_LIMIT_REACHED; handler(&ev, srq->ibsrq.srq_context); goto bail; } } spin_unlock_irqrestore(&rq->lock, flags); bail: return ret; }
/** * hfi1_create_qp - create a queue pair for a device * @ibpd: the protection domain who's device we create the queue pair for * @init_attr: the attributes of the queue pair * @udata: user data for libibverbs.so * * Returns the queue pair on success, otherwise returns an errno. * * Called by the ib_create_qp() core verbs function. */ struct ib_qp *hfi1_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { struct hfi1_qp *qp; int err; struct hfi1_swqe *swq = NULL; struct hfi1_ibdev *dev; struct hfi1_devdata *dd; size_t sz; size_t sg_list_sz; struct ib_qp *ret; if (init_attr->cap.max_send_sge > hfi1_max_sges || init_attr->cap.max_send_wr > hfi1_max_qp_wrs || init_attr->create_flags) { ret = ERR_PTR(-EINVAL); goto bail; } /* Check receive queue parameters if no SRQ is specified. */ if (!init_attr->srq) { if (init_attr->cap.max_recv_sge > hfi1_max_sges || init_attr->cap.max_recv_wr > hfi1_max_qp_wrs) { ret = ERR_PTR(-EINVAL); goto bail; } if (init_attr->cap.max_send_sge + init_attr->cap.max_send_wr + init_attr->cap.max_recv_sge + init_attr->cap.max_recv_wr == 0) { ret = ERR_PTR(-EINVAL); goto bail; } } switch (init_attr->qp_type) { case IB_QPT_SMI: case IB_QPT_GSI: if (init_attr->port_num == 0 || init_attr->port_num > ibpd->device->phys_port_cnt) { ret = ERR_PTR(-EINVAL); goto bail; } case IB_QPT_UC: case IB_QPT_RC: case IB_QPT_UD: sz = sizeof(struct hfi1_sge) * init_attr->cap.max_send_sge + sizeof(struct hfi1_swqe); swq = vmalloc((init_attr->cap.max_send_wr + 1) * sz); if (swq == NULL) { ret = ERR_PTR(-ENOMEM); goto bail; } sz = sizeof(*qp); sg_list_sz = 0; if (init_attr->srq) { struct hfi1_srq *srq = to_isrq(init_attr->srq); if (srq->rq.max_sge > 1) sg_list_sz = sizeof(*qp->r_sg_list) * (srq->rq.max_sge - 1); } else if (init_attr->cap.max_recv_sge > 1) sg_list_sz = sizeof(*qp->r_sg_list) * (init_attr->cap.max_recv_sge - 1); qp = kzalloc(sz + sg_list_sz, GFP_KERNEL); if (!qp) { ret = ERR_PTR(-ENOMEM); goto bail_swq; } RCU_INIT_POINTER(qp->next, NULL); qp->s_hdr = kzalloc(sizeof(*qp->s_hdr), GFP_KERNEL); if (!qp->s_hdr) { ret = ERR_PTR(-ENOMEM); goto bail_qp; } qp->timeout_jiffies = usecs_to_jiffies((4096UL * (1UL << qp->timeout)) / 1000UL); if (init_attr->srq) sz = 0; else { qp->r_rq.size = init_attr->cap.max_recv_wr + 1; qp->r_rq.max_sge = init_attr->cap.max_recv_sge; sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) + sizeof(struct hfi1_rwqe); qp->r_rq.wq = vmalloc_user(sizeof(struct hfi1_rwq) + qp->r_rq.size * sz); if (!qp->r_rq.wq) { ret = ERR_PTR(-ENOMEM); goto bail_qp; } } /* * ib_create_qp() will initialize qp->ibqp * except for qp->ibqp.qp_num. */ spin_lock_init(&qp->r_lock); spin_lock_init(&qp->s_lock); spin_lock_init(&qp->r_rq.lock); atomic_set(&qp->refcount, 0); init_waitqueue_head(&qp->wait); init_timer(&qp->s_timer); qp->s_timer.data = (unsigned long)qp; INIT_LIST_HEAD(&qp->rspwait); qp->state = IB_QPS_RESET; qp->s_wq = swq; qp->s_size = init_attr->cap.max_send_wr + 1; qp->s_max_sge = init_attr->cap.max_send_sge; if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR) qp->s_flags = HFI1_S_SIGNAL_REQ_WR; dev = to_idev(ibpd->device); dd = dd_from_dev(dev); err = alloc_qpn(dd, &dev->qp_dev->qpn_table, init_attr->qp_type, init_attr->port_num); if (err < 0) { ret = ERR_PTR(err); vfree(qp->r_rq.wq); goto bail_qp; } qp->ibqp.qp_num = err; qp->port_num = init_attr->port_num; reset_qp(qp, init_attr->qp_type); break; default: /* Don't support raw QPs */ ret = ERR_PTR(-ENOSYS); goto bail; } init_attr->cap.max_inline_data = 0; /* * Return the address of the RWQ as the offset to mmap. * See hfi1_mmap() for details. */ if (udata && udata->outlen >= sizeof(__u64)) { if (!qp->r_rq.wq) { __u64 offset = 0; err = ib_copy_to_udata(udata, &offset, sizeof(offset)); if (err) { ret = ERR_PTR(err); goto bail_ip; } } else { u32 s = sizeof(struct hfi1_rwq) + qp->r_rq.size * sz; qp->ip = hfi1_create_mmap_info(dev, s, ibpd->uobject->context, qp->r_rq.wq); if (!qp->ip) { ret = ERR_PTR(-ENOMEM); goto bail_ip; } err = ib_copy_to_udata(udata, &(qp->ip->offset), sizeof(qp->ip->offset)); if (err) { ret = ERR_PTR(err); goto bail_ip; } } } spin_lock(&dev->n_qps_lock); if (dev->n_qps_allocated == hfi1_max_qps) { spin_unlock(&dev->n_qps_lock); ret = ERR_PTR(-ENOMEM); goto bail_ip; } dev->n_qps_allocated++; spin_unlock(&dev->n_qps_lock); if (qp->ip) { spin_lock_irq(&dev->pending_lock); list_add(&qp->ip->pending_mmaps, &dev->pending_mmaps); spin_unlock_irq(&dev->pending_lock); } ret = &qp->ibqp; /* * We have our QP and its good, now keep track of what types of opcodes * can be processed on this QP. We do this by keeping track of what the * 3 high order bits of the opcode are. */ switch (init_attr->qp_type) { case IB_QPT_SMI: case IB_QPT_GSI: case IB_QPT_UD: qp->allowed_ops = IB_OPCODE_UD_SEND_ONLY & OPCODE_QP_MASK; break; case IB_QPT_RC: qp->allowed_ops = IB_OPCODE_RC_SEND_ONLY & OPCODE_QP_MASK; break; case IB_QPT_UC: qp->allowed_ops = IB_OPCODE_UC_SEND_ONLY & OPCODE_QP_MASK; break; default: ret = ERR_PTR(-EINVAL); goto bail_ip; } goto bail; bail_ip: if (qp->ip) kref_put(&qp->ip->ref, hfi1_release_mmap_info); else vfree(qp->r_rq.wq); free_qpn(&dev->qp_dev->qpn_table, qp->ibqp.qp_num); bail_qp: kfree(qp->s_hdr); kfree(qp); bail_swq: vfree(swq); bail: return ret; }