int siw_ofed_modify_qp(struct ib_qp *ofa_qp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
struct siw_qp_attrs new_attrs;
enum siw_qp_attr_mask siw_attr_mask = 0;
struct siw_qp *qp = siw_qp_ofa2siw(ofa_qp);
int rv = 0;
if (!attr_mask) {
dprint(DBG_CM, "(QP%d): attr_mask==0 ignored\n", QP_ID(qp));
goto out;
}
siw_dprint_qp_attr_mask(attr_mask);
memset(&new_attrs, 0, sizeof new_attrs);
if (attr_mask & IB_QP_ACCESS_FLAGS) {
siw_attr_mask |= SIW_QP_ATTR_ACCESS_FLAGS;
if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
new_attrs.flags |= SIW_RDMA_READ_ENABLED;
if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
}
if (attr_mask & IB_QP_STATE) {
dprint(DBG_CM, "(QP%d): Desired IB QP state: %s\n",
QP_ID(qp), ib_qp_state_to_string[attr->qp_state]);
new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
if (new_attrs.state > SIW_QP_STATE_RTS)
qp->tx_ctx.tx_suspend = 1;
/* TODO: SIW_QP_STATE_UNDEF is currently not possible ... */
if (new_attrs.state == SIW_QP_STATE_UNDEF)
return -EINVAL;
siw_attr_mask |= SIW_QP_ATTR_STATE;
}
if (!attr_mask)
goto out;
down_write(&qp->state_lock);
rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
up_write(&qp->state_lock);
out:
dprint(DBG_CM, "(QP%d): Exit with %d\n", QP_ID(qp), rv);
return rv;
}
void siw_qp_put_ref(struct ib_qp *ofa_qp)
{
struct siw_qp *qp = siw_qp_ofa2siw(ofa_qp);
pr_debug(DBG_OBJ DBG_CM "(QP%d): Put Reference\n", QP_ID(qp));
siw_qp_put(qp);
}
void siw_qp_put_ref(struct ib_qp *ofa_qp)
{
struct siw_qp *qp = siw_qp_ofa2siw(ofa_qp);
dprint(DBG_OBJ|DBG_CM, "(QP%d): Put Reference\n", QP_ID(qp));
siw_qp_put(qp);
}
static void siw_qp_llp_data_ready(struct sock *sk)
#endif
{
struct siw_qp *qp;
read_lock(&sk->sk_callback_lock);
if (unlikely(!sk->sk_user_data || !sk_to_qp(sk))) {
dprint(DBG_ON, " No QP: %p\n", sk->sk_user_data);
goto done;
}
qp = sk_to_qp(sk);
if (down_read_trylock(&qp->state_lock)) {
read_descriptor_t rd_desc = {.arg.data = qp, .count = 1};
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0)
dprint(DBG_SK|DBG_RX, "(QP%d): "
"state (before tcp_read_sock)=%d, bytes=%x\n",
QP_ID(qp), qp->attrs.state, bytes);
#else
dprint(DBG_SK|DBG_RX, "(QP%d): "
"state (before tcp_read_sock)=%d\n",
QP_ID(qp), qp->attrs.state);
#endif
if (likely(qp->attrs.state == SIW_QP_STATE_RTS))
/*
* Implements data receive operation during
* socket callback. TCP gracefully catches
* the case where there is nothing to receive
* (not calling siw_tcp_rx_data() then).
*/
tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data);
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0)
dprint(DBG_SK|DBG_RX, "(QP%d): "
"state (after tcp_read_sock)=%d, bytes=%x\n",
QP_ID(qp), qp->attrs.state, bytes);
#else
dprint(DBG_SK|DBG_RX, "(QP%d): "
"state (after tcp_read_sock)=%d\n",
QP_ID(qp), qp->attrs.state);
#endif
up_read(&qp->state_lock);
} else {
int siw_qp_add(struct siw_dev *sdev, struct siw_qp *qp)
{
int rv = siw_add_obj(&sdev->idr_lock, &sdev->qp_idr, &qp->hdr);
if (!rv) {
dprint(DBG_OBJ, "(QP%d): New Object\n", QP_ID(qp));
qp->hdr.sdev = sdev;
}
return rv;
}
/*
* siw_copy_inline_sgl()
*
* Prepare sgl of inlined data for sending. For userland callers
* function checks if given buffer addresses and len's are within
* process context bounds.
* Data from all provided sge's are copied together into the wqe,
* referenced by a single sge.
*/
static int siw_copy_inline_sgl(struct ib_send_wr *ofa_wr, struct siw_wqe *wqe)
{
struct ib_sge *ofa_sge = ofa_wr->sg_list;
char *kbuf = wqe->wr.inlined_data.data;
int num_sge = ofa_wr->num_sge,
bytes = 0;
wqe->wr.inlined_data.sge.mem.buf = NULL;
wqe->wr.inlined_data.sge.addr = (u64)kbuf;
wqe->wr.inlined_data.sge.lkey = 0;
while (num_sge--) {
if (!ofa_sge->length) {
ofa_sge++;
continue;
}
bytes += ofa_sge->length;
if (bytes > SIW_MAX_INLINE) {
bytes = -EINVAL;
break;
}
if (wqe->qp->attrs.flags & SIW_KERNEL_VERBS)
memcpy(kbuf,
(void *)(uintptr_t)ofa_sge->addr,
ofa_sge->length);
else {
if (!access_ok(VERIFY_READ,
(__user void *)(uintptr_t)ofa_sge->addr,
ofa_sge->length)) {
bytes = -EFAULT;
break;
}
if (__copy_from_user(kbuf, (__user void *)
(uintptr_t)ofa_sge->addr,
ofa_sge->length)) {
bytes = -EFAULT;
break;
}
}
kbuf += ofa_sge->length;
ofa_sge++;
}
wqe->wr.inlined_data.sge.len = bytes > 0 ? bytes : 0;
wqe->wr.inlined_data.num_sge = bytes > 0 ? 1 : 0;
dprint(DBG_WR, "(QP%d): Copied inline data: %d\n", QP_ID(wqe->qp),
bytes);
return bytes;
}
static void siw_free_qp(struct kref *ref)
{
struct siw_qp *qp =
container_of(container_of(ref, struct siw_objhdr, ref),
struct siw_qp, hdr);
struct siw_dev *sdev = qp->hdr.sdev;
pr_debug(DBG_OBJ DBG_CM "(QP%d): Free Object\n", QP_ID(qp));
if (qp->cep)
siw_cep_put(qp->cep);
siw_remove_obj(&sdev->idr_lock, &sdev->qp_idr, &qp->hdr);
atomic_dec(&sdev->num_qp);
kfree(qp);
}
int siw_destroy_qp(struct ib_qp *ofa_qp)
{
struct siw_qp *qp = siw_qp_ofa2siw(ofa_qp);
struct siw_qp_attrs qp_attrs;
dprint(DBG_CM, "(QP%d): SIW QP state=%d, cep=0x%p\n",
QP_ID(qp), qp->attrs.state, qp->cep);
/*
* Mark QP as in process of destruction to prevent from eventual async
* callbacks to OFA core
*/
qp->attrs.flags |= SIW_QP_IN_DESTROY;
qp->rx_ctx.rx_suspend = 1;
down_write(&qp->state_lock);
qp_attrs.state = SIW_QP_STATE_ERROR;
(void)siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
if (qp->cep) {
siw_cep_put(qp->cep);
qp->cep = NULL;
}
up_write(&qp->state_lock);
if (qp->rx_ctx.crc_enabled)
crypto_free_hash(qp->rx_ctx.mpa_crc_hd.tfm);
if (qp->tx_ctx.crc_enabled)
crypto_free_hash(qp->tx_ctx.mpa_crc_hd.tfm);
/* Drop references */
siw_cq_put(qp->scq);
siw_cq_put(qp->rcq);
siw_pd_put(qp->pd);
qp->scq = qp->rcq = NULL;
siw_qp_put(qp);
return 0;
}
static void siw_free_qp(struct kref *ref)
{
struct siw_qp *qp =
container_of(container_of(ref, struct siw_objhdr, ref),
struct siw_qp, hdr);
struct siw_dev *sdev = qp->hdr.sdev;
unsigned long flags;
dprint(DBG_OBJ|DBG_CM, "(QP%d): Free Object\n", QP_ID(qp));
if (qp->cep)
siw_cep_put(qp->cep);
siw_drain_wq(&qp->freeq);
siw_remove_obj(&sdev->idr_lock, &sdev->qp_idr, &qp->hdr);
spin_lock_irqsave(&sdev->idr_lock, flags);
list_del(&qp->devq);
spin_unlock_irqrestore(&sdev->idr_lock, flags);
atomic_dec(&sdev->num_qp);
kfree(qp);
}
int siw_destroy_qp(struct ib_qp *ofa_qp)
{
struct siw_qp *qp = siw_qp_ofa2siw(ofa_qp);
struct siw_qp_attrs qp_attrs;
pr_debug(DBG_CM "(QP%d): SIW QP state=%d, cep=0x%p\n",
QP_ID(qp), qp->attrs.state, qp->cep);
/*
* Mark QP as in process of destruction to prevent from eventual async
* callbacks to OFA core
*/
qp->attrs.flags |= SIW_QP_IN_DESTROY;
down_write(&qp->state_lock);
qp_attrs.state = SIW_QP_STATE_ERROR;
(void)siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
if (qp->cep) {
siw_cep_put(qp->cep);
qp->cep = NULL;
}
up_write(&qp->state_lock);
/* Drop references */
siw_cq_put(qp->scq);
siw_cq_put(qp->rcq);
siw_pd_put(qp->pd);
qp->scq = qp->rcq = NULL;
siw_qp_put(qp);
return 0;
}
tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data);
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0)
dprint(DBG_SK|DBG_RX, "(QP%d): "
"state (after tcp_read_sock)=%d, bytes=%x\n",
QP_ID(qp), qp->attrs.state, bytes);
#else
dprint(DBG_SK|DBG_RX, "(QP%d): "
"state (after tcp_read_sock)=%d\n",
QP_ID(qp), qp->attrs.state);
#endif
up_read(&qp->state_lock);
} else {
dprint(DBG_SK|DBG_RX, "(QP%d): "
"Unable to acquire state_lock\n", QP_ID(qp));
}
done:
read_unlock(&sk->sk_callback_lock);
}
void siw_qp_llp_close(struct siw_qp *qp)
{
dprint(DBG_CM, "(QP%d): Enter: SIW QP state = %s, cep=0x%p\n",
QP_ID(qp), siw_qp_state_to_string[qp->attrs.state],
qp->cep);
down_write(&qp->state_lock);
qp->rx_ctx.rx_suspend = 1;
/*
* siw_post_send()
*
* Post a list of S-WR's to a SQ.
*
* @ofa_qp: OFA QP contained in siw QP
* @wr: Null terminated list of user WR's
* @bad_wr: Points to failing WR in case of synchronous failure.
*/
int siw_post_send(struct ib_qp *ofa_qp, struct ib_send_wr *wr,
struct ib_send_wr **bad_wr)
{
struct siw_wqe *wqe = NULL;
struct siw_qp *qp = siw_qp_ofa2siw(ofa_qp);
unsigned long flags;
int rv = 0;
dprint(DBG_WR|DBG_TX, "(QP%d): state=%d\n",
QP_ID(qp), qp->attrs.state);
/*
* Try to acquire QP state lock. Must be non-blocking
* to accommodate kernel clients needs.
*/
if (!down_read_trylock(&qp->state_lock)) {
*bad_wr = wr;
return -ENOTCONN;
}
if (qp->attrs.state != SIW_QP_STATE_RTS) {
dprint(DBG_WR|DBG_ON, "(QP%d): state=%d\n",
QP_ID(qp), qp->attrs.state);
up_read(&qp->state_lock);
*bad_wr = wr;
return -ENOTCONN;
}
dprint(DBG_WR|DBG_TX, "(QP%d): sq_space(#1)=%d\n",
QP_ID(qp), atomic_read(&qp->sq_space));
while (wr) {
wqe = siw_wqe_alloc(qp, opcode_ofa2siw(wr->opcode));
if (!wqe) {
dprint(DBG_ON, " siw_wqe_alloc\n");
rv = -ENOMEM;
break;
}
wr_type(wqe) = opcode_ofa2siw(wr->opcode);
wr_id(wqe) = wr->wr_id;
wr_flags(wqe) = wr->send_flags;
if (qp->attrs.flags & SIW_SIGNAL_ALL_WR)
wr_flags(wqe) |= IB_SEND_SIGNALED;
if (wr->num_sge > qp->attrs.sq_max_sges) {
/*
* NOTE: we allow for zero length wr's here.
*/
dprint(DBG_WR, "(QP%d): Num SGE: %d\n",
QP_ID(qp), wr->num_sge);
rv = -EINVAL;
break;
}
switch (wr->opcode) {
case IB_WR_SEND:
if (!SIW_INLINED_DATA(wqe)) {
rv = siw_copy_sgl(wr->sg_list,
wqe->wr.send.sge,
wr->num_sge);
wqe->wr.send.num_sge = wr->num_sge;
} else
rv = siw_copy_inline_sgl(wr, wqe);
if (rv < 0) {
rv = -EINVAL;
break;
}
wqe->bytes = rv;
break;
case IB_WR_RDMA_READ:
/*
* OFED WR restricts RREAD sink to SGL containing
* 1 SGE only. we could relax to SGL with multiple
* elements referring the SAME ltag or even sending
* a private per-rreq tag referring to a checked
* local sgl with MULTIPLE ltag's. would be easy
* to do...
*/
if (wr->num_sge != 1) {
rv = -EINVAL;
break;
}
rv = siw_copy_sgl(wr->sg_list, wqe->wr.rread.sge, 1);
/*
* NOTE: zero length RREAD is allowed!
*/
wqe->wr.rread.raddr = wr->wr.rdma.remote_addr;
wqe->wr.rread.rtag = wr->wr.rdma.rkey;
wqe->wr.rread.num_sge = 1;
wqe->bytes = rv;
break;
case IB_WR_RDMA_WRITE:
if (!SIW_INLINED_DATA(wqe)) {
rv = siw_copy_sgl(wr->sg_list,
wqe->wr.send.sge,
wr->num_sge);
wqe->wr.write.num_sge = wr->num_sge;
} else
rv = siw_copy_inline_sgl(wr, wqe);
/*
* NOTE: zero length WRITE is allowed!
*/
if (rv < 0) {
rv = -EINVAL;
break;
}
wqe->wr.write.raddr = wr->wr.rdma.remote_addr;
wqe->wr.write.rtag = wr->wr.rdma.rkey;
wqe->bytes = rv;
break;
default:
dprint(DBG_WR|DBG_TX|DBG_ON,
"(QP%d): Opcode %d not yet implemented\n",
QP_ID(qp), wr->opcode);
wqe->wr.sgl.num_sge = 0;
rv = -ENOSYS;
break;
}
dprint(DBG_WR|DBG_TX, "(QP%d): opcode %d, bytes %d, "
"flags 0x%x\n",
QP_ID(qp), wr_type(wqe), wqe->bytes,
wr_flags(wqe));
if (rv < 0)
break;
wqe->wr_status = SR_WR_QUEUED;
lock_sq_rxsave(qp, flags);
list_add_tail(&wqe->list, &qp->sq);
unlock_sq_rxsave(qp, flags);
wr = wr->next;
}
/*
* Send directly if SQ processing is not in progress.
* Eventual immediate errors (rv < 0) do not affect the involved
* RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
* processing, if new work is already pending. But rv must be passed
* to caller.
*/
lock_sq_rxsave(qp, flags);
if (tx_wqe(qp) == NULL) {
struct siw_wqe *next = siw_next_tx_wqe(qp);
if (next != NULL) {
if (wr_type(next) != SIW_WR_RDMA_READ_REQ ||
!ORD_SUSPEND_SQ(qp)) {
tx_wqe(qp) = next;
if (wr_type(next) != SIW_WR_RDMA_READ_REQ)
list_del_init(&next->list);
else
siw_rreq_queue(next, qp);
unlock_sq_rxsave(qp, flags);
dprint(DBG_WR|DBG_TX,
"(QP%d): Direct sending...\n",
QP_ID(qp));
if (qp->attrs.flags & SIW_KERNEL_VERBS)
siw_sq_queue_work(qp);
else if (siw_qp_sq_process(qp, 1) != 0 &&
!(qp->tx_ctx.tx_suspend))
siw_qp_cm_drop(qp, 0);
} else
unlock_sq_rxsave(qp, flags);
} else
unlock_sq_rxsave(qp, flags);
} else
unlock_sq_rxsave(qp, flags);
up_read(&qp->state_lock);
dprint(DBG_WR|DBG_TX, "(QP%d): sq_space(#2)=%d\n", QP_ID(qp),
atomic_read(&qp->sq_space));
if (rv >= 0)
return 0;
/*
* Immediate error
*/
dprint(DBG_WR|DBG_ON, "(QP%d): error=%d\n", QP_ID(qp), rv);
if (wqe != NULL)
siw_wqe_put(wqe);
*bad_wr = wr;
return rv;
}
struct ib_qp *siw_create_qp(struct ib_pd *ofa_pd,
struct ib_qp_init_attr *attrs,
struct ib_udata *udata)
{
struct siw_qp *qp = NULL;
struct siw_pd *pd = siw_pd_ofa2siw(ofa_pd);
struct ib_device *ofa_dev = ofa_pd->device;
struct siw_dev *sdev = siw_dev_ofa2siw(ofa_dev);
struct siw_cq *scq = NULL, *rcq = NULL;
struct siw_uresp_create_qp uresp;
unsigned long flags;
int kernel_verbs = ofa_pd->uobject ? 0 : 1;
int rv = 0;
dprint(DBG_OBJ|DBG_CM, ": new QP on device %s\n",
ofa_dev->name);
if (attrs->qp_type != IB_QPT_RC) {
dprint(DBG_ON, ": Only RC QP's supported\n");
return ERR_PTR(-ENOSYS);
}
if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
(attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
(attrs->cap.max_send_sge > SIW_MAX_SGE) ||
(attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
dprint(DBG_ON, ": QP Size!\n");
return ERR_PTR(-EINVAL);
}
if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
dprint(DBG_ON, ": Max Inline Send %d > %d!\n",
attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
return ERR_PTR(-EINVAL);
}
/*
* NOTE: we allow for zero element SQ and RQ WQE's SGL's
* but not for a QP unable to hold any WQE (SQ + RQ)
*/
if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0)
return ERR_PTR(-EINVAL);
if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
dprint(DBG_ON, ": Out of QP's\n");
rv = -ENOMEM;
goto err_out;
}
scq = siw_cq_id2obj(sdev, ((struct siw_cq *)attrs->send_cq)->hdr.id);
rcq = siw_cq_id2obj(sdev, ((struct siw_cq *)attrs->recv_cq)->hdr.id);
if (!scq || !rcq) {
dprint(DBG_OBJ, ": Fail: SCQ: 0x%p, RCQ: 0x%p\n",
scq, rcq);
rv = -EINVAL;
goto err_out;
}
qp = kzalloc(sizeof(*qp), GFP_KERNEL);
if (!qp) {
dprint(DBG_ON, ": kzalloc\n");
rv = -ENOMEM;
goto err_out;
}
INIT_LIST_HEAD(&qp->freeq);
INIT_LIST_HEAD(&qp->sq);
INIT_LIST_HEAD(&qp->rq);
INIT_LIST_HEAD(&qp->orq);
INIT_LIST_HEAD(&qp->irq);
init_rwsem(&qp->state_lock);
spin_lock_init(&qp->freeq_lock);
spin_lock_init(&qp->sq_lock);
spin_lock_init(&qp->rq_lock);
spin_lock_init(&qp->orq_lock);
init_waitqueue_head(&qp->tx_ctx.waitq);
rv = siw_qp_add(sdev, qp);
if (rv)
goto err_out;
if (kernel_verbs) {
int num_wqe = attrs->cap.max_send_wr + attrs->cap.max_recv_wr;
while (num_wqe--) {
struct siw_wqe *wqe = kzalloc(sizeof *wqe, GFP_KERNEL);
if (!wqe) {
rv = -ENOMEM;
goto err_out_idr;
}
SIW_INC_STAT_WQE;
INIT_LIST_HEAD(&wqe->list);
list_add(&wqe->list, &qp->freeq);
}
qp->attrs.flags |= SIW_KERNEL_VERBS;
}
if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
else {
rv = -EINVAL;
goto err_out;
}
}
qp->pd = pd;
qp->scq = scq;
qp->rcq = rcq;
if (attrs->srq) {
/*
* SRQ support.
* Verbs 6.3.7: ignore RQ size, if SRQ present
* Verbs 6.3.5: do not check PD of SRQ against PD of QP
*/
qp->srq = siw_srq_ofa2siw(attrs->srq);
qp->attrs.rq_size = 0;
atomic_set(&qp->rq_space, 0);
dprint(DBG_OBJ, " QP(%d): SRQ(%p) attached\n",
QP_ID(qp), qp->srq);
} else {
qp->srq = NULL;
qp->attrs.rq_size = attrs->cap.max_recv_wr;
atomic_set(&qp->rq_space, qp->attrs.rq_size);
}
qp->attrs.sq_size = attrs->cap.max_send_wr;
atomic_set(&qp->sq_space, qp->attrs.sq_size);
qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
/*
* ofed has no max_send_sge_rdmawrite
*/
qp->attrs.sq_max_sges_rdmaw = attrs->cap.max_send_sge;
qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
qp->attrs.state = SIW_QP_STATE_IDLE;
if (udata) {
uresp.sq_size = qp->attrs.sq_size;
uresp.rq_size = qp->attrs.rq_size;
uresp.qp_id = QP_ID(qp);
rv = ib_copy_to_udata(udata, &uresp, sizeof uresp);
if (rv)
goto err_out_idr;
}
atomic_set(&qp->tx_ctx.in_use, 0);
qp->ofa_qp.qp_num = QP_ID(qp);
siw_pd_get(pd);
INIT_LIST_HEAD(&qp->devq);
spin_lock_irqsave(&sdev->idr_lock, flags);
list_add_tail(&qp->devq, &sdev->qp_list);
spin_unlock_irqrestore(&sdev->idr_lock, flags);
return &qp->ofa_qp;
err_out_idr:
siw_remove_obj(&sdev->idr_lock, &sdev->qp_idr, &qp->hdr);
err_out:
if (scq)
siw_cq_put(scq);
if (rcq)
siw_cq_put(rcq);
if (qp)
siw_drain_wq(&qp->freeq);
kfree(qp);
atomic_dec(&sdev->num_qp);
return ERR_PTR(rv);
}
/*
* siw_post_receive()
*
* Post a list of R-WR's to a RQ.
*
* @ofa_qp: OFA QP contained in siw QP
* @wr: Null terminated list of user WR's
* @bad_wr: Points to failing WR in case of synchronous failure.
*/
int siw_post_receive(struct ib_qp *ofa_qp, struct ib_recv_wr *wr,
struct ib_recv_wr **bad_wr)
{
struct siw_wqe *wqe = NULL;
struct siw_qp *qp = siw_qp_ofa2siw(ofa_qp);
unsigned long flags;
int rv = 0;
dprint(DBG_WR|DBG_TX, "(QP%d): state=%d\n", QP_ID(qp),
qp->attrs.state);
if (unlikely(qp->srq)) {
*bad_wr = wr;
return -EOPNOTSUPP; /* what else from errno.h? */
}
/*
* Try to acquire QP state lock. Must be non-blocking
* to accommodate kernel clients needs.
*/
if (!down_read_trylock(&qp->state_lock)) {
*bad_wr = wr;
return -ENOTCONN;
}
if (qp->attrs.state > SIW_QP_STATE_RTS) {
up_read(&qp->state_lock);
dprint(DBG_ON, " (QP%d): state=%d\n", QP_ID(qp),
qp->attrs.state);
*bad_wr = wr;
return -EINVAL;
}
while (wr) {
wqe = siw_wqe_alloc(qp, SIW_WR_RECEIVE);
if (!wqe) {
rv = -ENOMEM;
break;
}
if (wr->num_sge > qp->attrs.rq_max_sges) {
dprint(DBG_WR|DBG_ON, "(QP%d): Num SGE: %d\n",
QP_ID(qp), wr->num_sge);
rv = -EINVAL;
break;
}
wr_type(wqe) = SIW_WR_RECEIVE;
wr_id(wqe) = wr->wr_id;
rv = siw_copy_sgl(wr->sg_list, wqe->wr.recv.sge, wr->num_sge);
if (rv < 0) {
/*
* XXX tentatively allow zero length receive
*/
rv = -EINVAL;
break;
}
wqe->wr.recv.num_sge = wr->num_sge;
wqe->bytes = rv;
wqe->wr_status = SR_WR_QUEUED;
lock_rq_rxsave(qp, flags);
list_add_tail(&wqe->list, &qp->rq);
unlock_rq_rxsave(qp, flags);
wr = wr->next;
}
if (rv <= 0) {
dprint(DBG_WR|DBG_ON, "(QP%d): error=%d\n", QP_ID(qp), rv);
if (wqe != NULL)
siw_wqe_put(wqe);
*bad_wr = wr;
}
dprint(DBG_WR|DBG_RX, "(QP%d): rq_space=%d\n", QP_ID(qp),
atomic_read(&qp->rq_space));
up_read(&qp->state_lock);
return rv > 0 ? 0 : rv;
}
struct ib_qp *siw_create_qp(struct ib_pd *ofa_pd,
struct ib_qp_init_attr *attrs,
struct ib_udata *udata)
{
struct siw_qp *qp = NULL;
struct siw_pd *pd = siw_pd_ofa2siw(ofa_pd);
struct ib_device *ofa_dev = ofa_pd->device;
struct siw_dev *sdev = siw_dev_ofa2siw(ofa_dev);
struct siw_cq *scq = NULL, *rcq = NULL;
int rv = 0;
pr_debug(DBG_OBJ DBG_CM ": new QP on device %s\n",
ofa_dev->name);
if (!ofa_pd->uobject) {
pr_debug(": This driver does not support kernel clients\n");
return ERR_PTR(-EINVAL);
}
if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
pr_debug(": Out of QP's\n");
rv = -ENOMEM;
goto err_out;
}
if (attrs->qp_type != IB_QPT_RC) {
pr_debug(": Only RC QP's supported\n");
rv = -EINVAL;
goto err_out;
}
if (attrs->srq) {
pr_debug(": SRQ is not supported\n");
rv = -EINVAL;
goto err_out;
}
scq = siw_cq_id2obj(sdev, ((struct siw_cq *)attrs->send_cq)->hdr.id);
rcq = siw_cq_id2obj(sdev, ((struct siw_cq *)attrs->recv_cq)->hdr.id);
if (!scq || !rcq) {
pr_debug(DBG_OBJ ": Fail: SCQ: 0x%p, RCQ: 0x%p\n",
scq, rcq);
rv = -EINVAL;
goto err_out;
}
qp = kzalloc(sizeof *qp, GFP_KERNEL);
if (!qp) {
pr_debug(": kzalloc\n");
rv = -ENOMEM;
goto err_out;
}
init_rwsem(&qp->state_lock);
rv = siw_qp_add(sdev, qp);
if (rv)
goto err_out;
qp->pd = pd;
qp->scq = scq;
qp->rcq = rcq;
qp->attrs.state = SIW_QP_STATE_IDLE;
if (udata) {
struct urdma_udata_create_qp ureq;
struct urdma_uresp_create_qp uresp;
rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
if (rv)
goto err_out_idr;
qp->attrs.irq_size = ureq.ird_max;
qp->attrs.orq_size = ureq.ord_max;
qp->attrs.urdma_devid = ureq.urdmad_dev_id;
qp->attrs.urdma_qp_id = ureq.urdmad_qp_id;
qp->attrs.urdma_rxq = ureq.rxq;
qp->attrs.urdma_txq = ureq.txq;
memset(&uresp, 0, sizeof uresp);
uresp.kmod_qp_id = QP_ID(qp);
rv = ib_copy_to_udata(udata, &uresp, sizeof uresp);
if (rv)
goto err_out_idr;
}
qp->ofa_qp.qp_num = QP_ID(qp);
siw_pd_get(pd);
return &qp->ofa_qp;
err_out_idr:
siw_remove_obj(&sdev->idr_lock, &sdev->qp_idr, &qp->hdr);
err_out:
if (scq)
siw_cq_put(scq);
if (rcq)
siw_cq_put(rcq);
if (qp) {
kfree(qp);
}
atomic_dec(&sdev->num_qp);
return ERR_PTR(rv);
}
void siw_qp_put(struct siw_qp *qp)
{
pr_debug(DBG_OBJ "(QP%d): Old refcount: %d\n",
QP_ID(qp), kref_read(&qp->hdr.ref));
kref_put(&qp->hdr.ref, siw_free_qp);
}
