static int rxe_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
struct ib_send_wr **bad_wr)
{
int err = 0;
struct rxe_qp *qp = to_rqp(ibqp);
unsigned int mask;
unsigned int length = 0;
int i;
int must_sched;
if (unlikely(!qp->valid)) {
*bad_wr = wr;
return -EINVAL;
}
if (unlikely(qp->req.state < QP_STATE_READY)) {
*bad_wr = wr;
return -EINVAL;
}
while (wr) {
mask = wr_opcode_mask(wr->opcode, qp);
if (unlikely(!mask)) {
err = -EINVAL;
*bad_wr = wr;
break;
}
if (unlikely((wr->send_flags & IB_SEND_INLINE) &&
!(mask & WR_INLINE_MASK))) {
err = -EINVAL;
*bad_wr = wr;
break;
}
length = 0;
for (i = 0; i < wr->num_sge; i++)
length += wr->sg_list[i].length;
err = post_one_send(qp, wr, mask, length);
if (err) {
*bad_wr = wr;
break;
}
wr = wr->next;
}
/*
* Must sched in case of GSI QP because ib_send_mad() hold irq lock,
* and the requester call ip_local_out_sk() that takes spin_lock_bh.
*/
must_sched = (qp_type(qp) == IB_QPT_GSI) ||
(queue_count(qp->sq.queue) > 1);
rxe_run_task(&qp->req.task, must_sched);
return err;
}
static void req_retry(struct rxe_qp *qp)
{
struct rxe_send_wqe *wqe;
unsigned int wqe_index;
unsigned int mask;
int npsn;
int first = 1;
wqe = queue_head(qp->sq.queue);
npsn = (qp->comp.psn - wqe->first_psn) & BTH_PSN_MASK;
qp->req.wqe_index = consumer_index(qp->sq.queue);
qp->req.psn = qp->comp.psn;
qp->req.opcode = -1;
for (wqe_index = consumer_index(qp->sq.queue);
wqe_index != producer_index(qp->sq.queue);
wqe_index = next_index(qp->sq.queue, wqe_index)) {
wqe = addr_from_index(qp->sq.queue, wqe_index);
mask = wr_opcode_mask(wqe->wr.opcode, qp);
if (wqe->state == wqe_state_posted)
break;
if (wqe->state == wqe_state_done)
continue;
wqe->iova = (mask & WR_ATOMIC_MASK) ?
wqe->wr.wr.atomic.remote_addr :
(mask & WR_READ_OR_WRITE_MASK) ?
wqe->wr.wr.rdma.remote_addr :
0;
if (!first || (mask & WR_READ_MASK) == 0) {
wqe->dma.resid = wqe->dma.length;
wqe->dma.cur_sge = 0;
wqe->dma.sge_offset = 0;
}
if (first) {
first = 0;
if (mask & WR_WRITE_OR_SEND_MASK)
retry_first_write_send(qp, wqe, mask, npsn);
if (mask & WR_READ_MASK)
wqe->iova += npsn * qp->mtu;
}
wqe->state = wqe_state_posted;
}
}
static int rxe_post_send_kernel(struct rxe_qp *qp, const struct ib_send_wr *wr,
const struct ib_send_wr **bad_wr)
{
int err = 0;
unsigned int mask;
unsigned int length = 0;
int i;
while (wr) {
mask = wr_opcode_mask(wr->opcode, qp);
if (unlikely(!mask)) {
err = -EINVAL;
*bad_wr = wr;
break;
}
if (unlikely((wr->send_flags & IB_SEND_INLINE) &&
!(mask & WR_INLINE_MASK))) {
err = -EINVAL;
*bad_wr = wr;
break;
}
length = 0;
for (i = 0; i < wr->num_sge; i++)
length += wr->sg_list[i].length;
err = post_one_send(qp, wr, mask, length);
if (err) {
*bad_wr = wr;
break;
}
wr = wr->next;
}
rxe_run_task(&qp->req.task, 1);
if (unlikely(qp->req.state == QP_STATE_ERROR))
rxe_run_task(&qp->comp.task, 1);
return err;
}
static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
{
struct rxe_send_wqe *wqe = queue_head(qp->sq.queue);
unsigned long flags;
if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
/* check to see if we are drained;
* state_lock used by requester and completer
*/
spin_lock_irqsave(&qp->state_lock, flags);
do {
if (qp->req.state != QP_STATE_DRAIN) {
/* comp just finished */
spin_unlock_irqrestore(&qp->state_lock,
flags);
break;
}
if (wqe && ((qp->req.wqe_index !=
consumer_index(qp->sq.queue)) ||
(wqe->state != wqe_state_posted))) {
/* comp not done yet */
spin_unlock_irqrestore(&qp->state_lock,
flags);
break;
}
qp->req.state = QP_STATE_DRAINED;
spin_unlock_irqrestore(&qp->state_lock, flags);
if (qp->ibqp.event_handler) {
struct ib_event ev;
ev.device = qp->ibqp.device;
ev.element.qp = &qp->ibqp;
ev.event = IB_EVENT_SQ_DRAINED;
qp->ibqp.event_handler(&ev,
qp->ibqp.qp_context);
}
} while (0);
}
if (qp->req.wqe_index == producer_index(qp->sq.queue))
return NULL;
wqe = addr_from_index(qp->sq.queue, qp->req.wqe_index);
if (unlikely((qp->req.state == QP_STATE_DRAIN ||
qp->req.state == QP_STATE_DRAINED) &&
(wqe->state != wqe_state_processing)))
return NULL;
if (unlikely((wqe->wr.send_flags & IB_SEND_FENCE) &&
(qp->req.wqe_index != consumer_index(qp->sq.queue)))) {
qp->req.wait_fence = 1;
return NULL;
}
wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp);
return wqe;
}
