/* Executes a new request. A retried request never reach that function (send
* and writes are discarded, and reads and atomics are retried elsewhere.
*/
static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
{
enum resp_states err;
if (pkt->mask & RXE_SEND_MASK) {
if (qp_type(qp) == IB_QPT_UD ||
qp_type(qp) == IB_QPT_SMI ||
qp_type(qp) == IB_QPT_GSI) {
union rdma_network_hdr hdr;
struct sk_buff *skb = PKT_TO_SKB(pkt);
memset(&hdr, 0, sizeof(hdr));
if (skb->protocol == htons(ETH_P_IP))
memcpy(&hdr.roce4grh, ip_hdr(skb), sizeof(hdr.roce4grh));
else if (skb->protocol == htons(ETH_P_IPV6))
memcpy(&hdr.ibgrh, ipv6_hdr(skb), sizeof(hdr.ibgrh));
err = send_data_in(qp, &hdr, sizeof(hdr));
if (err)
return err;
}
err = send_data_in(qp, payload_addr(pkt), payload_size(pkt));
if (err)
return err;
} else if (pkt->mask & RXE_WRITE_MASK) {
err = write_data_in(qp, pkt);
if (err)
return err;
} else if (pkt->mask & RXE_READ_MASK) {
/* For RDMA Read we can increment the msn now. See C9-148. */
qp->resp.msn++;
return RESPST_READ_REPLY;
} else if (pkt->mask & RXE_ATOMIC_MASK) {
err = process_atomic(qp, pkt);
if (err)
return err;
} else
/* Unreachable */
WARN_ON(1);
/* We successfully processed this new request. */
qp->resp.msn++;
/* next expected psn, read handles this separately */
qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
qp->resp.opcode = pkt->opcode;
qp->resp.status = IB_WC_SUCCESS;
if (pkt->mask & RXE_COMP_MASK)
return RESPST_COMPLETE;
else if (qp_type(qp) == IB_QPT_RC)
return RESPST_ACKNOWLEDGE;
else
return RESPST_CLEANUP;
}
/* Executes a new request. A retried request never reach that function (send
* and writes are discarded, and reads and atomics are retried elsewhere.
*/
static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
{
enum resp_states err;
if (pkt->mask & RXE_SEND_MASK) {
if (qp_type(qp) == IB_QPT_UD ||
qp_type(qp) == IB_QPT_SMI ||
qp_type(qp) == IB_QPT_GSI) {
union rdma_network_hdr hdr;
build_rdma_network_hdr(&hdr, pkt);
err = send_data_in(qp, &hdr, sizeof(hdr));
if (err)
return err;
}
err = send_data_in(qp, payload_addr(pkt), payload_size(pkt));
if (err)
return err;
} else if (pkt->mask & RXE_WRITE_MASK) {
err = write_data_in(qp, pkt);
if (err)
return err;
} else if (pkt->mask & RXE_READ_MASK) {
/* For RDMA Read we can increment the msn now. See C9-148. */
qp->resp.msn++;
return RESPST_READ_REPLY;
} else if (pkt->mask & RXE_ATOMIC_MASK) {
err = process_atomic(qp, pkt);
if (err)
return err;
} else {
/* Unreachable */
WARN_ON_ONCE(1);
}
/* next expected psn, read handles this separately */
qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;
qp->resp.ack_psn = qp->resp.psn;
qp->resp.opcode = pkt->opcode;
qp->resp.status = IB_WC_SUCCESS;
if (pkt->mask & RXE_COMP_MASK) {
/* We successfully processed this new request. */
qp->resp.msn++;
return RESPST_COMPLETE;
} else if (qp_type(qp) == IB_QPT_RC)
return RESPST_ACKNOWLEDGE;
else
return RESPST_CLEANUP;
}
static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
struct rxe_pkt_info *pkt, struct sk_buff *skb,
int paylen)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
u32 crc = 0;
u32 *p;
int err;
err = rxe_prepare(rxe, pkt, skb, &crc);
if (err)
return err;
if (pkt->mask & RXE_WRITE_OR_SEND) {
if (wqe->wr.send_flags & IB_SEND_INLINE) {
u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset];
crc = rxe_crc32(rxe, crc, tmp, paylen);
memcpy(payload_addr(pkt), tmp, paylen);
wqe->dma.resid -= paylen;
wqe->dma.sge_offset += paylen;
} else {
err = copy_data(rxe, qp->pd, 0, &wqe->dma,
payload_addr(pkt), paylen,
from_mem_obj,
&crc);
if (err)
return err;
}
}
p = payload_addr(pkt) + paylen + bth_pad(pkt);
*p = ~crc;
return 0;
}
static inline enum comp_state do_read(struct rxe_qp *qp,
struct rxe_pkt_info *pkt,
struct rxe_send_wqe *wqe)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
int ret;
ret = copy_data(rxe, qp->pd, IB_ACCESS_LOCAL_WRITE,
&wqe->dma, payload_addr(pkt),
payload_size(pkt), to_mem_obj, NULL);
if (ret)
return COMPST_ERROR;
if (wqe->dma.resid == 0 && (pkt->mask & RXE_END_MASK))
return COMPST_COMP_ACK;
else
return COMPST_UPDATE_COMP;
}
static enum resp_states write_data_in(struct rxe_qp *qp,
struct rxe_pkt_info *pkt)
{
enum resp_states rc = RESPST_NONE;
int err;
int data_len = payload_size(pkt);
err = rxe_mem_copy(qp->resp.mr, qp->resp.va, payload_addr(pkt),
data_len, to_mem_obj, NULL);
if (err) {
rc = RESPST_ERR_RKEY_VIOLATION;
goto out;
}
qp->resp.va += data_len;
qp->resp.resid -= data_len;
out:
return rc;
}
/* RDMA read response. If res is not NULL, then we have a current RDMA request
* being processed or replayed.
*/
static enum resp_states read_reply(struct rxe_qp *qp,
struct rxe_pkt_info *req_pkt)
{
struct rxe_pkt_info ack_pkt;
struct sk_buff *skb;
int mtu = qp->mtu;
enum resp_states state;
int payload;
int opcode;
int err;
struct resp_res *res = qp->resp.res;
u32 icrc;
u32 *p;
if (!res) {
/* This is the first time we process that request. Get a
* resource
*/
res = &qp->resp.resources[qp->resp.res_head];
free_rd_atomic_resource(qp, res);
rxe_advance_resp_resource(qp);
res->type = RXE_READ_MASK;
res->replay = 0;
res->read.va = qp->resp.va;
res->read.va_org = qp->resp.va;
res->first_psn = req_pkt->psn;
if (reth_len(req_pkt)) {
res->last_psn = (req_pkt->psn +
(reth_len(req_pkt) + mtu - 1) /
mtu - 1) & BTH_PSN_MASK;
} else {
res->last_psn = res->first_psn;
}
res->cur_psn = req_pkt->psn;
res->read.resid = qp->resp.resid;
res->read.length = qp->resp.resid;
res->read.rkey = qp->resp.rkey;
/* note res inherits the reference to mr from qp */
res->read.mr = qp->resp.mr;
qp->resp.mr = NULL;
qp->resp.res = res;
res->state = rdatm_res_state_new;
}
if (res->state == rdatm_res_state_new) {
if (res->read.resid <= mtu)
opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY;
else
opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
} else {
if (res->read.resid > mtu)
opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;
else
opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
}
res->state = rdatm_res_state_next;
payload = min_t(int, res->read.resid, mtu);
skb = prepare_ack_packet(qp, req_pkt, &ack_pkt, opcode, payload,
res->cur_psn, AETH_ACK_UNLIMITED, &icrc);
if (!skb)
return RESPST_ERR_RNR;
err = rxe_mem_copy(res->read.mr, res->read.va, payload_addr(&ack_pkt),
payload, from_mem_obj, &icrc);
if (err)
pr_err("Failed copying memory\n");
p = payload_addr(&ack_pkt) + payload + bth_pad(&ack_pkt);
*p = ~icrc;
err = rxe_xmit_packet(qp, &ack_pkt, skb);
if (err) {
pr_err("Failed sending RDMA reply.\n");
return RESPST_ERR_RNR;
}
res->read.va += payload;
res->read.resid -= payload;
res->cur_psn = (res->cur_psn + 1) & BTH_PSN_MASK;
if (res->read.resid > 0) {
state = RESPST_DONE;
} else {
qp->resp.res = NULL;
if (!res->replay)
qp->resp.opcode = -1;
if (psn_compare(res->cur_psn, qp->resp.psn) >= 0)
qp->resp.psn = res->cur_psn;
state = RESPST_CLEANUP;
}
return state;
}
static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,
struct rxe_pkt_info *pkt,
struct rxe_pkt_info *ack,
int opcode,
int payload,
u32 psn,
u8 syndrome,
u32 *crcp)
{
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
struct sk_buff *skb;
u32 crc = 0;
u32 *p;
int paylen;
int pad;
int err;
/*
* allocate packet
*/
pad = (-payload) & 0x3;
paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE;
skb = rxe_init_packet(rxe, &qp->pri_av, paylen, ack);
if (!skb)
return NULL;
ack->qp = qp;
ack->opcode = opcode;
ack->mask = rxe_opcode[opcode].mask;
ack->offset = pkt->offset;
ack->paylen = paylen;
/* fill in bth using the request packet headers */
memcpy(ack->hdr, pkt->hdr, pkt->offset + RXE_BTH_BYTES);
bth_set_opcode(ack, opcode);
bth_set_qpn(ack, qp->attr.dest_qp_num);
bth_set_pad(ack, pad);
bth_set_se(ack, 0);
bth_set_psn(ack, psn);
bth_set_ack(ack, 0);
ack->psn = psn;
if (ack->mask & RXE_AETH_MASK) {
aeth_set_syn(ack, syndrome);
aeth_set_msn(ack, qp->resp.msn);
}
if (ack->mask & RXE_ATMACK_MASK)
atmack_set_orig(ack, qp->resp.atomic_orig);
err = rxe_prepare(ack, skb, &crc);
if (err) {
kfree_skb(skb);
return NULL;
}
if (crcp) {
/* CRC computation will be continued by the caller */
*crcp = crc;
} else {
p = payload_addr(ack) + payload + bth_pad(ack);
*p = ~crc;
}
return skb;
}
