static void init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr, const struct ib_send_wr *ibwr) { wr->wr_id = ibwr->wr_id; wr->num_sge = ibwr->num_sge; wr->opcode = ibwr->opcode; wr->send_flags = ibwr->send_flags; if (qp_type(qp) == IB_QPT_UD || qp_type(qp) == IB_QPT_SMI || qp_type(qp) == IB_QPT_GSI) { wr->wr.ud.remote_qpn = ud_wr(ibwr)->remote_qpn; wr->wr.ud.remote_qkey = ud_wr(ibwr)->remote_qkey; if (qp_type(qp) == IB_QPT_GSI) wr->wr.ud.pkey_index = ud_wr(ibwr)->pkey_index; if (wr->opcode == IB_WR_SEND_WITH_IMM) wr->ex.imm_data = ibwr->ex.imm_data; } else { switch (wr->opcode) { case IB_WR_RDMA_WRITE_WITH_IMM: wr->ex.imm_data = ibwr->ex.imm_data; /* fall through */ case IB_WR_RDMA_READ: case IB_WR_RDMA_WRITE: wr->wr.rdma.remote_addr = rdma_wr(ibwr)->remote_addr; wr->wr.rdma.rkey = rdma_wr(ibwr)->rkey; break; case IB_WR_SEND_WITH_IMM: wr->ex.imm_data = ibwr->ex.imm_data; break; case IB_WR_SEND_WITH_INV: wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey; break; case IB_WR_ATOMIC_CMP_AND_SWP: case IB_WR_ATOMIC_FETCH_AND_ADD: wr->wr.atomic.remote_addr = atomic_wr(ibwr)->remote_addr; wr->wr.atomic.compare_add = atomic_wr(ibwr)->compare_add; wr->wr.atomic.swap = atomic_wr(ibwr)->swap; wr->wr.atomic.rkey = atomic_wr(ibwr)->rkey; break; case IB_WR_LOCAL_INV: wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey; break; case IB_WR_REG_MR: wr->wr.reg.mr = reg_wr(ibwr)->mr; wr->wr.reg.key = reg_wr(ibwr)->key; wr->wr.reg.access = reg_wr(ibwr)->access; break; default: break; } } }
static int init_send_wqe(struct rxe_qp *qp, struct ib_send_wr *ibwr, unsigned int mask, unsigned int length, struct rxe_send_wqe *wqe) { int num_sge = ibwr->num_sge; struct ib_sge *sge; int i; u8 *p; init_send_wr(qp, &wqe->wr, ibwr); if (qp_type(qp) == IB_QPT_UD || qp_type(qp) == IB_QPT_SMI || qp_type(qp) == IB_QPT_GSI) memcpy(&wqe->av, &to_rah(ud_wr(ibwr)->ah)->av, sizeof(wqe->av)); if (unlikely(ibwr->send_flags & IB_SEND_INLINE)) { p = wqe->dma.inline_data; sge = ibwr->sg_list; for (i = 0; i < num_sge; i++, sge++) { if (qp->is_user && copy_from_user(p, (__user void *) (uintptr_t)sge->addr, sge->length)) return -EFAULT; else if (!qp->is_user) memcpy(p, (void *)(uintptr_t)sge->addr, sge->length); p += sge->length; } } else if (mask & WR_REG_MASK) { wqe->mask = mask; wqe->state = wqe_state_posted; return 0; } else memcpy(wqe->dma.sge, ibwr->sg_list, num_sge * sizeof(struct ib_sge)); wqe->iova = (mask & WR_ATOMIC_MASK) ? atomic_wr(ibwr)->remote_addr : rdma_wr(ibwr)->remote_addr; wqe->mask = mask; wqe->dma.length = length; wqe->dma.resid = length; wqe->dma.num_sge = num_sge; wqe->dma.cur_sge = 0; wqe->dma.sge_offset = 0; wqe->state = wqe_state_posted; wqe->ssn = atomic_add_return(1, &qp->ssn); return 0; }
static inline int qedr_gsi_build_header(struct qedr_dev *dev, struct qedr_qp *qp, struct ib_send_wr *swr, struct ib_ud_header *udh, int *roce_mode) { bool has_vlan = false, has_grh_ipv6 = true; struct rdma_ah_attr *ah_attr = &get_qedr_ah(ud_wr(swr)->ah)->attr; const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); union ib_gid sgid; int send_size = 0; u16 vlan_id = 0; u16 ether_type; struct ib_gid_attr sgid_attr; int rc; int ip_ver = 0; bool has_udp = false; int i; send_size = 0; for (i = 0; i < swr->num_sge; ++i) send_size += swr->sg_list[i].length; rc = ib_get_cached_gid(qp->ibqp.device, rdma_ah_get_port_num(ah_attr), grh->sgid_index, &sgid, &sgid_attr); if (rc) { DP_ERR(dev, "gsi post send: failed to get cached GID (port=%d, ix=%d)\n", rdma_ah_get_port_num(ah_attr), grh->sgid_index); return rc; } if (sgid_attr.ndev) { vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev); if (vlan_id < VLAN_CFI_MASK) has_vlan = true; dev_put(sgid_attr.ndev); } if (!memcmp(&sgid, &zgid, sizeof(sgid))) { DP_ERR(dev, "gsi post send: GID not found GID index %d\n", grh->sgid_index); return -ENOENT; } has_udp = (sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP); if (!has_udp) { /* RoCE v1 */ ether_type = ETH_P_IBOE; *roce_mode = ROCE_V1; } else if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) { /* RoCE v2 IPv4 */ ip_ver = 4; ether_type = ETH_P_IP; has_grh_ipv6 = false; *roce_mode = ROCE_V2_IPV4; } else { /* RoCE v2 IPv6 */ ip_ver = 6; ether_type = ETH_P_IPV6; *roce_mode = ROCE_V2_IPV6; } rc = ib_ud_header_init(send_size, false, true, has_vlan, has_grh_ipv6, ip_ver, has_udp, 0, udh); if (rc) { DP_ERR(dev, "gsi post send: failed to init header\n"); return rc; } /* ENET + VLAN headers */ ether_addr_copy(udh->eth.dmac_h, ah_attr->roce.dmac); ether_addr_copy(udh->eth.smac_h, dev->ndev->dev_addr); if (has_vlan) { udh->eth.type = htons(ETH_P_8021Q); udh->vlan.tag = htons(vlan_id); udh->vlan.type = htons(ether_type); } else { udh->eth.type = htons(ether_type); } /* BTH */ udh->bth.solicited_event = !!(swr->send_flags & IB_SEND_SOLICITED); udh->bth.pkey = QEDR_ROCE_PKEY_DEFAULT; udh->bth.destination_qpn = htonl(ud_wr(swr)->remote_qpn); udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1)); udh->bth.opcode = IB_OPCODE_UD_SEND_ONLY; /* DETH */ udh->deth.qkey = htonl(0x80010000); udh->deth.source_qpn = htonl(QEDR_GSI_QPN); if (has_grh_ipv6) { /* GRH / IPv6 header */ udh->grh.traffic_class = grh->traffic_class; udh->grh.flow_label = grh->flow_label; udh->grh.hop_limit = grh->hop_limit; udh->grh.destination_gid = grh->dgid; memcpy(&udh->grh.source_gid.raw, &sgid.raw, sizeof(udh->grh.source_gid.raw)); } else { /* IPv4 header */ u32 ipv4_addr; udh->ip4.protocol = IPPROTO_UDP; udh->ip4.tos = htonl(grh->flow_label); udh->ip4.frag_off = htons(IP_DF); udh->ip4.ttl = grh->hop_limit; ipv4_addr = qedr_get_ipv4_from_gid(sgid.raw); udh->ip4.saddr = ipv4_addr; ipv4_addr = qedr_get_ipv4_from_gid(grh->dgid.raw); udh->ip4.daddr = ipv4_addr; /* note: checksum is calculated by the device */ } /* UDP */ if (has_udp) { udh->udp.sport = htons(QEDR_ROCE_V2_UDP_SPORT); udh->udp.dport = htons(ROCE_V2_UDP_DPORT); udh->udp.csum = 0; /* UDP length is untouched hence is zero */ } return 0; }
/** * rvt_post_one_wr - post one RC, UC, or UD send work request * @qp: the QP to post on * @wr: the work request to send */ static int rvt_post_one_wr(struct rvt_qp *qp, struct ib_send_wr *wr, int *call_send) { struct rvt_swqe *wqe; u32 next; int i; int j; int acc; struct rvt_lkey_table *rkt; struct rvt_pd *pd; struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); u8 log_pmtu; int ret; /* IB spec says that num_sge == 0 is OK. */ if (unlikely(wr->num_sge > qp->s_max_sge)) return -EINVAL; /* * Don't allow RDMA reads or atomic operations on UC or * undefined operations. * Make sure buffer is large enough to hold the result for atomics. */ if (qp->ibqp.qp_type == IB_QPT_UC) { if ((unsigned)wr->opcode >= IB_WR_RDMA_READ) return -EINVAL; } else if (qp->ibqp.qp_type != IB_QPT_RC) { /* Check IB_QPT_SMI, IB_QPT_GSI, IB_QPT_UD opcode */ if (wr->opcode != IB_WR_SEND && wr->opcode != IB_WR_SEND_WITH_IMM) return -EINVAL; /* Check UD destination address PD */ if (qp->ibqp.pd != ud_wr(wr)->ah->pd) return -EINVAL; } else if ((unsigned)wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD) { return -EINVAL; } else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP && (wr->num_sge == 0 || wr->sg_list[0].length < sizeof(u64) || wr->sg_list[0].addr & (sizeof(u64) - 1))) { return -EINVAL; } else if (wr->opcode >= IB_WR_RDMA_READ && !qp->s_max_rd_atomic) { return -EINVAL; } /* check for avail */ if (unlikely(!qp->s_avail)) { qp->s_avail = qp_get_savail(qp); if (WARN_ON(qp->s_avail > (qp->s_size - 1))) rvt_pr_err(rdi, "More avail entries than QP RB size.\nQP: %u, size: %u, avail: %u\nhead: %u, tail: %u, cur: %u, acked: %u, last: %u", qp->ibqp.qp_num, qp->s_size, qp->s_avail, qp->s_head, qp->s_tail, qp->s_cur, qp->s_acked, qp->s_last); if (!qp->s_avail) return -ENOMEM; } next = qp->s_head + 1; if (next >= qp->s_size) next = 0; rkt = &rdi->lkey_table; pd = ibpd_to_rvtpd(qp->ibqp.pd); wqe = rvt_get_swqe_ptr(qp, qp->s_head); if (qp->ibqp.qp_type != IB_QPT_UC && qp->ibqp.qp_type != IB_QPT_RC) memcpy(&wqe->ud_wr, ud_wr(wr), sizeof(wqe->ud_wr)); else if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || wr->opcode == IB_WR_RDMA_WRITE || wr->opcode == IB_WR_RDMA_READ) memcpy(&wqe->rdma_wr, rdma_wr(wr), sizeof(wqe->rdma_wr)); else if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) memcpy(&wqe->atomic_wr, atomic_wr(wr), sizeof(wqe->atomic_wr)); else memcpy(&wqe->wr, wr, sizeof(wqe->wr)); wqe->length = 0; j = 0; if (wr->num_sge) { acc = wr->opcode >= IB_WR_RDMA_READ ? IB_ACCESS_LOCAL_WRITE : 0; for (i = 0; i < wr->num_sge; i++) { u32 length = wr->sg_list[i].length; int ok; if (length == 0) continue; ok = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j], &wr->sg_list[i], acc); if (!ok) { ret = -EINVAL; goto bail_inval_free; } wqe->length += length; j++; } wqe->wr.num_sge = j; } /* general part of wqe valid - allow for driver checks */ if (rdi->driver_f.check_send_wqe) { ret = rdi->driver_f.check_send_wqe(qp, wqe); if (ret < 0) goto bail_inval_free; if (ret) *call_send = ret; } log_pmtu = qp->log_pmtu; if (qp->ibqp.qp_type != IB_QPT_UC && qp->ibqp.qp_type != IB_QPT_RC) { struct rvt_ah *ah = ibah_to_rvtah(wqe->ud_wr.ah); log_pmtu = ah->log_pmtu; atomic_inc(&ibah_to_rvtah(ud_wr(wr)->ah)->refcount); } wqe->ssn = qp->s_ssn++; wqe->psn = qp->s_next_psn; wqe->lpsn = wqe->psn + (wqe->length ? ((wqe->length - 1) >> log_pmtu) : 0); qp->s_next_psn = wqe->lpsn + 1; trace_rvt_post_one_wr(qp, wqe); smp_wmb(); /* see request builders */ qp->s_avail--; qp->s_head = next; return 0; bail_inval_free: /* release mr holds */ while (j) { struct rvt_sge *sge = &wqe->sg_list[--j]; rvt_put_mr(sge->mr); } return ret; }