int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context, struct ib_udata *udata) { struct ib_device *ib_dev = ibpd->device; struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); struct device *dev = hr_dev->dev; struct hns_roce_pd *pd = to_hr_pd(ibpd); int ret; ret = hns_roce_pd_alloc(to_hr_dev(ib_dev), &pd->pdn); if (ret) { dev_err(dev, "[alloc_pd]hns_roce_pd_alloc failed!\n"); return ret; } if (context) { struct hns_roce_ib_alloc_pd_resp uresp = {.pdn = pd->pdn}; if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { hns_roce_pd_free(to_hr_dev(ib_dev), pd->pdn); dev_err(dev, "[alloc_pd]ib_copy_to_udata failed!\n"); return -EFAULT; } } return 0; } EXPORT_SYMBOL_GPL(hns_roce_alloc_pd); void hns_roce_dealloc_pd(struct ib_pd *pd) { hns_roce_pd_free(to_hr_dev(pd->device), to_hr_pd(pd)->pdn); }
static void hns_roce_ib_cq_event(struct hns_roce_cq *hr_cq, enum hns_roce_event event_type) { struct hns_roce_dev *hr_dev; struct ib_event event; struct ib_cq *ibcq; ibcq = &hr_cq->ib_cq; hr_dev = to_hr_dev(ibcq->device); if (event_type != HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID && event_type != HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR && event_type != HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW) { dev_err(hr_dev->dev, "hns_roce_ib: Unexpected event type 0x%x on CQ %06lx\n", event_type, hr_cq->cqn); return; } if (ibcq->event_handler) { event.device = ibcq->device; event.event = IB_EVENT_CQ_ERR; event.element.cq = ibcq; ibcq->event_handler(&event, ibcq->cq_context); } }
int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); int ret = 0; if (hr_dev->hw->destroy_cq) { ret = hr_dev->hw->destroy_cq(ib_cq); } else { hns_roce_free_cq(hr_dev, hr_cq); hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); if (ib_cq->uobject) { ib_umem_release(hr_cq->umem); if (hr_cq->db_en == 1) hns_roce_db_unmap_user( to_hr_ucontext(ib_cq->uobject->context), &hr_cq->db); } else { /* Free the buff of stored cq */ hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, ib_cq->cqe); if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) hns_roce_free_db(hr_dev, &hr_cq->db); } kfree(hr_cq); } return ret; }
static int hns_roce_u_v1_poll_cq(struct ibv_cq *ibvcq, int ne, struct ibv_wc *wc) { int npolled; int err = CQ_OK; struct hns_roce_qp *qp = NULL; struct hns_roce_cq *cq = to_hr_cq(ibvcq); struct hns_roce_context *ctx = to_hr_ctx(ibvcq->context); struct hns_roce_device *dev = to_hr_dev(ibvcq->context->device); pthread_spin_lock(&cq->lock); for (npolled = 0; npolled < ne; ++npolled) { err = hns_roce_v1_poll_one(cq, &qp, wc + npolled); if (err != CQ_OK) break; } if (npolled) { if (dev->hw_version == HNS_ROCE_HW_VER1) { *cq->set_ci_db = (cq->cons_index & ((cq->cq_depth << 1) - 1)); mmio_ordered_writes_hack(); } hns_roce_update_cq_cons_index(ctx, cq); } pthread_spin_unlock(&cq->lock); return err == CQ_POLL_ERR ? err : npolled; }
struct ib_ah *hns_roce_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *ah_attr, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibpd->device); const struct ib_gid_attr *gid_attr; struct device *dev = hr_dev->dev; struct hns_roce_ah *ah; u16 vlan_tag = 0xffff; const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); bool vlan_en = false; ah = kzalloc(sizeof(*ah), GFP_ATOMIC); if (!ah) return ERR_PTR(-ENOMEM); /* Get mac address */ memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN); gid_attr = ah_attr->grh.sgid_attr; if (is_vlan_dev(gid_attr->ndev)) { vlan_tag = vlan_dev_vlan_id(gid_attr->ndev); vlan_en = true; } if (vlan_tag < 0x1000) vlan_tag |= (rdma_ah_get_sl(ah_attr) & HNS_ROCE_VLAN_SL_BIT_MASK) << HNS_ROCE_VLAN_SL_SHIFT; ah->av.port_pd = cpu_to_be32(to_hr_pd(ibpd)->pdn | (rdma_ah_get_port_num(ah_attr) << HNS_ROCE_PORT_NUM_SHIFT)); ah->av.gid_index = grh->sgid_index; ah->av.vlan = cpu_to_le16(vlan_tag); ah->av.vlan_en = vlan_en; dev_dbg(dev, "gid_index = 0x%x,vlan = 0x%x\n", ah->av.gid_index, ah->av.vlan); if (rdma_ah_get_static_rate(ah_attr)) ah->av.stat_rate = IB_RATE_10_GBPS; memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE); ah->av.sl_tclass_flowlabel = cpu_to_le32(rdma_ah_get_sl(ah_attr) << HNS_ROCE_SL_SHIFT); return &ah->ibah; }
struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, const struct ib_cq_init_attr *attr, struct ib_ucontext *context, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); struct device *dev = hr_dev->dev; struct hns_roce_ib_create_cq ucmd; struct hns_roce_ib_create_cq_resp resp = {}; struct hns_roce_cq *hr_cq = NULL; struct hns_roce_uar *uar = NULL; int vector = attr->comp_vector; int cq_entries = attr->cqe; int ret; if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) { dev_err(dev, "Creat CQ failed. entries=%d, max=%d\n", cq_entries, hr_dev->caps.max_cqes); return ERR_PTR(-EINVAL); } hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL); if (!hr_cq) return ERR_PTR(-ENOMEM); if (hr_dev->caps.min_cqes) cq_entries = max(cq_entries, hr_dev->caps.min_cqes); cq_entries = roundup_pow_of_two((unsigned int)cq_entries); hr_cq->ib_cq.cqe = cq_entries - 1; spin_lock_init(&hr_cq->lock); if (context) { if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { dev_err(dev, "Failed to copy_from_udata.\n"); ret = -EFAULT; goto err_cq; } /* Get user space address, write it into mtt table */ ret = hns_roce_ib_get_cq_umem(hr_dev, udata, &hr_cq->hr_buf, &hr_cq->umem, ucmd.buf_addr, cq_entries); if (ret) { dev_err(dev, "Failed to get_cq_umem.\n"); goto err_cq; } if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && (udata->outlen >= sizeof(resp))) { ret = hns_roce_db_map_user(to_hr_ucontext(context), udata, ucmd.db_addr, &hr_cq->db); if (ret) { dev_err(dev, "cq record doorbell map failed!\n"); goto err_mtt; } hr_cq->db_en = 1; resp.cap_flags |= HNS_ROCE_SUPPORT_CQ_RECORD_DB; } /* Get user space parameters */ uar = &to_hr_ucontext(context)->uar; } else { if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) { ret = hns_roce_alloc_db(hr_dev, &hr_cq->db, 1); if (ret) goto err_cq; hr_cq->set_ci_db = hr_cq->db.db_record; *hr_cq->set_ci_db = 0; hr_cq->db_en = 1; } /* Init mmt table and write buff address to mtt table */ ret = hns_roce_ib_alloc_cq_buf(hr_dev, &hr_cq->hr_buf, cq_entries); if (ret) { dev_err(dev, "Failed to alloc_cq_buf.\n"); goto err_db; } uar = &hr_dev->priv_uar; hr_cq->cq_db_l = hr_dev->reg_base + hr_dev->odb_offset + DB_REG_OFFSET * uar->index; } /* Allocate cq index, fill cq_context */ ret = hns_roce_cq_alloc(hr_dev, cq_entries, &hr_cq->hr_buf.hr_mtt, uar, hr_cq, vector); if (ret) { dev_err(dev, "Creat CQ .Failed to cq_alloc.\n"); goto err_dbmap; } /* * For the QP created by kernel space, tptr value should be initialized * to zero; For the QP created by user space, it will cause synchronous * problems if tptr is set to zero here, so we initialze it in user * space. */ if (!context && hr_cq->tptr_addr) *hr_cq->tptr_addr = 0; /* Get created cq handler and carry out event */ hr_cq->comp = hns_roce_ib_cq_comp; hr_cq->event = hns_roce_ib_cq_event; hr_cq->cq_depth = cq_entries; if (context) { resp.cqn = hr_cq->cqn; ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); if (ret) goto err_cqc; } return &hr_cq->ib_cq; err_cqc: hns_roce_free_cq(hr_dev, hr_cq); err_dbmap: if (context && (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && (udata->outlen >= sizeof(resp))) hns_roce_db_unmap_user(to_hr_ucontext(context), &hr_cq->db); err_mtt: hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); if (context) ib_umem_release(hr_cq->umem); else hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, hr_cq->ib_cq.cqe); err_db: if (!context && (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)) hns_roce_free_db(hr_dev, &hr_cq->db); err_cq: kfree(hr_cq); return ERR_PTR(ret); }
int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); enum ib_qp_state cur_state, new_state; struct device *dev = hr_dev->dev; int ret = -EINVAL; int p; enum ib_mtu active_mtu; mutex_lock(&hr_qp->mutex); cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : (enum ib_qp_state)hr_qp->state; new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask, IB_LINK_LAYER_ETHERNET)) { dev_err(dev, "ib_modify_qp_is_ok failed\n"); goto out; } if ((attr_mask & IB_QP_PORT) && (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { dev_err(dev, "attr port_num invalid.attr->port_num=%d\n", attr->port_num); goto out; } if (attr_mask & IB_QP_PKEY_INDEX) { p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { dev_err(dev, "attr pkey_index invalid.attr->pkey_index=%d\n", attr->pkey_index); goto out; } } if (attr_mask & IB_QP_PATH_MTU) { p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); if ((hr_dev->caps.max_mtu == IB_MTU_4096 && attr->path_mtu > IB_MTU_4096) || (hr_dev->caps.max_mtu == IB_MTU_2048 && attr->path_mtu > IB_MTU_2048) || attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { dev_err(dev, "attr path_mtu(%d)invalid while modify qp", attr->path_mtu); goto out; } } if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { dev_err(dev, "attr max_rd_atomic invalid.attr->max_rd_atomic=%d\n", attr->max_rd_atomic); goto out; } if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { dev_err(dev, "attr max_dest_rd_atomic invalid.attr->max_dest_rd_atomic=%d\n", attr->max_dest_rd_atomic); goto out; } if (cur_state == new_state && cur_state == IB_QPS_RESET) { ret = 0; goto out; } ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, new_state); out: mutex_unlock(&hr_qp->mutex); return ret; }
struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); struct device *dev = hr_dev->dev; struct hns_roce_sqp *hr_sqp; struct hns_roce_qp *hr_qp; int ret; switch (init_attr->qp_type) { case IB_QPT_RC: { hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); if (!hr_qp) return ERR_PTR(-ENOMEM); ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 0, hr_qp); if (ret) { dev_err(dev, "Create RC QP failed\n"); kfree(hr_qp); return ERR_PTR(ret); } hr_qp->ibqp.qp_num = hr_qp->qpn; break; } case IB_QPT_GSI: { /* Userspace is not allowed to create special QPs: */ if (pd->uobject) { dev_err(dev, "not support usr space GSI\n"); return ERR_PTR(-EINVAL); } hr_sqp = kzalloc(sizeof(*hr_sqp), GFP_KERNEL); if (!hr_sqp) return ERR_PTR(-ENOMEM); hr_qp = &hr_sqp->hr_qp; hr_qp->port = init_attr->port_num - 1; hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; /* when hw version is v1, the sqpn is allocated */ if (hr_dev->caps.max_sq_sg <= 2) hr_qp->ibqp.qp_num = HNS_ROCE_MAX_PORTS + hr_dev->iboe.phy_port[hr_qp->port]; else hr_qp->ibqp.qp_num = 1; ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp->ibqp.qp_num, hr_qp); if (ret) { dev_err(dev, "Create GSI QP failed!\n"); kfree(hr_sqp); return ERR_PTR(ret); } break; } default:{ dev_err(dev, "not support QP type %d\n", init_attr->qp_type); return ERR_PTR(-EINVAL); } } return &hr_qp->ibqp; }