int iser_post_recvm(struct iser_conn *ib_conn, int count)
{
struct ib_recv_wr *rx_wr, *rx_wr_failed;
int i, ib_ret;
unsigned int my_rx_head = ib_conn->rx_desc_head;
struct iser_rx_desc *rx_desc;
for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
rx_desc = &ib_conn->rx_descs[my_rx_head];
rx_wr->wr_id = (unsigned long)rx_desc;
rx_wr->sg_list = &rx_desc->rx_sg;
rx_wr->num_sge = 1;
rx_wr->next = rx_wr + 1;
my_rx_head = (my_rx_head + 1) & (ISER_QP_MAX_RECV_DTOS - 1);
}
rx_wr--;
rx_wr->next = NULL; /* mark end of work requests list */
ib_conn->post_recv_buf_count += count;
ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, &rx_wr_failed);
if (ib_ret) {
iser_err("ib_post_recv failed ret=%d\n", ib_ret);
ib_conn->post_recv_buf_count -= count;
} else
ib_conn->rx_desc_head = my_rx_head;
return ib_ret;
}
static int
post_recv(struct p9_client *client, struct p9_rdma_context *c)
{
struct p9_trans_rdma *rdma = client->trans;
struct ib_recv_wr wr, *bad_wr;
struct ib_sge sge;
c->busa = ib_dma_map_single(rdma->cm_id->device,
c->rc->sdata, client->msize,
DMA_FROM_DEVICE);
if (ib_dma_mapping_error(rdma->cm_id->device, c->busa))
goto error;
sge.addr = c->busa;
sge.length = client->msize;
sge.lkey = rdma->lkey;
wr.next = NULL;
c->wc_op = IB_WC_RECV;
wr.wr_id = (unsigned long) c;
wr.sg_list = &sge;
wr.num_sge = 1;
return ib_post_recv(rdma->qp, &wr, &bad_wr);
error:
p9_debug(P9_DEBUG_ERROR, "EIO\n");
return -EIO;
}
void IBInterface::modiyQueuePairAttribute( )
{
if ( !_setAttributeReadyToReceive() )
return;
if ( !_setAttributeReadyToSend( ))
return;
_completionQueue->triggerRead();
ResetEvent( _completionQueue->getReadNotifier());
for ( int i = 0; i < EQ_NUMBLOCKMEMORY ; i++ )
{
_recvList.vaddr = _readBlocks[ i ]->getVaddr();
_recvList.length = EQ_MAXBLOCKBUFFER;
_recvList.lkey = _readBlocks[ i ]->getLocalKey();
//receive
_rwr.wr_id = i;
_rwr.ds_array = &_recvList;
_rwr.num_ds = 1;
_rwr.p_next = 0;
ib_api_status_t ibStatus = ib_post_recv( _queuePair, &_rwr, 0);
_readPoll.getData()[ i ];
}
}
int iser_post_recv(struct iser_desc *rx_desc)
{
int ib_ret, ret_val = 0;
struct ib_recv_wr recv_wr, *recv_wr_failed;
struct ib_sge iov[2];
struct iser_conn *ib_conn;
struct iser_dto *recv_dto = &rx_desc->dto;
ib_conn = recv_dto->ib_conn;
iser_dto_to_iov(recv_dto, iov, 2);
recv_wr.next = NULL;
recv_wr.sg_list = iov;
recv_wr.num_sge = recv_dto->regd_vector_len;
recv_wr.wr_id = (unsigned long)rx_desc;
atomic_inc(&ib_conn->post_recv_buf_count);
ib_ret = ib_post_recv(ib_conn->qp, &recv_wr, &recv_wr_failed);
if (ib_ret) {
iser_err("ib_post_recv failed ret=%d\n", ib_ret);
atomic_dec(&ib_conn->post_recv_buf_count);
ret_val = -1;
}
return ret_val;
}
static int
post_recv(struct p9_client *client, struct p9_rdma_context *c)
{
struct p9_trans_rdma *rdma = client->trans;
struct ib_recv_wr wr;
struct ib_sge sge;
c->busa = ib_dma_map_single(rdma->cm_id->device,
c->rc.sdata, client->msize,
DMA_FROM_DEVICE);
if (ib_dma_mapping_error(rdma->cm_id->device, c->busa))
goto error;
c->cqe.done = recv_done;
sge.addr = c->busa;
sge.length = client->msize;
sge.lkey = rdma->pd->local_dma_lkey;
wr.next = NULL;
wr.wr_cqe = &c->cqe;
wr.sg_list = &sge;
wr.num_sge = 1;
return ib_post_recv(rdma->qp, &wr, NULL);
error:
p9_debug(P9_DEBUG_ERROR, "EIO\n");
return -EIO;
}
static int ipoib_cm_post_receive_nonsrq(struct net_device *dev,
struct ipoib_cm_rx *rx,
struct ib_recv_wr *wr,
struct ib_sge *sge, int id)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
struct ib_recv_wr *bad_wr;
int i, ret;
wr->wr_id = id | IPOIB_OP_CM | IPOIB_OP_RECV;
for (i = 0; i < IPOIB_CM_RX_SG; ++i)
sge[i].addr = rx->rx_ring[id].mapping[i];
ret = ib_post_recv(rx->qp, wr, &bad_wr);
if (unlikely(ret)) {
ipoib_warn(priv, "post recv failed for buf %d (%d)\n", id, ret);
ipoib_cm_dma_unmap_rx(priv, IPOIB_CM_RX_SG - 1,
rx->rx_ring[id].mapping);
dev_kfree_skb_any(rx->rx_ring[id].skb);
rx->rx_ring[id].skb = NULL;
}
return ret;
}
static int
svc_rdma_post_recv(struct svcxprt_rdma *xprt)
{
struct ib_recv_wr recv_wr, *bad_recv_wr;
struct svc_rdma_op_ctxt *ctxt;
struct page *page;
dma_addr_t pa;
int sge_no;
int buflen;
int ret;
ctxt = svc_rdma_get_context(xprt);
buflen = 0;
ctxt->direction = DMA_FROM_DEVICE;
ctxt->cqe.done = svc_rdma_wc_receive;
for (sge_no = 0; buflen < xprt->sc_max_req_size; sge_no++) {
if (sge_no >= xprt->sc_max_sge) {
pr_err("svcrdma: Too many sges (%d)\n", sge_no);
goto err_put_ctxt;
}
page = alloc_page(GFP_KERNEL);
if (!page)
goto err_put_ctxt;
ctxt->pages[sge_no] = page;
pa = ib_dma_map_page(xprt->sc_cm_id->device,
page, 0, PAGE_SIZE,
DMA_FROM_DEVICE);
if (ib_dma_mapping_error(xprt->sc_cm_id->device, pa))
goto err_put_ctxt;
svc_rdma_count_mappings(xprt, ctxt);
ctxt->sge[sge_no].addr = pa;
ctxt->sge[sge_no].length = PAGE_SIZE;
ctxt->sge[sge_no].lkey = xprt->sc_pd->local_dma_lkey;
ctxt->count = sge_no + 1;
buflen += PAGE_SIZE;
}
recv_wr.next = NULL;
recv_wr.sg_list = &ctxt->sge[0];
recv_wr.num_sge = ctxt->count;
recv_wr.wr_cqe = &ctxt->cqe;
svc_xprt_get(&xprt->sc_xprt);
ret = ib_post_recv(xprt->sc_qp, &recv_wr, &bad_recv_wr);
if (ret) {
svc_rdma_unmap_dma(ctxt);
svc_rdma_put_context(ctxt, 1);
svc_xprt_put(&xprt->sc_xprt);
}
return ret;
err_put_ctxt:
svc_rdma_unmap_dma(ctxt);
svc_rdma_put_context(ctxt, 1);
return -ENOMEM;
}
static void krping_run_server(struct krping_cb *cb)
{
struct ib_recv_wr *bad_wr;
int ret;
ret = krping_bind_server(cb);
if (ret)
return;
ret = krping_setup_qp(cb, cb->child_cm_id);
if (ret) {
log(LOG_ERR, "setup_qp failed: %d\n", ret);
return;
}
ret = krping_setup_buffers(cb);
if (ret) {
log(LOG_ERR, "krping_setup_buffers failed: %d\n", ret);
goto err1;
}
ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr);
if (ret) {
log(LOG_ERR, "ib_post_recv failed: %d\n", ret);
goto err2;
}
ret = krping_accept(cb);
if (ret) {
log(LOG_ERR, "connect error %d\n", ret);
goto err2;
}
if (cb->wlat)
krping_wlat_test_server(cb);
else if (cb->rlat)
krping_rlat_test_server(cb);
else if (cb->bw)
krping_bw_test_server(cb);
else
krping_test_server(cb);
rdma_disconnect(cb->child_cm_id);
rdma_destroy_id(cb->child_cm_id);
err2:
krping_free_buffers(cb);
err1:
krping_free_qp(cb);
}
int svc_rdma_post_recv(struct svcxprt_rdma *xprt)
{
struct ib_recv_wr recv_wr, *bad_recv_wr;
struct svc_rdma_op_ctxt *ctxt;
struct page *page;
dma_addr_t pa;
int sge_no;
int buflen;
int ret;
ctxt = svc_rdma_get_context(xprt);
buflen = 0;
ctxt->direction = DMA_FROM_DEVICE;
for (sge_no = 0; buflen < xprt->sc_max_req_size; sge_no++) {
BUG_ON(sge_no >= xprt->sc_max_sge);
page = svc_rdma_get_page();
ctxt->pages[sge_no] = page;
pa = ib_dma_map_page(xprt->sc_cm_id->device,
page, 0, PAGE_SIZE,
DMA_FROM_DEVICE);
if (ib_dma_mapping_error(xprt->sc_cm_id->device, pa))
goto err_put_ctxt;
atomic_inc(&xprt->sc_dma_used);
ctxt->sge[sge_no].addr = pa;
ctxt->sge[sge_no].length = PAGE_SIZE;
ctxt->sge[sge_no].lkey = xprt->sc_dma_lkey;
ctxt->count = sge_no + 1;
buflen += PAGE_SIZE;
}
recv_wr.next = NULL;
recv_wr.sg_list = &ctxt->sge[0];
recv_wr.num_sge = ctxt->count;
recv_wr.wr_id = (u64)(unsigned long)ctxt;
svc_xprt_get(&xprt->sc_xprt);
ret = ib_post_recv(xprt->sc_qp, &recv_wr, &bad_recv_wr);
if (ret) {
svc_rdma_unmap_dma(ctxt);
svc_rdma_put_context(ctxt, 1);
svc_xprt_put(&xprt->sc_xprt);
}
return ret;
err_put_ctxt:
svc_rdma_unmap_dma(ctxt);
svc_rdma_put_context(ctxt, 1);
return -ENOMEM;
}
static int __svc_rdma_post_recv(struct svcxprt_rdma *rdma,
struct svc_rdma_recv_ctxt *ctxt)
{
int ret;
svc_xprt_get(&rdma->sc_xprt);
ret = ib_post_recv(rdma->sc_qp, &ctxt->rc_recv_wr, NULL);
trace_svcrdma_post_recv(&ctxt->rc_recv_wr, ret);
if (ret)
goto err_post;
return 0;
err_post:
svc_rdma_recv_ctxt_put(rdma, ctxt);
svc_xprt_put(&rdma->sc_xprt);
return ret;
}
bool IBInterface::_ibPostRecv( uint32_t numBuffer )
{
_recvList.vaddr = _readBlocks[ numBuffer ]->getVaddr();
_recvList.length = EQ_MAXBLOCKBUFFER;
_recvList.lkey = _readBlocks[ numBuffer ]->getLocalKey();
//receive
_rwr.wr_id = numBuffer;
_rwr.ds_array = &_recvList;
_completionQueue->resetEventRead();
eq::lunchbox::ScopedMutex mutex( _completionQueue->_mutex );
ib_api_status_t ibStatus = ib_post_recv( _queuePair, &_rwr, 0);
if ( ibStatus != IB_SUCCESS )
{
LBERROR << "Error during ib_post_recv" << std::endl;
return false;
}
return true;
}
static void verbs_post_recv_req (void)
{
struct ib_recv_wr wr, *bad_wr;
struct ib_sge sge;
int ret;
static int id = 1;
memset (&wr, 0, sizeof (wr));
wr.wr_id = id++;
wr.num_sge = 1;
wr.sg_list = &sge;
sge.addr = recv_key;
sge.length = buf_size + 40;
sge.lkey = mr->lkey;
ret = ib_post_recv (qp, &wr, &bad_wr);
if (ret)
printk (KERN_INFO "post_recv failed: %d\n", ret);
}
int iser_post_recvl(struct iser_conn *ib_conn)
{
struct ib_recv_wr rx_wr, *rx_wr_failed;
struct ib_sge sge;
int ib_ret;
sge.addr = ib_conn->login_dma;
sge.length = ISER_RX_LOGIN_SIZE;
sge.lkey = ib_conn->device->mr->lkey;
rx_wr.wr_id = (unsigned long)ib_conn->login_buf;
rx_wr.sg_list = &sge;
rx_wr.num_sge = 1;
rx_wr.next = NULL;
ib_conn->post_recv_buf_count++;
ib_ret = ib_post_recv(ib_conn->qp, &rx_wr, &rx_wr_failed);
if (ib_ret) {
iser_err("ib_post_recv failed ret=%d\n", ib_ret);
ib_conn->post_recv_buf_count--;
}
return ib_ret;
}
int isert_post_recv(struct isert_connection *isert_conn,
struct isert_wr *first_wr,
int num_wr)
{
struct ib_recv_wr *first_ib_wr = &first_wr->recv_wr;
struct ib_recv_wr *bad_wr;
int num_posted;
int err;
TRACE_ENTRY();
err = ib_post_recv(isert_conn->qp, first_ib_wr, &bad_wr);
if (unlikely(err)) {
num_posted = isert_num_recv_posted_on_err(first_ib_wr, bad_wr);
pr_err("conn:%p recv posted:%d/%d 1st wr_id:0x%llx sz:%d err:%d\n",
isert_conn, num_posted, num_wr, first_ib_wr->wr_id,
first_ib_wr->sg_list->length, err);
}
TRACE_EXIT_RES(err);
return err;
}
static void krping_cq_event_handler(struct ib_cq *cq, void *ctx)
{
struct krping_cb *cb = ctx;
struct ib_wc wc;
struct ib_recv_wr *bad_wr;
int ret;
mtx_lock(&cb->lock);
KASSERT(cb->cq == cq, ("bad condition"));
if (cb->state == ERROR) {
log(LOG_ERR, "cq completion in ERROR state\n");
mtx_unlock(&cb->lock);
return;
}
if (!cb->wlat && !cb->rlat && !cb->bw)
ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP);
while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) {
if (wc.status) {
if (wc.status == IB_WC_WR_FLUSH_ERR) {
DEBUG_LOG("cq flushed\n");
continue;
} else {
log(LOG_CRIT, "cq completion failed status %d\n",
wc.status);
goto error;
}
}
switch (wc.opcode) {
case IB_WC_SEND:
DEBUG_LOG(PFX "send completion\n");
cb->stats.send_bytes += cb->send_sgl.length;
cb->stats.send_msgs++;
break;
case IB_WC_RDMA_WRITE:
DEBUG_LOG(PFX "rdma write completion\n");
cb->stats.write_bytes += cb->rdma_sq_wr.sg_list->length;
cb->stats.write_msgs++;
cb->state = RDMA_WRITE_COMPLETE;
wakeup(cb);
break;
case IB_WC_RDMA_READ:
DEBUG_LOG(PFX "rdma read completion\n");
cb->stats.read_bytes += cb->rdma_sq_wr.sg_list->length;
cb->stats.read_msgs++;
cb->state = RDMA_READ_COMPLETE;
wakeup(cb);
break;
case IB_WC_RECV:
DEBUG_LOG(PFX "recv completion\n");
cb->stats.recv_bytes += sizeof(cb->recv_buf);
cb->stats.recv_msgs++;
if (cb->wlat || cb->rlat || cb->bw)
ret = server_recv(cb, &wc);
else
ret = cb->server ? server_recv(cb, &wc) :
client_recv(cb, &wc);
if (ret) {
log(LOG_ERR, "recv wc error: %d\n", ret);
goto error;
}
ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr);
if (ret) {
log(LOG_ERR, "post recv error: %d\n",
ret);
goto error;
}
wakeup(cb);
break;
default:
log(LOG_ERR, "unknown!!!!! completion\n");
goto error;
}
}
if (ret) {
log(LOG_ERR, "poll error %d\n", ret);
goto error;
}
mtx_unlock(&cb->lock);
return;
error:
cb->state = ERROR;
wakeup(cb);
mtx_unlock(&cb->lock);
}
static int
sdp_post_recv(struct sdp_sock *ssk)
{
struct sdp_buf *rx_req;
int i, rc;
u64 addr;
struct ib_device *dev;
struct ib_recv_wr rx_wr = { NULL };
struct ib_sge ibsge[SDP_MAX_RECV_SGES];
struct ib_sge *sge = ibsge;
struct ib_recv_wr *bad_wr;
struct mbuf *mb, *m;
struct sdp_bsdh *h;
int id = ring_head(ssk->rx_ring);
/* Now, allocate and repost recv */
sdp_prf(ssk->socket, mb, "Posting mb");
mb = m_getm2(NULL, ssk->recv_bytes, M_NOWAIT, MT_DATA, M_PKTHDR);
if (mb == NULL) {
/* Retry so we can't stall out with no memory. */
if (!rx_ring_posted(ssk))
queue_work(rx_comp_wq, &ssk->rx_comp_work);
return -1;
}
for (m = mb; m != NULL; m = m->m_next) {
m->m_len = (m->m_flags & M_EXT) ? m->m_ext.ext_size :
((m->m_flags & M_PKTHDR) ? MHLEN : MLEN);
mb->m_pkthdr.len += m->m_len;
}
h = mtod(mb, struct sdp_bsdh *);
rx_req = ssk->rx_ring.buffer + (id & (SDP_RX_SIZE - 1));
rx_req->mb = mb;
dev = ssk->ib_device;
for (i = 0; mb != NULL; i++, mb = mb->m_next, sge++) {
addr = ib_dma_map_single(dev, mb->m_data, mb->m_len,
DMA_TO_DEVICE);
/* TODO: proper error handling */
BUG_ON(ib_dma_mapping_error(dev, addr));
BUG_ON(i >= SDP_MAX_RECV_SGES);
rx_req->mapping[i] = addr;
sge->addr = addr;
sge->length = mb->m_len;
sge->lkey = ssk->sdp_dev->mr->lkey;
}
rx_wr.next = NULL;
rx_wr.wr_id = id | SDP_OP_RECV;
rx_wr.sg_list = ibsge;
rx_wr.num_sge = i;
rc = ib_post_recv(ssk->qp, &rx_wr, &bad_wr);
if (unlikely(rc)) {
sdp_warn(ssk->socket, "ib_post_recv failed. status %d\n", rc);
sdp_cleanup_sdp_buf(ssk, rx_req, DMA_FROM_DEVICE);
m_freem(mb);
sdp_notify(ssk, ECONNRESET);
return -1;
}
atomic_inc(&ssk->rx_ring.head);
SDPSTATS_COUNTER_INC(post_recv);
return 0;
}