static
int _pscom_openib_do_read(pscom_con_t *con, psoib_con_info_t *mcon)
{
void *buf;
ssize_t size;
size = psoib_recvlook(mcon, &buf);
if (size >= 0) {
perf_add("openib_do_read");
pscom_read_done(con, buf, size);
psoib_recvdone(mcon);
return 1;
} else if ((size == -EINTR) || (size == -EAGAIN)) {
// Nothing received
return 0;
} else {
// Error
errno = -size;
pscom_con_error(con, PSCOM_OP_READ, PSCOM_ERR_STDERROR);
return 1;
}
}
static inline
void x_send(unsigned msgsize)
{
int rc;
mem_info_t *send_bufs = &mcon->send.bufs;
volatile uint32_t *mark = send_bufs->ptr;
*mark = 0x42424242;
struct ibv_sge list = {
.addr = (uintptr_t) mark,
.length = msgsize,
.lkey = send_bufs->mr->lkey,
};
struct ibv_send_wr wr = {
.next = NULL,
.wr_id = (uint64_t)mcon,
.sg_list = &list,
.num_sge = 1,
.opcode = IBV_WR_RDMA_WRITE,
.send_flags = (
(ENABLE_SEND_NOTIFICATION ? IBV_SEND_SIGNALED : 0) | /* no cq entry, if unsignaled */
((list.length <= IB_MAX_INLINE) ? IBV_SEND_INLINE : 0)),
.imm_data = 42117,
.wr.rdma = {
.remote_addr = (uint64_t)mcon->remote_ptr + arg_recvoffset,
.rkey = mcon->remote_rkey,
},
};
struct ibv_send_wr *bad_wr;
rc = ibv_post_send(mcon->qp, &wr, &bad_wr);
psoib_rc_check("ibv_post_send", rc);
{
/* poll on cq */
struct ibv_wc wc;
do {
rc = ibv_poll_cq(default_hca.cq, 1, &wc);
if (rc > 0) {
if (wc.status != IBV_WC_SUCCESS) {
fprintf(stderr, "Completion with error\n");
fprintf(stderr, "Failed status %d: wr_id %d\n",
wc.status, (int) wc.wr_id);
exit(1);
}
} else if (rc < 0) {
fprintf(stderr, "poll CQ failed %d\n", rc);
exit(1);
} // else: rc == 0
} while (rc);
}
}
static
int x_recv_pscom(void)
{
void *buf;
int size;
while (1) {
size = psoib_recvlook(mcon, &buf);
if (size >= 0) {
psoib_recvdone(mcon);
return size;
} else if ((size == -EINTR) || (size == -EAGAIN)) {
continue;
} else {
// Error
errno = -size;
perror("psoib_recvlook");
exit(1);
}
}
}
