void do_multirecv(int len) { int i, ret; ssize_t sz; struct fi_cq_tagged_entry s_cqe, d_cqe; struct iovec iov; struct fi_msg msg; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; int nrecvs = 3; rdm_sr_init_data(source, len, 0xab); rdm_sr_init_data(target, len, 0); /* Post receives first to force matching in SMSG callback. */ iov.iov_base = target; iov.iov_len = len * nrecvs + 63; msg.msg_iov = &iov; msg.desc = (void **)rem_mr; msg.iov_count = 1; msg.addr = gni_addr[0]; msg.context = source; msg.data = (uint64_t)source; sz = fi_recvmsg(ep[1], &msg, FI_MULTI_RECV); cr_assert_eq(sz, 0); for (i = 0; i < nrecvs; i++) { sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target); cr_assert_eq(sz, 0); } /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(msg_cq[0], &s_cqe, 1); if (ret == 1) { rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0); s[0]++; } ret = fi_cq_read(msg_cq[1], &d_cqe, 1); if (ret == 1) { rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_MULTI_RECV), target + (r[1] * len), len, 0); cr_assert(rdm_sr_check_data(source, d_cqe.buf, len), "Data mismatch"); r[1]++; } } while (s[0] < nrecvs || r[1] < nrecvs); rdm_sr_check_cntrs(s, r, s_e, r_e); dbg_printf("got context events!\n"); }
void do_write_fence(int len) { int ret; ssize_t sz; struct fi_cq_tagged_entry cqe; struct iovec iov; struct fi_msg_rma msg; struct fi_rma_iov rma_iov; iov.iov_base = source; iov.iov_len = len; rma_iov.addr = (uint64_t)target; rma_iov.len = sizeof(target); rma_iov.key = mr_key; msg.msg_iov = &iov; msg.desc = (void **)&loc_mr; msg.iov_count = 1; msg.addr = gni_addr[1]; msg.rma_iov = &rma_iov; msg.rma_iov_count = 1; msg.context = target; msg.data = (uint64_t)target; init_data(source, len, 0xef); init_data(target, len, 0); /* write A */ sz = fi_writemsg(ep[0], &msg, 0); cr_assert_eq(sz, 0); /* write B */ sz = fi_writemsg(ep[0], &msg, FI_FENCE); cr_assert_eq(sz, 0); /* event A */ while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0); /* event B */ while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0); rdm_rma_check_cntrs(2, 0, 0, 0); dbg_printf("got write context event!\n"); cr_assert(check_data(source, target, len), "Data mismatch"); }
/* ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc, size_t count, fi_addr_t src_addr, void *context); */ void do_recvv(int len) { int i, ret, iov_cnt; ssize_t sz; int source_done = 0, dest_done = 0; struct fi_cq_tagged_entry s_cqe, d_cqe; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; sz = fi_recvv(ep[1], NULL, NULL, IOV_CNT, gni_addr[0], iov_src_buf); cr_assert_eq(sz, -FI_EINVAL); sz = fi_recvv(ep[1], dest_iov, NULL, IOV_CNT + 1, gni_addr[0], iov_src_buf); cr_assert_eq(sz, -FI_EINVAL); for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) { rdm_sr_init_data(iov_src_buf, len * iov_cnt, 0xab); for (i = 0; i < iov_cnt; i++) { rdm_sr_init_data(dest_iov[i].iov_base, len, 0); dest_iov[i].iov_len = len; } sz = fi_send(ep[0], iov_src_buf, len * iov_cnt, NULL, gni_addr[1], dest_iov); cr_assert_eq(sz, 0); sz = fi_recvv(ep[1], dest_iov, NULL, iov_cnt, gni_addr[0], iov_src_buf); cr_assert_eq(sz, 0); /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(msg_cq[0], &s_cqe, 1); if (ret == 1) { source_done = 1; } ret = fi_cq_read(msg_cq[1], &d_cqe, 1); if (ret == 1) { dest_done = 1; } } while (!(source_done && dest_done)); rdm_sr_check_cqe(&s_cqe, dest_iov, (FI_MSG|FI_SEND), 0, 0, 0, false); rdm_sr_check_cqe(&d_cqe, iov_src_buf, (FI_MSG|FI_RECV), dest_iov, len * iov_cnt, 0, false); s[0] = 1; r[1] = 1; rdm_sr_check_cntrs(s, r, s_e, r_e); dbg_printf("got context events!\n"); cr_assert(rdm_sr_check_iov_data(dest_iov, iov_src_buf, iov_cnt, len * iov_cnt), "Data mismatch"); source_done = dest_done = 0; } }
void do_sendrecvv_alignment(int slen, int dlen, int offset) { int i, ret, iov_cnt; ssize_t sz; int source_done = 0, dest_done = 0; struct fi_cq_tagged_entry s_cqe, d_cqe; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; uint64_t iov_s_buf = (uint64_t) iov_src_buf; iov_s_buf += offset; for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) { for (i = 0; i < iov_cnt; i++) { d_iov[i].iov_base = dest_iov[i].iov_base; d_iov[i].iov_base = (void *) ((uint64_t)d_iov[i].iov_base + offset); rdm_sr_init_data(d_iov[i].iov_base, dlen - offset, 0); d_iov[i].iov_len = dlen - offset; } rdm_sr_init_data((void *) iov_s_buf, (slen - offset) * iov_cnt, 0xab); sz = fi_send(ep[0], (void *) iov_s_buf, (slen - offset) * iov_cnt, NULL, gni_addr[1], d_iov); cr_assert_eq(sz, 0); sz = fi_recvv(ep[1], d_iov, NULL, iov_cnt, gni_addr[0], (void *) iov_s_buf); cr_assert_eq(sz, 0); /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(msg_cq[0], &s_cqe, 1); if (ret == 1) { source_done = 1; } ret = fi_cq_read(msg_cq[1], &d_cqe, 1); if (ret == 1) { dest_done = 1; } } while (!(source_done && dest_done)); rdm_sr_check_cqe(&s_cqe, d_iov, (FI_MSG|FI_SEND), 0, 0, 0, false); rdm_sr_check_cqe(&d_cqe, (void *) iov_s_buf, (FI_MSG|FI_RECV), d_iov, MIN((slen - offset) * iov_cnt, (dlen - offset) * iov_cnt), 0, false); s[0] = 1; r[1] = 1; rdm_sr_check_cntrs(s, r, s_e, r_e); dbg_printf("got context events!\n"); cr_assert(rdm_sr_check_iov_data(d_iov, (void *) iov_s_buf, iov_cnt, (slen - offset) * iov_cnt), "Data mismatch"); source_done = dest_done = 0; } }
static int send_recv() { struct fi_cq_entry comp; int ret; ret = fi_recv(ep, rx_buf, rx_size + ft_rx_prefix_size(), mr_desc, 0, &rx_ctx); if (ret) return ret; ft_sync(); fprintf(stdout, "Posting a send...\n"); ret = ft_post_tx(ep, remote_fi_addr, tx_size, NO_CQ_DATA, &tx_ctx); if (ret) return ret; while ((tx_cq_cntr < tx_seq) || (rx_cq_cntr < rx_seq)) { /* Wait for completion events on CQs */ ret = fi_wait(waitset, -1); if (ret < 0) { FT_PRINTERR("fi_wait", ret); return ret; } /* Read the send completion entry */ ret = fi_cq_read(txcq, &comp, 1); if (ret > 0) { tx_cq_cntr++; fprintf(stdout, "Received send completion event!\n"); } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { ret = ft_cq_readerr(txcq); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } /* Read the recv completion entry */ ret = fi_cq_read(rxcq, &comp, 1); if (ret > 0) { rx_cq_cntr++; fprintf(stdout, "Received recv completion event!\n"); } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { ret = ft_cq_readerr(rxcq); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } return 0; }
void do_sendmsgdata(int len) { int ret; ssize_t sz; int source_done = 0, dest_done = 0; struct fi_cq_tagged_entry s_cqe, d_cqe; struct fi_msg msg; struct iovec iov; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; iov.iov_base = source; iov.iov_len = len; msg.msg_iov = &iov; msg.desc = (void **)loc_mr; msg.iov_count = 1; msg.addr = gni_addr[1]; msg.context = target; msg.data = (uint64_t)source; rdm_sr_init_data(source, len, 0xef); rdm_sr_init_data(target, len, 0); sz = fi_sendmsg(ep[0], &msg, FI_REMOTE_CQ_DATA); cr_assert_eq(sz, 0); sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source); cr_assert_eq(sz, 0); /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(msg_cq[0], &s_cqe, 1); if (ret == 1) { source_done = 1; } ret = fi_cq_read(msg_cq[1], &d_cqe, 1); if (ret == 1) { dest_done = 1; } } while (!(source_done && dest_done)); rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0); rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA), target, len, (uint64_t)source); s[0] = 1; r[1] = 1; rdm_sr_check_cntrs(s, r, s_e, r_e); dbg_printf("got context events!\n"); cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch"); }
void api_write_read(int len) { int ret; struct fi_cq_tagged_entry cqe; struct fi_cq_err_entry err_cqe = {0}; rdm_api_init_data(source, len, 0xab); rdm_api_init_data(target, len, 0); fi_write(ep[0], source, len, loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1], target); while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN) pthread_yield(); if (ret == -FI_EAVAIL) { fi_cq_readerr(msg_cq[0], &err_cqe, 0); dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err); } if (write_allowed(FI_RMA, fi[0]->caps, fi[1]->caps)) { cr_assert(ret == 1, "fi_write failed caps:0x%lx ret:%d", fi[0]->caps, ret); } else { cr_assert(err_cqe.err == FI_EOPNOTSUPP, "fi_write should fail caps:0x%lx err:%d", fi[0]->caps, err_cqe.err); } fi_read(ep[0], source, len, loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1], (void *)READ_CTX); while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN) pthread_yield(); if (ret == -FI_EAVAIL) { fi_cq_readerr(msg_cq[0], &err_cqe, 0); dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err); } if (read_allowed(FI_RMA, fi[0]->caps, fi[1]->caps)) { cr_assert(ret == 1, "fi_read failed caps:0x%lx rcaps:0x%lx", fi[0]->caps, fi[1]->caps); } else { cr_assert(err_cqe.err == FI_EOPNOTSUPP, "fi_read should fail caps:0x%lx rcaps:0x%lx", fi[0]->caps, fi[1]->caps); } }
/* * ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len, * void *desc, fi_addr_t dest_addr, void *context); * * ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len, * void *desc, fi_addr_t src_addr, void *context); */ void do_send(int len) { int ret; int source_done = 0, dest_done = 0; int scanceled = 0, dcanceled = 0; struct fi_cq_tagged_entry s_cqe, d_cqe; ssize_t sz; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; rdm_sr_init_data(source, len, 0xab); rdm_sr_init_data(target, len, 0); sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target); cr_assert_eq(sz, 0); sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source); cr_assert_eq(sz, 0); /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(msg_cq[0], &s_cqe, 1); if (ret == 1) { source_done = 1; } if (ret == -FI_EAVAIL) { if (rdm_sr_check_canceled(msg_cq[0])) scanceled = 1; } ret = fi_cq_read(msg_cq[1], &d_cqe, 1); if (ret == 1) { dest_done = 1; } if (ret == -FI_EAVAIL) { if (rdm_sr_check_canceled(msg_cq[1])) dcanceled = 1; } } while (!((source_done || scanceled) && (dest_done || dcanceled))); /* no further checking needed */ if (dgram_should_fail && (scanceled || dcanceled)) return; rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0); rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0); s[0] = 1; r[1] = 1; rdm_sr_check_cntrs(s, r, s_e, r_e); dbg_printf("got context events!\n"); cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch"); }
void do_readmsgdata(int len) { int ret; ssize_t sz; struct fi_cq_tagged_entry cqe, dcqe; struct iovec iov; struct fi_msg_rma msg; struct fi_rma_iov rma_iov; iov.iov_base = source; iov.iov_len = len; rma_iov.addr = (uint64_t)target; rma_iov.len = len; rma_iov.key = mr_key; msg.msg_iov = &iov; msg.desc = (void **)&loc_mr; msg.iov_count = 1; msg.addr = gni_addr[1]; msg.rma_iov = &rma_iov; msg.rma_iov_count = 1; msg.context = target; msg.data = (uint64_t)READ_DATA; init_data(target, len, 0xef); init_data(source, len, 0); sz = fi_readmsg(ep[0], &msg, FI_REMOTE_CQ_DATA); cr_assert_eq(sz, 0); while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_READ, 0); rdm_rma_check_cntrs(0, 1, 0, 0); dbg_printf("got write context event!\n"); cr_assert(check_data(source, target, len), "Data mismatch"); while ((ret = fi_cq_read(recv_cq, &dcqe, 1)) == -FI_EAGAIN) { ret = fi_cq_read(send_cq, &cqe, 1); /* for progress */ pthread_yield(); } cr_assert(ret != FI_SUCCESS, "Missing remote data"); rdm_rma_check_tcqe(&dcqe, NULL, (FI_RMA | FI_REMOTE_READ | FI_REMOTE_CQ_DATA), READ_DATA); }
static int send_recv() { struct fi_cq_entry comp; int ret; if (opts.dst_addr) { /* Client */ fprintf(stdout, "Posting a send...\n"); sprintf(buf, "Hello from Client!"); ret = fi_send(ep, buf, sizeof("Hello from Client!"), fi_mr_desc(mr), remote_fi_addr, &fi_ctx_send); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } /* Read send queue */ do { ret = fi_cq_read(scq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { FT_PRINTERR("fi_cq_read", ret); return ret; } } while (ret == -FI_EAGAIN); fprintf(stdout, "Send completion received\n"); } else { /* Server */ fprintf(stdout, "Posting a recv...\n"); ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, &fi_ctx_recv); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } /* Read recv queue */ fprintf(stdout, "Waiting for client...\n"); do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { FT_PRINTERR("fi_cq_read", ret); return ret; } } while (ret == -FI_EAGAIN); fprintf(stdout, "Received data from client: %s\n", (char *)buf); } return 0; }
static int send_recv() { struct fi_cq_entry comp; int ret; fprintf(stdout, "Posting a send...\n"); ret = ft_post_tx(tx_size); if (ret) return ret; while ((tx_cq_cntr < tx_seq) || (rx_cq_cntr < rx_seq)) { /* Wait for completion events on CQs */ ret = fi_wait(waitset, -1); if (ret < 0) { FT_PRINTERR("fi_wait", ret); return ret; } /* Read the send completion entry */ ret = fi_cq_read(txcq, &comp, 1); if (ret > 0) { tx_cq_cntr++; fprintf(stdout, "Received send completion event!\n"); } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { ret = ft_cq_readerr(txcq); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } /* Read the recv completion entry */ ret = fi_cq_read(rxcq, &comp, 1); if (ret > 0) { rx_cq_cntr++; fprintf(stdout, "Received recv completion event!\n"); } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { ret = ft_cq_readerr(rxcq); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } return 0; }
void do_write_autoreg_uncached(int len) { int ret; ssize_t sz; struct fi_cq_tagged_entry cqe; init_data(uc_source, len, 0xab); init_data(target, len, 0); sz = fi_write(ep[0], uc_source, len, NULL, gni_addr[1], (uint64_t)target, mr_key, target); cr_assert_eq(sz, 0); while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0); rdm_rma_check_cntrs(1, 0, 0, 0); dbg_printf("got write context event!\n"); cr_assert(check_data(uc_source, target, len), "Data mismatch"); }
void do_readv(int len) { int ret; ssize_t sz; struct fi_cq_tagged_entry cqe; struct iovec iov; iov.iov_base = source; iov.iov_len = len; init_data(target, len, 0x25); init_data(source, len, 0); sz = fi_readv(ep[0], &iov, (void **)&loc_mr, 1, gni_addr[1], (uint64_t)target, mr_key, target); cr_assert_eq(sz, 0); while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_READ, 0); rdm_rma_check_cntrs(0, 1, 0, 0); dbg_printf("got write context event!\n"); cr_assert(check_data(source, target, len), "Data mismatch"); }
static int recv_xfer(int size) { struct fi_cq_tagged_entry comp; int ret; do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(rcq, "rcq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } while (ret == -FI_EAGAIN); /* Posting recv for next send. Hence tag_data + 1 */ ret = fi_trecv(ep, buf, buffer_size, fi_mr_desc(mr), remote_fi_addr, tag_data + 1, 0, &fi_ctx_trecv); if (ret) FT_PRINTERR("fi_trecv", ret); return ret; }
void do_tinject(int len) { int ret; ssize_t sz; struct fi_cq_tagged_entry cqe; rdm_tagged_sr_init_data(source, len, 0x23); rdm_tagged_sr_init_data(target, len, 0); sz = fi_tinject(ep[0], source, len, gni_addr[1], len); cr_assert_eq(sz, 0); sz = fi_trecv(ep[1], target, len, rem_mr, gni_addr[0], len, 0, source); cr_assert_eq(sz, 0); while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)source); dbg_printf("got recv context event!\n"); cr_assert(rdm_tagged_sr_check_data(source, target, len), "Data mismatch"); }
void api_do_read_buf(void) { int ret; int len = 8*1024; ssize_t sz; struct fi_cq_tagged_entry cqe; struct fi_cq_err_entry err_cqe; rdm_api_init_data(source, BUF_SZ, 0); rdm_api_init_data(target, BUF_SZ, 0xad); /* cause a chained transaction */ sz = fi_read(ep[0], source+6, len, loc_mr[0], gni_addr[1], (uint64_t)target+6, mr_key[1], (void *)READ_CTX); cr_assert_eq(sz, 0); while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN) pthread_yield(); if (ret == -FI_EAVAIL) { fi_cq_readerr(msg_cq[0], &err_cqe, 0); dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err); } if (read_allowed(FI_RMA, fi[0]->caps, fi[1]->caps)) { cr_assert(ret == 1, "fi_read failed caps:0x%lx rcaps:0x%lx", fi[0]->caps, fi[1]->caps); } else { cr_assert(err_cqe.err == FI_EOPNOTSUPP, "fi_read should fail caps:0x%lx rcaps:0x%lx", fi[0]->caps, fi[1]->caps); } }
/* * fi_cq_err_entry can be cast to any CQ entry format. */ static int ft_fdwait_for_comp(struct fid_cq *cq, uint64_t *cur, uint64_t total, int timeout) { struct fi_cq_err_entry comp; struct fid *fids[1]; int fd, ret; fd = cq == txcq ? tx_fd : rx_fd; fids[0] = &cq->fid; while (total - *cur > 0) { ret = fi_trywait(fabric, fids, 1); if (ret == FI_SUCCESS) { ret = ft_poll_fd(fd, timeout); if (ret) return ret; } ret = fi_cq_read(cq, &comp, 1); if (ret > 0) { (*cur)++; } else if (ret < 0 && ret != -FI_EAGAIN) { return ret; } } return 0; }
/* * fi_cq_err_entry can be cast to any CQ entry format. */ static int ft_spin_for_comp(struct fid_cq *cq, uint64_t *cur, uint64_t total, int timeout) { struct fi_cq_err_entry comp; struct timespec a, b; int ret; if (timeout >= 0) clock_gettime(CLOCK_MONOTONIC, &a); while (total - *cur > 0) { ret = fi_cq_read(cq, &comp, 1); if (ret > 0) { if (timeout >= 0) clock_gettime(CLOCK_MONOTONIC, &a); (*cur)++; } else if (ret < 0 && ret != -FI_EAGAIN) { return ret; } else if (timeout >= 0) { clock_gettime(CLOCK_MONOTONIC, &b); if ((b.tv_sec - a.tv_sec) > timeout) { fprintf(stderr, "%ds timeout expired\n", timeout); return -FI_ENODATA; } } } return 0; }
void rxd_cq_progress(struct util_cq *util_cq) { ssize_t ret = 0; struct rxd_cq *cq; struct fi_cq_msg_entry cq_entry; struct dlist_entry *item, *next; struct rxd_unexp_cq_entry *unexp; cq = container_of(util_cq, struct rxd_cq, util_cq); fastlock_acquire(&cq->lock); do { ret = fi_cq_read(cq->dg_cq, &cq_entry, 1); if (ret == -FI_EAGAIN) break; if (cq_entry.flags & FI_SEND) { rxd_handle_send_comp(&cq_entry); } else if (cq_entry.flags & FI_RECV) { rxd_handle_recv_comp(cq, &cq_entry, 0); } else assert (0); } while (ret > 0); for (item = cq->unexp_list.next; item != &cq->unexp_list;) { unexp = container_of(item, struct rxd_unexp_cq_entry, entry); next = item->next; rxd_handle_recv_comp(cq, &unexp->cq_entry, 1); item = next; } fastlock_release(&cq->lock); }
static int wait_remote_writedata_completion(void) { struct fi_cq_data_entry comp; int ret; do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(rcq, "rcq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } while (ret == -FI_EAGAIN); ret = 0; if (comp.data != cq_data) { fprintf(stderr, "Got unexpected completion data %" PRIu64 "\n", comp.data); } assert(comp.op_context == buf || comp.op_context == NULL); if (comp.op_context == buf) { /* We need to repost the receive */ ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf); if (ret) FT_PRINTERR("fi_recv", ret); } return ret; }
static int send_xfer(int size) { struct fi_cq_entry comp; int ret; while (!credits) { ret = fi_cq_read(scq, &comp, 1); if (ret > 0) { goto post; } else if (ret < 0) { printf("RCQ read %d (%s)\n", ret, fi_strerror(-ret)); return ret; } } credits--; post: ret = dst_addr ? fi_send(ep, buf_ptr, (size_t) size, fi_mr_desc(mr), NULL) : fi_sendto(ep, buf_ptr, (size_t) size, fi_mr_desc(mr), client_addr, NULL); if (ret) printf("fi_send %d (%s)\n", ret, fi_strerror(-ret)); return ret; }
static int wait_recvs() { struct fi_cq_tagged_entry entry; int ret; if (opts.comp_method == FT_COMP_SREAD) { ret = fi_cq_sread(rxcq, &entry, 1, NULL, -1); } else { do { ret = fi_cq_read(rxcq, &entry, 1); } while (ret == -FI_EAGAIN); } if ((ret == 1) && send_data) { if (entry.data != opts.transfer_size) { printf("ERROR incorrect remote CQ data value. Got %lu, expected %d\n", (unsigned long)entry.data, opts.transfer_size); return -FI_EOTHER; } } if (ret < 1) printf("ERROR fi_cq_(s)read returned %d %s\n", ret, fi_strerror(-ret)); return ret; }
static int send_xfer(int size) { struct fi_cq_entry comp; int ret; while (!credits) { ret = fi_cq_read(scq, &comp, 1); if (ret > 0) { goto post; } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(scq, "scq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } credits--; post: ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), remote_fi_addr, &fi_ctx_send); if (ret) FT_PRINTERR("fi_send", ret); return ret; }
void do_read(int len) { int ret; ssize_t sz; struct fi_cq_tagged_entry cqe; #define READ_CTX 0x4e3dda1aULL init_data(source, len, 0); init_data(target, len, 0xad); sz = fi_read(ep[0], source, len, loc_mr, gni_addr[1], (uint64_t)target, mr_key, (void *)READ_CTX); cr_assert_eq(sz, 0); while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); rdm_rma_check_tcqe(&cqe, (void *)READ_CTX, FI_RMA | FI_READ, 0); rdm_rma_check_cntrs(0, 1, 0, 0); dbg_printf("got read context event!\n"); cr_assert(check_data(source, target, len), "Data mismatch"); }
void sep_read(int index, int len) { int ret; ssize_t sz; struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX, (void *) -1, UINT_MAX, UINT_MAX }; uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0}; #define READ_CTX 0x4e3dda1aULL sep_init_data(source, len, 0); sep_init_data(target, len, 0xad); sz = fi_read(tx_ep[0][index], source, len, loc_mr[0], rx_addr[index], (uint64_t)target, mr_key[1], (void *)READ_CTX); cr_assert_eq(sz, 0); while ((ret = fi_cq_read(tx_cq[0][index], &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); sep_check_tcqe(&cqe, (void *)READ_CTX, FI_RMA | FI_READ, 0); r[0] = 1; sep_check_cntrs(w, r, w_e, r_e); cr_assert(sep_check_data(source, target, len), "Data mismatch"); }
static int rxm_msg_cq_read(struct util_cq *util_cq, struct fid_cq *cq, struct fi_cq_tagged_entry *comp) { struct util_cq_err_entry *entry; int ret; ret = fi_cq_read(cq, comp, 1); if (ret == -FI_EAVAIL) { entry = calloc(1, sizeof(*entry)); if (!entry) { FI_WARN(&rxm_prov, FI_LOG_CQ, "Unable to allocate util_cq_err_entry\n"); return -FI_ENOMEM; } OFI_CQ_READERR(&rxm_prov, FI_LOG_CQ, cq, ret, entry->err_entry); if (ret < 0) { free(entry); return ret; } slist_insert_tail(&entry->list_entry, &util_cq->err_list); comp->flags = UTIL_FLAG_ERROR; } return ret; }
static int send_xfer(int size) { struct fi_cq_entry comp; int ret; while (!credits) { ret = fi_cq_read(scq, &comp, 1); if (ret > 0) { goto post; } else if (ret < 0) { if (ret == -FI_EAVAIL) { cq_readerr(scq, "scq"); } else { printf("scq read %d (%s)\n", ret, fi_strerror(-ret)); } return ret; } } credits--; post: ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), NULL); if (ret) printf("fi_send %d (%s)\n", ret, fi_strerror(-ret)); return ret; }
void sep_writev(int index, int len) { int ret; ssize_t sz; struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX, (void *) -1, UINT_MAX, UINT_MAX }; struct iovec iov; uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0}; iov.iov_base = source; iov.iov_len = len; sep_init_data(source, len, 0x25 + index); sep_init_data(target, len, 0); sz = fi_writev(tx_ep[0][index], &iov, (void **)loc_mr, 1, gni_addr[1], (uint64_t)target, mr_key[1], target); cr_assert_eq(sz, 0); while ((ret = fi_cq_read(tx_cq[0][index], &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); sep_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0); w[0] = 1; sep_check_cntrs(w, r, w_e, r_e); cr_assert(sep_check_data(source, target, len), "Data mismatch"); }
int wait_for_recv_completion(int num_completions) { int i, ret; struct fi_cq_data_entry comp; while (num_completions > 0) { ret = fi_cq_read(rxcq, &comp, 1); if (ret == -FI_EAGAIN) continue; if (ret < 0) { FT_PRINTERR("fi_cq_read", ret); return ret; } if (comp.len) num_completions--; if (comp.flags & FI_MULTI_RECV) { i = (comp.op_context == &ctx_multi_recv[0]) ? 0 : 1; ret = fi_recv(ep, rx_buf + (rx_size / 2) * i, rx_size / 2, fi_mr_desc(mr_multi_recv), 0, &ctx_multi_recv[i]); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } } } return 0; }
void sep_atomic_rw(int index) { int ret; ssize_t sz; struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX, (void *) -1, UINT_MAX, UINT_MAX }; uint64_t operand = SOURCE_DATA; uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; /* u64 */ *((uint64_t *)source) = FETCH_SOURCE_DATA; *((uint64_t *)target) = TARGET_DATA; sz = fi_fetch_atomic(tx_ep[0][index], &operand, 1, NULL, source, loc_mr[0], rx_addr[index], (uint64_t)target, mr_key[1], FI_UINT64, FI_SUM, target); cr_assert_eq(sz, 0); while ((ret = fi_cq_read(tx_cq[0][index], &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); sep_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0); r[0] = 1; sep_check_cntrs(w, r, w_e, r_e); ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA); cr_assert(ret, "Data mismatch"); ret = *((uint64_t *)source) == TARGET_DATA; cr_assert(ret, "Fetch data mismatch"); }