/* ssize_t fi_injectdata(struct fid_ep *ep, const void *buf, size_t len, uint64_t data, fi_addr_t dest_addr) */ void do_injectdata(int len) { int ret; ssize_t sz; struct fi_cq_entry cqe; rdm_sr_init_data(source, len, 0xab); rdm_sr_init_data(target, len, 0); sz = fi_injectdata(ep[0], source, len, (uint64_t)source, gni_addr[1]); cr_assert_eq(sz, 0); sz = fi_recv(ep[1], target, len, rem_mr, gni_addr[0], source); cr_assert_eq(sz, 0); /* TODO get REMOTE_CQ_DATA */ while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) { pthread_yield(); } cr_assert_eq(ret, 1); cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)source); dbg_printf("got recv context event!\n"); cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch"); }
static int bind_ep_res(void) { int ret; ret = fi_bind(&ep->fid, &scq->fid, FI_SEND); if (ret) { printf("fi_bind scq %d (%s)\n", ret, fi_strerror(-ret)); return ret; } ret = fi_bind(&ep->fid, &rcq->fid, FI_RECV); if (ret) { printf("fi_bind rcq %d (%s)\n", ret, fi_strerror(-ret)); return ret; } ret = fi_bind(&ep->fid, &av->fid, 0); if (ret) { printf("fi_bind av %d (%s)\n", ret, fi_strerror(-ret)); return ret; } ret = fi_enable(ep); if (ret) { printf("fi_enable %d (%s)\n", ret, fi_strerror(-ret)); return ret; } ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), buf); if (ret) { printf("fi_recv %d (%s)\n", ret, fi_strerror(-ret)); } return ret; }
void cm_basic_send(void) { int ret; int source_done = 0, dest_done = 0; struct fi_cq_tagged_entry cqe; ssize_t sz; uint64_t source = 0xa4321234a4321234, target = 0xb5678901b5678901; sz = fi_send(cli_ep, &source, 8, 0, 0, &target); cr_assert_eq(sz, 0); sz = fi_recv(srv_ep, &target, 8, 0, 0, &source); cr_assert_eq(sz, 0); /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(cli_cq, &cqe, 1); if (ret == 1) { cr_assert_eq(cqe.op_context, &target); source_done = 1; } ret = fi_cq_read(srv_cq, &cqe, 1); if (ret == 1) { cr_assert_eq(cqe.op_context, &source); dest_done = 1; } } while (!source_done || !dest_done); cr_assert_eq(source, target); dbg_printf("Basic send/recv complete! (0x%lx, 0x%lx)\n", source, target); }
int wait_for_recv_completion(int num_completions) { int i, ret; struct fi_cq_data_entry comp; while (num_completions > 0) { ret = fi_cq_read(rxcq, &comp, 1); if (ret == -FI_EAGAIN) continue; if (ret < 0) { FT_PRINTERR("fi_cq_read", ret); return ret; } if (comp.len) num_completions--; if (comp.flags & FI_MULTI_RECV) { i = (comp.op_context == &ctx_multi_recv[0]) ? 0 : 1; ret = fi_recv(ep, rx_buf + (rx_size / 2) * i, rx_size / 2, fi_mr_desc(mr_multi_recv), 0, &ctx_multi_recv[i]); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } } } return 0; }
static int bind_ep_res(void) { int ret; ret = fi_ep_bind(ep, &cmeq->fid, 0); if (ret) { FT_PRINTERR("fi_ep_bind", ret); return ret; } ret = fi_ep_bind(ep, &scq->fid, FI_SEND); if (ret) { FT_PRINTERR("fi_ep_bind", ret); return ret; } ret = fi_ep_bind(ep, &rcq->fid, FI_RECV); if (ret) { FT_PRINTERR("fi_ep_bind", ret); return ret; } ret = fi_enable(ep); if (ret) { FT_PRINTERR("fi_enable", ret); return ret; } /* Post the first recv buffer */ ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf); if (ret) FT_PRINTERR("fi_recv", ret); return ret; }
int rxm_ep_repost_buf(struct rxm_rx_buf *rx_buf) { struct fid_mr *mr; void *desc = NULL; int ret; rx_buf->conn = NULL; rx_buf->recv_fs = NULL; rx_buf->recv_entry = NULL; memset(&rx_buf->unexp_msg, 0, sizeof(rx_buf->unexp_msg)); rx_buf->state = RXM_LMT_NONE; rx_buf->rma_iov = NULL; if (rx_buf->ep->msg_info->mode & FI_LOCAL_MR) { mr = util_buf_get_ctx(rx_buf->ep->rx_pool, rx_buf); desc = fi_mr_desc(mr); } FI_DBG(&rxm_prov, FI_LOG_EP_CTRL, "Re-posting rx buf\n"); ret = fi_recv(rx_buf->ep->srx_ctx, &rx_buf->pkt, RXM_BUF_SIZE, desc, FI_ADDR_UNSPEC, rx_buf); if (ret) FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to repost buf\n"); return ret; }
/* * ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len, * void *desc, fi_addr_t dest_addr, void *context); * * ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len, * void *desc, fi_addr_t src_addr, void *context); */ void api_send_recv(int len) { ssize_t sz; uint64_t caps = fi[0]->caps; rdm_api_init_data(source, len, 0xab); rdm_api_init_data(target, len, 0); sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target); if (MSG_SEND_ALLOWED(caps)) { cr_assert(sz == 0, "fi_send failed caps:0x%lx err:%ld", caps, sz); } else { cr_assert(sz < 0, "fi_send should fail caps:0x%lx err:%ld", caps, sz); } sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source); if (MSG_RECV_ALLOWED(caps)) { cr_assert(sz == 0, "fi_recv failed caps:0x%lx err:%ld", caps, sz); } else { cr_assert(sz < 0, "fi_recv should fail caps:0x%lx err:%ld", caps, sz); } }
void sep_senddata(int index, int len) { ssize_t sz; struct fi_cq_tagged_entry s_cqe = { (void *) -1, UINT_MAX, UINT_MAX, (void *) -1, UINT_MAX, UINT_MAX }; struct fi_cq_tagged_entry d_cqe = { (void *) -1, UINT_MAX, UINT_MAX, (void *) -1, UINT_MAX, UINT_MAX }; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; sep_init_data(source, len, 0xab + index); sep_init_data(target, len, 0); sz = fi_senddata(tx_ep[0][index], source, len, loc_mr[0], (uint64_t)source, rx_addr[index], target); cr_assert_eq(sz, 0); sz = fi_recv(rx_ep[1][index], target, len, rem_mr[0], FI_ADDR_UNSPEC, source); cr_assert_eq(sz, 0); wait_for_cqs(tx_cq[0][index], rx_cq[1][index], &s_cqe, &d_cqe); sep_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0, false); sep_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA), target, len, (uint64_t)source, false); s[0] = 1; r[1] = 1; sep_check_cntrs(s, r, s_e, r_e); cr_assert(sep_check_data(source, target, len), "Data mismatch"); }
static int wait_remote_writedata_completion(void) { struct fi_cq_data_entry comp; int ret; do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(rcq, "rcq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } while (ret == -FI_EAGAIN); ret = 0; if (comp.data != cq_data) { fprintf(stderr, "Got unexpected completion data %" PRIu64 "\n", comp.data); } assert(comp.op_context == buf || comp.op_context == NULL); if (comp.op_context == buf) { /* We need to repost the receive */ ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf); if (ret) FT_PRINTERR("fi_recv", ret); } return ret; }
/* ssize_t fi_sendv(struct fid_ep *ep, const struct iovec *iov, void **desc, size_t count, fi_addr_t dest_addr, void *context); */ void do_sendv(int len) { int i, ret, iov_cnt; int source_done = 0, dest_done = 0; struct fi_cq_tagged_entry s_cqe, d_cqe; ssize_t sz; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; sz = fi_sendv(ep[0], src_iov, NULL, 0, gni_addr[1], iov_dest_buf); cr_assert_eq(sz, -FI_EINVAL); for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) { for (i = 0; i < iov_cnt; i++) { rdm_sr_init_data(src_iov[i].iov_base, len, 0x25); src_iov[i].iov_len = len; } rdm_sr_init_data(iov_dest_buf, len * iov_cnt, 0); /* * TODO: Register src_iov and dest_iov. * Using NULL descriptor for now so that _gnix_send auto registers * the buffers for rndzv messages. */ sz = fi_sendv(ep[0], src_iov, NULL, iov_cnt, gni_addr[1], iov_dest_buf); cr_assert_eq(sz, 0); sz = fi_recv(ep[1], iov_dest_buf, len * iov_cnt, iov_dest_buf_mr[1], gni_addr[0], src_iov); cr_assert_eq(sz, 0); /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(msg_cq[0], &s_cqe, 1); if (ret == 1) { source_done = 1; } ret = fi_cq_read(msg_cq[1], &d_cqe, 1); if (ret == 1) { dest_done = 1; } } while (!(source_done && dest_done)); rdm_sr_check_cqe(&s_cqe, iov_dest_buf, (FI_MSG|FI_SEND), 0, 0, 0, false); rdm_sr_check_cqe(&d_cqe, src_iov, (FI_MSG|FI_RECV), iov_dest_buf, len * iov_cnt, 0, false); s[0] = 1; r[1] = 1; rdm_sr_check_cntrs(s, r, s_e, r_e); dbg_printf("got context events!\n"); cr_assert(rdm_sr_check_iov_data(src_iov, iov_dest_buf, iov_cnt, len * iov_cnt), "Data mismatch"); source_done = dest_done = 0; } }
static int run_test() { int ret, i; /* Post recvs */ for (i = 0; i < ep_cnt; i++) { fprintf(stdout, "Posting recv for ctx: %d\n", i); ret = fi_recv(srx_ctx, rx_buf, rx_size, fi_mr_desc(mr), FI_ADDR_UNSPEC, NULL); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } rx_seq++; } if (opts.dst_addr) { /* Post sends addressed to remote EPs */ for (i = 0; i < ep_cnt; i++) { fprintf(stdout, "Posting send to remote ctx: %d\n", i); ret = fi_send(ep_array[i], tx_buf, tx_size, fi_mr_desc(mr), addr_array[i], NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } ret = ft_get_tx_comp(++tx_seq); if (ret) return ret; } } /* Wait for recv completions */ ret = ft_get_rx_comp(rx_seq); if (ret) return ret; if (!opts.dst_addr) { /* Post sends addressed to remote EPs */ for (i = 0; i < ep_cnt; i++) { fprintf(stdout, "Posting send to remote ctx: %d\n", i); ret = fi_send(ep_array[i], tx_buf, tx_size, fi_mr_desc(mr), addr_array[i], NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } ret = ft_get_tx_comp(++tx_seq); if (ret) return ret; } } return 0; }
static int send_recv() { struct fi_cq_entry comp; int ret; ret = fi_recv(ep, rx_buf, rx_size + ft_rx_prefix_size(), mr_desc, 0, &rx_ctx); if (ret) return ret; ft_sync(); fprintf(stdout, "Posting a send...\n"); ret = ft_post_tx(ep, remote_fi_addr, tx_size, NO_CQ_DATA, &tx_ctx); if (ret) return ret; while ((tx_cq_cntr < tx_seq) || (rx_cq_cntr < rx_seq)) { /* Wait for completion events on CQs */ ret = fi_wait(waitset, -1); if (ret < 0) { FT_PRINTERR("fi_wait", ret); return ret; } /* Read the send completion entry */ ret = fi_cq_read(txcq, &comp, 1); if (ret > 0) { tx_cq_cntr++; fprintf(stdout, "Received send completion event!\n"); } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { ret = ft_cq_readerr(txcq); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } /* Read the recv completion entry */ ret = fi_cq_read(rxcq, &comp, 1); if (ret > 0) { rx_cq_cntr++; fprintf(stdout, "Received recv completion event!\n"); } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { ret = ft_cq_readerr(rxcq); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } return 0; }
static int init_av(void) { size_t addrlen; int ret, i; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; addrlen = FT_MAX_CTRL_MSG; ret = fi_getname(&sep->fid, tx_buf, &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } ret = fi_send(tx_ep[0], tx_buf, addrlen, fi_mr_desc(mr), remote_fi_addr, NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } ret = wait_for_comp(rxcq_array[0]); if (ret) return ret; } else { ret = wait_for_comp(rxcq_array[0]); if (ret) return ret; ret = ft_av_insert(av, rx_buf, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; ret = fi_send(tx_ep[0], tx_buf, 1, fi_mr_desc(mr), remote_fi_addr, NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } } for (i = 0; i < ctx_cnt; i++) remote_rx_addr[i] = fi_rx_addr(remote_fi_addr, i, rx_ctx_bits); ret = fi_recv(rx_ep[0], rx_buf, rx_size, fi_mr_desc(mr), 0, NULL); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } ret = wait_for_comp(txcq_array[0]); return ret; }
void do_sendvrecv_alignment(int slen, int dlen, int offset) { int i, ret, iov_cnt; int source_done = 0, dest_done = 0; struct fi_cq_tagged_entry s_cqe, d_cqe; ssize_t sz; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; uint64_t iov_d_buf = (uint64_t) iov_dest_buf; iov_d_buf += offset; for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) { for (i = 0; i < iov_cnt; i++) { s_iov[i].iov_base = src_iov[i].iov_base; s_iov[i].iov_base = (void *) ((uint64_t)s_iov[i].iov_base + offset); rdm_sr_init_data(s_iov[i].iov_base, slen - offset, 0x25); s_iov[i].iov_len = slen - offset; } rdm_sr_init_data((void *) iov_d_buf, (dlen - offset) * iov_cnt, 0); sz = fi_sendv(ep[0], s_iov, NULL, iov_cnt, gni_addr[1], (void *) iov_d_buf); cr_assert_eq(sz, 0); sz = fi_recv(ep[1], (void *) iov_d_buf, (dlen - offset) * iov_cnt, (void *) iov_dest_buf_mr[1], gni_addr[0], s_iov); cr_assert_eq(sz, 0); /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(msg_cq[0], &s_cqe, 1); if (ret == 1) { source_done = 1; } ret = fi_cq_read(msg_cq[1], &d_cqe, 1); if (ret == 1) { dest_done = 1; } } while (!(source_done && dest_done)); rdm_sr_check_cqe(&s_cqe, (void *) iov_d_buf, (FI_MSG|FI_SEND), 0, 0, 0, false); rdm_sr_check_cqe(&d_cqe, s_iov, (FI_MSG|FI_RECV), (void *) iov_d_buf, (dlen - offset) * iov_cnt, 0, false); s[0] = 1; r[1] = 1; rdm_sr_check_cntrs(s, r, s_e, r_e); dbg_printf("got context events!\n"); cr_assert(rdm_sr_check_iov_data(s_iov, (void *) iov_d_buf, iov_cnt, (dlen - offset) * iov_cnt), "Data mismatch"); source_done = dest_done = 0; } }
static int run_test() { int ret; size_t size = 1000; uint64_t remote_cq_data; struct fi_cq_data_entry comp; /* Set remote_cq_data based on the cq_data_size we got from fi_getinfo */ remote_cq_data = 0x0123456789abcdef & ((0x1ULL << (cq_data_size * 8)) - 1); if (dst_addr) { fprintf(stdout, "Posting send with immediate data: %lx\n", remote_cq_data); ret = fi_senddata(ep, buf, size, fi_mr_desc(mr), remote_cq_data, 0, buf); if (ret) { FI_PRINTERR("fi_send", ret); return ret; } wait_for_completion(scq, 1); fprintf(stdout, "Done\n"); } else { ret = fi_recv(ep, buf, size, fi_mr_desc(mr), 0, buf); if (ret) { FI_PRINTERR("fi_recv", ret); return ret; } fprintf(stdout, "Waiting for immediate data from client\n"); ret = fi_cq_sread(rcq, &comp, 1, NULL, -1); if (ret < 0) { if (ret == -FI_EAVAIL) { cq_readerr(rcq, "rcq"); } else { FI_PRINTERR("fi_cq_read: rcq", ret); } return ret; } /* Verify completion data */ if (comp.flags & FI_REMOTE_CQ_DATA) { if (comp.data == remote_cq_data) fprintf(stdout, "remote_cq_data: success\n"); else fprintf(stdout, "remote_cq_data: failure\n"); fprintf(stdout, "Expected data:0x%lx, Received data:0x%lx\n", remote_cq_data, comp.data); } } return 0; }
static int post_recv(void) { int ret; ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, &fi_ctx_recv); if (ret){ FT_PRINTERR("fi_recv", ret); return ret; } return wait_for_completion(rcq, 1); }
static int post_recv() { int ret; ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, &fi_ctx_recv); if (ret){ fprintf(stderr, "fi_recv %d (%s)\n", ret, fi_strerror(-ret)); return ret; } return wait_for_completion(rcq, 1); }
static int bind_ep_res(void) { int i, ret; ret = fi_scalable_ep_bind(sep, &av->fid, 0); if (ret) { FT_PRINTERR("fi_scalable_ep_bind", ret); return ret; } for (i = 0; i < ctx_cnt; i++) { ret = fi_ep_bind(tx_ep[i], &txcq_array[i]->fid, FI_SEND); if (ret) { FT_PRINTERR("fi_ep_bind", ret); return ret; } ret = fi_enable(tx_ep[i]); if (ret) { FT_PRINTERR("fi_enable", ret); return ret; } } for (i = 0; i < ctx_cnt; i++) { ret = fi_ep_bind(rx_ep[i], &rxcq_array[i]->fid, FI_RECV); if (ret) { FT_PRINTERR("fi_ep_bind", ret); return ret; } ret = fi_enable(rx_ep[i]); if (ret) { FT_PRINTERR("fi_enable", ret); return ret; } ret = fi_recv(rx_ep[i], rx_buf, MAX(rx_size, FT_MAX_CTRL_MSG), mr_desc, 0, NULL); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } } ret = fi_enable(sep); if (ret) { FT_PRINTERR("fi_enable", ret); return ret; } return 0; }
void do_sendmsgdata(int len) { int ret; ssize_t sz; int source_done = 0, dest_done = 0; struct fi_cq_tagged_entry s_cqe, d_cqe; struct fi_msg msg; struct iovec iov; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; iov.iov_base = source; iov.iov_len = len; msg.msg_iov = &iov; msg.desc = (void **)loc_mr; msg.iov_count = 1; msg.addr = gni_addr[1]; msg.context = target; msg.data = (uint64_t)source; rdm_sr_init_data(source, len, 0xef); rdm_sr_init_data(target, len, 0); sz = fi_sendmsg(ep[0], &msg, FI_REMOTE_CQ_DATA); cr_assert_eq(sz, 0); sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source); cr_assert_eq(sz, 0); /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(msg_cq[0], &s_cqe, 1); if (ret == 1) { source_done = 1; } ret = fi_cq_read(msg_cq[1], &d_cqe, 1); if (ret == 1) { dest_done = 1; } } while (!(source_done && dest_done)); rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0); rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA), target, len, (uint64_t)source); s[0] = 1; r[1] = 1; rdm_sr_check_cntrs(s, r, s_e, r_e); dbg_printf("got context events!\n"); cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch"); }
/* * ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len, * void *desc, fi_addr_t dest_addr, void *context); * * ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len, * void *desc, fi_addr_t src_addr, void *context); */ void do_send(int len) { int ret; int source_done = 0, dest_done = 0; int scanceled = 0, dcanceled = 0; struct fi_cq_tagged_entry s_cqe, d_cqe; ssize_t sz; uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0}; uint64_t r_e[NUMEPS] = {0}; rdm_sr_init_data(source, len, 0xab); rdm_sr_init_data(target, len, 0); sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target); cr_assert_eq(sz, 0); sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source); cr_assert_eq(sz, 0); /* need to progress both CQs simultaneously for rendezvous */ do { ret = fi_cq_read(msg_cq[0], &s_cqe, 1); if (ret == 1) { source_done = 1; } if (ret == -FI_EAVAIL) { if (rdm_sr_check_canceled(msg_cq[0])) scanceled = 1; } ret = fi_cq_read(msg_cq[1], &d_cqe, 1); if (ret == 1) { dest_done = 1; } if (ret == -FI_EAVAIL) { if (rdm_sr_check_canceled(msg_cq[1])) dcanceled = 1; } } while (!((source_done || scanceled) && (dest_done || dcanceled))); /* no further checking needed */ if (dgram_should_fail && (scanceled || dcanceled)) return; rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0); rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0); s[0] = 1; r[1] = 1; rdm_sr_check_cntrs(s, r, s_e, r_e); dbg_printf("got context events!\n"); cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch"); }
static int run_test() { int ret, i; /* Post recvs */ for (i = 0; i < ep_cnt; i++) { fprintf(stdout, "Posting recv for ctx: %d\n", i); ret = fi_recv(srx_ctx, buf, buffer_size, fi_mr_desc(mr), FI_ADDR_UNSPEC, NULL); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } } if (opts.dst_addr) { /* Post sends addressed to remote EPs */ for (i = 0; i < ep_cnt; i++) { fprintf(stdout, "Posting send to remote ctx: %d\n", i); ret = fi_send(ep[i], buf, transfer_size, fi_mr_desc(mr), remote_fi_addr[i], NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } wait_for_completion(scq, 1); } } /* Wait for recv completions */ for (i = 0; i < ep_cnt; i++) { wait_for_completion(rcq, 1); } if (!opts.dst_addr) { /* Post sends addressed to remote EPs */ for (i = 0; i < ep_cnt; i++) { fprintf(stdout, "Posting send to remote ctx: %d\n", i); ret = fi_send(ep[i], buf, transfer_size, fi_mr_desc(mr), remote_fi_addr[i], NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } wait_for_completion(scq, 1); } } return 0; }
void api_cntr_send_recv(int len) { ssize_t sz; api_cntr_init_data(source, len, 0xab); api_cntr_init_data(target, len, 0); sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target); api_cntr_send_allowed(sz, cntr_bind_flags, "fi_send"); sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source); api_cntr_recv_allowed(sz, cntr_bind_flags, "fi_recv"); }
static int recv_msg(void) { int ret; ret = fi_recv(srx_ctx, buf, rx_size, fi_mr_desc(mr), 0, &rx_ctx); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } ret = ft_get_rx_comp(++rx_seq); return ret; }
static int send_recv() { struct fi_cq_entry comp; int ret; if (opts.dst_addr) { /* Client */ fprintf(stdout, "Posting a send...\n"); sprintf(buf, "Hello from Client!"); ret = fi_send(ep, buf, sizeof("Hello from Client!"), fi_mr_desc(mr), remote_fi_addr, &fi_ctx_send); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } /* Read send queue */ do { ret = fi_cq_read(scq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { FT_PRINTERR("fi_cq_read", ret); return ret; } } while (ret == -FI_EAGAIN); fprintf(stdout, "Send completion received\n"); } else { /* Server */ fprintf(stdout, "Posting a recv...\n"); ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, &fi_ctx_recv); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } /* Read recv queue */ fprintf(stdout, "Waiting for client...\n"); do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { FT_PRINTERR("fi_cq_read", ret); return ret; } } while (ret == -FI_EAGAIN); fprintf(stdout, "Received data from client: %s\n", (char *)buf); } return 0; }
static int server_connect(void) { int ret; struct fi_cq_entry comp; ret = common_setup(); if (ret != 0) goto err; do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { FT_PRINTERR("fi_cq_read", ret); return ret; } } while (ret == -FI_EAGAIN); ret = fi_av_insert(av, buf_ptr, 1, &remote_fi_addr, 0, NULL); if (ret != 1) { if (ret == 0) { fprintf(stderr, "Unable to resolve remote address 0x%x 0x%x\n", ((uint32_t *)buf)[0], ((uint32_t *)buf)[1]); ret = -FI_EINVAL; } else { FT_PRINTERR("fi_av_insert", ret); } goto err; } ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf); if (ret != 0) { FT_PRINTERR("fi_recv", ret); goto err; } ret = send_xfer(4); if (ret != 0) goto err; return 0; err: free_ep_res(); if (dom) fi_close(&dom->fid); if (fab) fi_close(&fab->fid); return ret; }
static int recv_msg(void) { int ret; ret = fi_recv(ep, buf, rx_size, fi_mr_desc(mr), 0, &fi_ctx_recv); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } ret = ft_wait_for_comp(rxcq, 1); return ret; }
static int post_recvs(struct cma_node *node) { int i, ret = 0; for (i = 0; i < hints->tx_attr->size && !ret; i++ ) { ret = fi_recv(node->ep, node->mem, hints->ep_attr->max_msg_size, node->mrdesc, 0, node); if (ret) { FT_PRINTERR("fi_recv", ret); break; } } return ret; }
static int receive_loop(size_t size) { int ret; int q_opts = 0; struct fi_context recv_ctx[max_opts]; while (q_opts < max_opts) { do { ft_tag = q_opts + 1; if (tagged) ret = fi_trecv(ep, rx_buf, size, NULL, remote_fi_addr, ft_tag, 0x0, (void *) &recv_ctx[q_opts]); else ret = fi_recv(ep, rx_buf, size, NULL, remote_fi_addr, (void *) &recv_ctx[q_opts]); if (ret == FI_SUCCESS) { rx_seq++; q_opts++; } } while (!ret && (q_opts != max_opts)); if (ret < 0) { if (ret == -FI_EAGAIN) { if (delay > 0) sleep(delay); ret = ft_get_rx_comp(rx_seq); if (ret) return ret; } else { FT_PRINTERR("Recv OP", ret); return ret; } } } if (delay > 0) sleep(delay); ret = ft_get_rx_comp(rx_seq); if (ret) return ret; if (opts.verbose) printf("Success: Completed %d queued ops\n", q_opts); return 0; }
static int recv_msg(void) { int ret; /* Messages sent to scalable EP fi_addr are received in context 0 */ ret = fi_recv(rx_ep[0], buf, buffer_size, fi_mr_desc(mr), 0, &fi_ctx_recv); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } ret = wait_for_completion(rcq[0], 1); return ret; }
ssize_t rxm_recv(struct fid_ep *ep_fid, void *buf, size_t len, void *desc, fi_addr_t src_addr, void *context) { struct rxm_ep *rxm_ep; /* TODO Handle recv from particular src */ if (src_addr) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "Unable to post recv for a particular source\n"); return -FI_EINVAL; } rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return fi_recv(rxm_ep->srx_ctx, buf, len, desc, 0, context); }