static int send_xfer(int size) { struct fi_cq_entry comp; int ret; while (!credits) { ret = fi_cq_read(scq, &comp, 1); if (ret > 0) { goto post; } else if (ret < 0) { if (ret == -FI_EAVAIL) { cq_readerr(scq, "scq"); } else { printf("scq read %d (%s)\n", ret, fi_strerror(-ret)); } return ret; } } credits--; post: ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), NULL); if (ret) printf("fi_send %d (%s)\n", ret, fi_strerror(-ret)); return ret; }
static int send_xfer(int size) { struct fi_cq_entry comp; int ret; while (!credits) { ret = fi_cq_read(scq, &comp, 1); if (ret > 0) { goto post; } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(scq, "scq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } credits--; post: ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), remote_fi_addr, &fi_ctx_send); if (ret) FT_PRINTERR("fi_send", ret); return ret; }
static int wait_remote_writedata_completion(void) { struct fi_cq_data_entry comp; int ret; do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(rcq, "rcq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } while (ret == -FI_EAGAIN); ret = 0; if (comp.data != cq_data) { fprintf(stderr, "Got unexpected completion data %" PRIu64 "\n", comp.data); } assert(comp.op_context == buf || comp.op_context == NULL); if (comp.op_context == buf) { /* We need to repost the receive */ ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf); if (ret) FT_PRINTERR("fi_recv", ret); } return ret; }
static int recv_xfer(int size) { struct fi_cq_tagged_entry comp; int ret; do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(rcq, "rcq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } while (ret == -FI_EAGAIN); /* Posting recv for next send. Hence tag_data + 1 */ ret = fi_trecv(ep, buf, buffer_size, fi_mr_desc(mr), remote_fi_addr, tag_data + 1, 0, &fi_ctx_trecv); if (ret) FT_PRINTERR("fi_trecv", ret); return ret; }
static int run_test() { int ret; size_t size = 1000; uint64_t remote_cq_data; struct fi_cq_data_entry comp; if (fi->domain_attr->cq_data_size >= sizeof(uint64_t)) { remote_cq_data = 0x0123456789abcdefULL; } else { remote_cq_data = 0x0123456789abcdef & ((0x1ULL << (fi->domain_attr->cq_data_size * 8)) - 1); } if (opts.dst_addr) { fprintf(stdout, "Posting send with immediate data: 0x%" PRIx64 "\n", remote_cq_data); ret = fi_senddata(ep, buf, size, fi_mr_desc(mr), remote_cq_data, 0, buf); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } ft_wait_for_comp(txcq, 1); fprintf(stdout, "Done\n"); } else { fprintf(stdout, "Waiting for immediate data from client\n"); ret = fi_cq_sread(rxcq, &comp, 1, NULL, -1); if (ret < 0) { if (ret == -FI_EAVAIL) { cq_readerr(rxcq, "rxcq"); } else { FT_PRINTERR("fi_cq_sread", ret); } return ret; } /* Verify completion data */ if (comp.flags & FI_REMOTE_CQ_DATA) { if (comp.data == remote_cq_data) fprintf(stdout, "remote_cq_data: success\n"); else fprintf(stdout, "remote_cq_data: failure\n"); fprintf(stdout, "Expected data:0x%" PRIx64 ", Received data:0x%" PRIx64 "\n", remote_cq_data, comp.data); } } return 0; }
static int run_test() { int ret; size_t size = 1000; uint64_t remote_cq_data; struct fi_cq_data_entry comp; /* Set remote_cq_data based on the cq_data_size we got from fi_getinfo */ remote_cq_data = 0x0123456789abcdef & ((0x1ULL << (cq_data_size * 8)) - 1); if (dst_addr) { fprintf(stdout, "Posting send with immediate data: %lx\n", remote_cq_data); ret = fi_senddata(ep, buf, size, fi_mr_desc(mr), remote_cq_data, 0, buf); if (ret) { FI_PRINTERR("fi_send", ret); return ret; } wait_for_completion(scq, 1); fprintf(stdout, "Done\n"); } else { ret = fi_recv(ep, buf, size, fi_mr_desc(mr), 0, buf); if (ret) { FI_PRINTERR("fi_recv", ret); return ret; } fprintf(stdout, "Waiting for immediate data from client\n"); ret = fi_cq_sread(rcq, &comp, 1, NULL, -1); if (ret < 0) { if (ret == -FI_EAVAIL) { cq_readerr(rcq, "rcq"); } else { FI_PRINTERR("fi_cq_read: rcq", ret); } return ret; } /* Verify completion data */ if (comp.flags & FI_REMOTE_CQ_DATA) { if (comp.data == remote_cq_data) fprintf(stdout, "remote_cq_data: success\n"); else fprintf(stdout, "remote_cq_data: failure\n"); fprintf(stdout, "Expected data:0x%lx, Received data:0x%lx\n", remote_cq_data, comp.data); } } return 0; }
/* * fi_cq_err_entry can be cast to any CQ entry format. */ static int wait_for_comp(struct fid_cq *cq, int num_completions) { struct fi_cq_err_entry comp; int ret; while (num_completions > 0) { ret = fi_cq_read(cq, &comp, 1); if (ret > 0) { num_completions--; } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(cq, "cq"); } else { ct_print_fi_error("fi_cq_read", ret); } return ret; } } return 0; }
int wait_for_completion(struct fid_cq *cq, int num_completions) { int ret; struct fi_cq_entry comp; while (num_completions > 0) { ret = fi_cq_read(cq, &comp, 1); if (ret > 0) { num_completions--; } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(cq, "cq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } return 0; }
static int recv_xfer(int size) { struct fi_cq_entry comp; int ret; do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0) { if (ret == -FI_EAVAIL) { cq_readerr(rcq, "rcq"); } else { printf("rcq read %d (%s)\n", ret, fi_strerror(-ret)); } return ret; } } while (!ret); ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), buf); if (ret) printf("fi_recv %d (%s)\n", ret, fi_strerror(-ret)); return ret; }
static int recv_xfer(int size) { struct fi_cq_data_entry comp; int ret; do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(rcq, "rcq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } while (ret == -FI_EAGAIN); ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf); if (ret) FT_PRINTERR("fi_recv", ret); return ret; }
static int recv_xfer(int size) { struct fi_cq_entry comp; int ret; do { ret = fi_cq_read(rxcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { cq_readerr(rxcq, "rxcq"); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } while (ret == -FI_EAGAIN); ret = fi_recv(ep, rx_buf, rx_size, fi_mr_desc(mr), remote_fi_addr, &fi_ctx_recv); if (ret) FT_PRINTERR("fi_recv", ret); return ret; }
static int send_recv() { void *context[MAX_POLL_CNT]; struct fi_cq_entry comp; int ret, send_pending = 0, recv_pending = 0; int ret_count = 0; int i; fprintf(stdout, "Posting a recv...\n"); ret = fi_recv(ep, buf, rx_size, fi_mr_desc(mr), remote_fi_addr, &fi_ctx_recv); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } recv_pending++; fprintf(stdout, "Posting a send...\n"); ret = fi_send(ep, buf, tx_size, fi_mr_desc(mr), remote_fi_addr, &fi_ctx_send); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } send_pending++; while (send_pending || recv_pending) { struct fid_cq *cq; /* Poll send and recv CQs */ do { ret_count = fi_poll(pollset, context, MAX_POLL_CNT); if (ret_count < 0) { FT_PRINTERR("fi_poll", ret_count); return ret_count; } } while (!ret_count); fprintf(stdout, "Retreived %d event(s)\n", ret_count); for (i = 0; i < ret_count; i++) { if (context[i] == &txcq) { printf("Send completion received\n"); cq = txcq; send_pending--; } else if (context[i] == &rxcq) { printf("Recv completion received\n"); cq = rxcq; recv_pending--; } else { printf("Unknown completion received\n"); return -1; } /* Read the completion entry */ ret = fi_cq_sread(cq, &comp, 1, NULL, -1); if (ret < 0) { if (ret == -FI_EAVAIL) { cq_readerr(cq, "cq"); } else { FT_PRINTERR("fi_cq_sread", ret); } return ret; } } } return 0; }