ssize_t ft_post_rx(struct fid_ep *ep, size_t size, struct fi_context* ctx) { if (hints->caps & FI_TAGGED) { FT_POST(fi_trecv, ft_get_rx_comp, rx_seq, "receive", ep, rx_buf, MAX(size, FT_MAX_CTRL_MSG) + ft_rx_prefix_size(), fi_mr_desc(mr), 0, rx_seq, 0, ctx); } else { FT_POST(fi_recv, ft_get_rx_comp, rx_seq, "receive", ep, rx_buf, MAX(size, FT_MAX_CTRL_MSG) + ft_rx_prefix_size(), fi_mr_desc(mr), 0, ctx); } return 0; }
/* Common code will free allocated buffers and MR */ static int alloc_bufs(void) { int ret; tx_size = opts.transfer_size + ft_tx_prefix_size(); rx_size = opts.transfer_size + ft_rx_prefix_size(); buf_size = (tx_size + rx_size) * concurrent_msgs; buf = malloc(buf_size); tx_ctx_arr = calloc(concurrent_msgs, sizeof(*tx_ctx_arr)); rx_ctx_arr = calloc(concurrent_msgs, sizeof(*rx_ctx_arr)); if (!buf || !tx_ctx_arr || !rx_ctx_arr) return -FI_ENOMEM; rx_buf = buf; tx_buf = (char *) buf + rx_size * concurrent_msgs; if (fi->domain_attr->mr_mode & FI_MR_LOCAL) { ret = fi_mr_reg(domain, buf, buf_size, FI_SEND | FI_RECV, 0, FT_MR_KEY, 0, &mr, NULL); if (ret) return ret; mr_desc = fi_mr_desc(mr); } return 0; }
/* * Include FI_MSG_PREFIX space in the allocated buffer, and ensure that the * buffer is large enough for a control message used to exchange addressing * data. */ int ft_alloc_msgs(void) { int ret; long alignment = 1; /* TODO: support multi-recv tests */ if (fi->rx_attr->op_flags == FI_MULTI_RECV) return 0; tx_size = opts.options & FT_OPT_SIZE ? opts.transfer_size : test_size[TEST_CNT - 1].size; if (tx_size > fi->ep_attr->max_msg_size) tx_size = fi->ep_attr->max_msg_size; rx_size = tx_size + ft_rx_prefix_size(); tx_size += ft_tx_prefix_size(); buf_size = MAX(tx_size, FT_MAX_CTRL_MSG) + MAX(rx_size, FT_MAX_CTRL_MSG); if (opts.options & FT_OPT_ALIGN) { alignment = sysconf(_SC_PAGESIZE); if (alignment < 0) return -errno; buf_size += alignment; ret = posix_memalign(&buf, (size_t) alignment, buf_size); if (ret) { FT_PRINTERR("posix_memalign", ret); return ret; } } else { buf = malloc(buf_size); if (!buf) { perror("malloc"); return -FI_ENOMEM; } } memset(buf, 0, buf_size); rx_buf = buf; tx_buf = (char *) buf + MAX(rx_size, FT_MAX_CTRL_MSG); tx_buf = (void *) (((uintptr_t) tx_buf + alignment - 1) & ~(alignment - 1)); remote_cq_data = ft_init_cq_data(fi); if (!ft_skip_mr && ((fi->mode & FI_LOCAL_MR) || (fi->caps & (FI_RMA | FI_ATOMIC)))) { ret = fi_mr_reg(domain, buf, buf_size, ft_caps_to_mr_access(fi->caps), 0, FT_MR_KEY, 0, &mr, NULL); if (ret) { FT_PRINTERR("fi_mr_reg", ret); return ret; } } else { mr = &no_mr; } return 0; }
static int send_recv() { struct fi_cq_entry comp; int ret; ret = fi_recv(ep, rx_buf, rx_size + ft_rx_prefix_size(), mr_desc, 0, &rx_ctx); if (ret) return ret; ft_sync(); fprintf(stdout, "Posting a send...\n"); ret = ft_post_tx(ep, remote_fi_addr, tx_size, NO_CQ_DATA, &tx_ctx); if (ret) return ret; while ((tx_cq_cntr < tx_seq) || (rx_cq_cntr < rx_seq)) { /* Wait for completion events on CQs */ ret = fi_wait(waitset, -1); if (ret < 0) { FT_PRINTERR("fi_wait", ret); return ret; } /* Read the send completion entry */ ret = fi_cq_read(txcq, &comp, 1); if (ret > 0) { tx_cq_cntr++; fprintf(stdout, "Received send completion event!\n"); } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { ret = ft_cq_readerr(txcq); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } /* Read the recv completion entry */ ret = fi_cq_read(rxcq, &comp, 1); if (ret > 0) { rx_cq_cntr++; fprintf(stdout, "Received recv completion event!\n"); } else if (ret < 0 && ret != -FI_EAGAIN) { if (ret == -FI_EAVAIL) { ret = ft_cq_readerr(rxcq); } else { FT_PRINTERR("fi_cq_read", ret); } return ret; } } return 0; }
ssize_t ft_post_rx(size_t size) { ssize_t ret; if (hints->caps & FI_TAGGED) { ret = fi_trecv(ep, rx_buf, size + ft_rx_prefix_size(), fi_mr_desc(mr), 0, rx_seq, 0, &rx_ctx); } else { ret = fi_recv(ep, rx_buf, size + ft_rx_prefix_size(), fi_mr_desc(mr), 0, &rx_ctx); } if (ret) { FT_PRINTERR("receive", ret); return ret; } rx_seq++; return 0; }
int ft_exchange_keys(struct fi_rma_iov *peer_iov) { struct fi_rma_iov *rma_iov; int ret; if (opts.dst_addr) { rma_iov = tx_buf + ft_tx_prefix_size(); rma_iov->addr = fi->domain_attr->mr_mode == FI_MR_SCALABLE ? 0 : (uintptr_t) rx_buf + ft_rx_prefix_size(); rma_iov->key = fi_mr_key(mr); ret = ft_tx(sizeof *rma_iov); if (ret) return ret; ret = ft_get_rx_comp(rx_seq); if (ret) return ret; rma_iov = rx_buf + ft_rx_prefix_size(); *peer_iov = *rma_iov; ret = ft_post_rx(rx_size); } else { ret = ft_get_rx_comp(rx_seq); if (ret) return ret; rma_iov = rx_buf + ft_rx_prefix_size(); *peer_iov = *rma_iov; ret = ft_post_rx(rx_size); if (ret) return ret; rma_iov = tx_buf + ft_tx_prefix_size(); rma_iov->addr = fi->domain_attr->mr_mode == FI_MR_SCALABLE ? 0 : (uintptr_t) rx_buf + ft_rx_prefix_size(); rma_iov->key = fi_mr_key(mr); ret = ft_tx(sizeof *rma_iov); } return ret; }
ssize_t ft_rx(size_t size) { ssize_t ret; ret = ft_get_rx_comp(rx_seq); if (ret) return ret; if (ft_check_opts(FT_OPT_VERIFY_DATA | FT_OPT_ACTIVE)) { ret = ft_check_buf((char *) rx_buf + ft_rx_prefix_size(), size); if (ret) return ret; } /* TODO: verify CQ data, if available */ ret = ft_post_rx(rx_size); return ret; }
ssize_t ft_rx(struct fid_ep *ep, size_t size) { ssize_t ret; ret = ft_get_rx_comp(rx_seq); if (ret) return ret; if (ft_check_opts(FT_OPT_VERIFY_DATA | FT_OPT_ACTIVE)) { ret = ft_check_buf((char *) rx_buf + ft_rx_prefix_size(), size); if (ret) return ret; } /* TODO: verify CQ data, if available */ /* Ignore the size arg. Post a buffer large enough to handle all message * sizes. ft_sync() makes use of ft_rx() and gets called in tests just before * message size is updated. The recvs posted are always for the next incoming * message */ ret = ft_post_rx(ep, rx_size, &rx_ctx); return ret; }
/* TODO: retry send for unreliable endpoints */ int ft_init_av(void) { size_t addrlen; int ret; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; addrlen = FT_MAX_CTRL_MSG; ret = fi_getname(&ep->fid, (char *) tx_buf + ft_tx_prefix_size(), &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } ret = (int) ft_tx(addrlen); if (ret) return ret; ret = ft_rx(1); } else { ret = (int) ft_rx(FT_MAX_CTRL_MSG); if (ret) return ret; ret = ft_av_insert(av, (char *) rx_buf + ft_rx_prefix_size(), 1, &remote_fi_addr, 0, NULL); if (ret) return ret; ret = (int) ft_tx(1); } return ret; }
static int run(void) { int i, ret; ret = ft_init_fabric(); if (ret) return ret; /* Post an extra receive to avoid lacking a posted receive in the * finalize. */ ret = fi_recv(ep, rx_buf, rx_size + ft_rx_prefix_size(), fi_mr_desc(mr), 0, &rx_ctx); if (!(opts.options & FT_OPT_SIZE)) { for (i = 0; i < TEST_CNT; i++) { if (!ft_use_size(i, opts.sizes_enabled)) continue; opts.transfer_size = test_size[i].size; if (opts.transfer_size > fi->ep_attr->max_msg_size) continue; init_test(&opts, test_name, sizeof(test_name)); ret = pingpong(); if (ret) return ret; } } else { init_test(&opts, test_name, sizeof(test_name)); ret = pingpong(); if (ret) return ret; } return ft_finalize(); }
static int init_av(void) { int ret; int i; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &addr_array[0], 0, NULL); if (ret) return ret; } for (i = 0; i < ep_cnt; i++) { addrlen = tx_size; ret = fi_getname(&ep_array[i]->fid, tx_buf + ft_tx_prefix_size(), &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } if (opts.dst_addr) { ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx); if (ret) return ret; if (rx_shared_ctx) ret = ft_rx(srx_ctx, rx_size); else ret = ft_rx(ep_array[0], rx_size); if (ret) return ret; /* Skip the first address since we already have it in AV */ if (i) { ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1, &addr_array[i], 0, NULL); if (ret) return ret; } } else { if (rx_shared_ctx) ret = ft_rx(srx_ctx, rx_size); else ret = ft_rx(ep_array[0], rx_size); if (ret) return ret; ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1, &addr_array[i], 0, NULL); if (ret) return ret; ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx); if (ret) return ret; } } /* ACK */ if (opts.dst_addr) { ret = ft_tx(ep_array[0], addr_array[0], 1, &tx_ctx); } else { if (rx_shared_ctx) ret = ft_rx(srx_ctx, rx_size); else ret = ft_rx(ep_array[0], rx_size); } return ret; }
static int run_test_loop(void) { int ret = 0; uint64_t op_data = send_data ? opts.transfer_size : NO_CQ_DATA; uint64_t op_tag = 0x1234; char *op_buf; int i, j; for (i = 0; i < num_iters; i++) { for (j = 0; j < concurrent_msgs; j++) { op_buf = get_tx_buf(j); if (ft_check_opts(FT_OPT_VERIFY_DATA)) ft_fill_buf(op_buf + ft_tx_prefix_size(), opts.transfer_size); ret = ft_post_tx_buf(ep, remote_fi_addr, opts.transfer_size, op_data, &tx_ctx_arr[j], op_buf, mr_desc, op_tag); if (ret) { printf("ERROR send_msg returned %d\n", ret); return ret; } } ret = ft_sync(); if (ret) return ret; for (j = 0; j < concurrent_msgs; j++) { op_buf = get_rx_buf(j); ret = ft_post_rx_buf(ep, opts.transfer_size, &rx_ctx_arr[j], op_buf, mr_desc, op_tag); if (ret) { printf("ERROR recv_msg returned %d\n", ret); return ret; } } for (j = 0; j < concurrent_msgs; j++) { ret = wait_recvs(); if (ret < 1) return ret; } if (ft_check_opts(FT_OPT_VERIFY_DATA)) { for (j = 0; j < concurrent_msgs; j++) { op_buf = get_rx_buf(j); if (ft_check_buf(op_buf + ft_rx_prefix_size(), opts.transfer_size)) return -FI_EOTHER; } } for (j = 0; j < concurrent_msgs; j++) { ret = ft_get_tx_comp(tx_seq); if (ret) return ret; } if (i % 100 == 0) printf("PID %d GOOD iter %d/%ld completed\n", getpid(), i, num_iters); } (void) ft_sync(); printf("PID %d GOOD all done\n", getpid()); return ret; }