int ft_finalize(void) { struct iovec iov; int ret; struct fi_context ctx; void *desc = fi_mr_desc(mr); strcpy(tx_buf + ft_tx_prefix_size(), "fin"); iov.iov_base = tx_buf; iov.iov_len = 4 + ft_tx_prefix_size(); if (hints->caps & FI_TAGGED) { struct fi_msg_tagged tmsg; memset(&tmsg, 0, sizeof tmsg); tmsg.msg_iov = &iov; tmsg.desc = &desc; tmsg.iov_count = 1; tmsg.addr = remote_fi_addr; tmsg.tag = tx_seq; tmsg.ignore = 0; tmsg.context = &ctx; ret = fi_tsendmsg(ep, &tmsg, FI_INJECT | FI_TRANSMIT_COMPLETE); } else { struct fi_msg msg; memset(&msg, 0, sizeof msg); msg.msg_iov = &iov; msg.desc = &desc; msg.iov_count = 1; msg.addr = remote_fi_addr; msg.context = &ctx; ret = fi_sendmsg(ep, &msg, FI_INJECT | FI_TRANSMIT_COMPLETE); } if (ret) { FT_PRINTERR("transmit", ret); return ret; } ret = ft_get_tx_comp(++tx_seq); if (ret) return ret; ret = ft_get_rx_comp(rx_seq); if (ret) return ret; return 0; }
ssize_t ft_post_tx(struct fid_ep *ep, fi_addr_t fi_addr, size_t size, struct fi_context* ctx) { if (hints->caps & FI_TAGGED) { FT_POST(fi_tsend, ft_get_tx_comp, tx_seq, "transmit", ep, tx_buf, size + ft_tx_prefix_size(), fi_mr_desc(mr), fi_addr, tx_seq, ctx); } else { FT_POST(fi_send, ft_get_tx_comp, tx_seq, "transmit", ep, tx_buf, size + ft_tx_prefix_size(), fi_mr_desc(mr), fi_addr, ctx); } return 0; }
ssize_t ft_post_inject(struct fid_ep *ep, size_t size) { if (hints->caps & FI_TAGGED) { FT_POST(fi_tinject, ft_get_tx_comp, tx_seq, "inject", ep, tx_buf, size + ft_tx_prefix_size(), remote_fi_addr, tx_seq); } else { FT_POST(fi_inject, ft_get_tx_comp, tx_seq, "inject", ep, tx_buf, size + ft_tx_prefix_size(), remote_fi_addr); } tx_cq_cntr++; return 0; }
/* Common code will free allocated buffers and MR */ static int alloc_bufs(void) { int ret; tx_size = opts.transfer_size + ft_tx_prefix_size(); rx_size = opts.transfer_size + ft_rx_prefix_size(); buf_size = (tx_size + rx_size) * concurrent_msgs; buf = malloc(buf_size); tx_ctx_arr = calloc(concurrent_msgs, sizeof(*tx_ctx_arr)); rx_ctx_arr = calloc(concurrent_msgs, sizeof(*rx_ctx_arr)); if (!buf || !tx_ctx_arr || !rx_ctx_arr) return -FI_ENOMEM; rx_buf = buf; tx_buf = (char *) buf + rx_size * concurrent_msgs; if (fi->domain_attr->mr_mode & FI_MR_LOCAL) { ret = fi_mr_reg(domain, buf, buf_size, FI_SEND | FI_RECV, 0, FT_MR_KEY, 0, &mr, NULL); if (ret) return ret; mr_desc = fi_mr_desc(mr); } return 0; }
/* * Include FI_MSG_PREFIX space in the allocated buffer, and ensure that the * buffer is large enough for a control message used to exchange addressing * data. */ int ft_alloc_msgs(void) { int ret; long alignment = 1; /* TODO: support multi-recv tests */ if (fi->rx_attr->op_flags == FI_MULTI_RECV) return 0; tx_size = opts.options & FT_OPT_SIZE ? opts.transfer_size : test_size[TEST_CNT - 1].size; if (tx_size > fi->ep_attr->max_msg_size) tx_size = fi->ep_attr->max_msg_size; rx_size = tx_size + ft_rx_prefix_size(); tx_size += ft_tx_prefix_size(); buf_size = MAX(tx_size, FT_MAX_CTRL_MSG) + MAX(rx_size, FT_MAX_CTRL_MSG); if (opts.options & FT_OPT_ALIGN) { alignment = sysconf(_SC_PAGESIZE); if (alignment < 0) return -errno; buf_size += alignment; ret = posix_memalign(&buf, (size_t) alignment, buf_size); if (ret) { FT_PRINTERR("posix_memalign", ret); return ret; } } else { buf = malloc(buf_size); if (!buf) { perror("malloc"); return -FI_ENOMEM; } } memset(buf, 0, buf_size); rx_buf = buf; tx_buf = (char *) buf + MAX(rx_size, FT_MAX_CTRL_MSG); tx_buf = (void *) (((uintptr_t) tx_buf + alignment - 1) & ~(alignment - 1)); remote_cq_data = ft_init_cq_data(fi); if (!ft_skip_mr && ((fi->mode & FI_LOCAL_MR) || (fi->caps & (FI_RMA | FI_ATOMIC)))) { ret = fi_mr_reg(domain, buf, buf_size, ft_caps_to_mr_access(fi->caps), 0, FT_MR_KEY, 0, &mr, NULL); if (ret) { FT_PRINTERR("fi_mr_reg", ret); return ret; } } else { mr = &no_mr; } return 0; }
ssize_t ft_post_tx(size_t size) { ssize_t ret; if (hints->caps & FI_TAGGED) { ret = fi_tsend(ep, tx_buf, size + ft_tx_prefix_size(), fi_mr_desc(mr), remote_fi_addr, tx_seq, &tx_ctx); } else { ret = fi_send(ep, tx_buf, size + ft_tx_prefix_size(), fi_mr_desc(mr), remote_fi_addr, &tx_ctx); } if (ret) { FT_PRINTERR("transmit", ret); return ret; } tx_seq++; return 0; }
ssize_t ft_post_inject(size_t size) { ssize_t ret; if (hints->caps & FI_TAGGED) { ret = fi_tinject(ep, tx_buf, size + ft_tx_prefix_size(), remote_fi_addr, tx_seq); } else { ret = fi_inject(ep, tx_buf, size + ft_tx_prefix_size(), remote_fi_addr); } if (ret) { FT_PRINTERR("transmit", ret); return ret; } tx_seq++; tx_cq_cntr++; return 0; }
int ft_exchange_keys(struct fi_rma_iov *peer_iov) { struct fi_rma_iov *rma_iov; int ret; if (opts.dst_addr) { rma_iov = tx_buf + ft_tx_prefix_size(); rma_iov->addr = fi->domain_attr->mr_mode == FI_MR_SCALABLE ? 0 : (uintptr_t) rx_buf + ft_rx_prefix_size(); rma_iov->key = fi_mr_key(mr); ret = ft_tx(sizeof *rma_iov); if (ret) return ret; ret = ft_get_rx_comp(rx_seq); if (ret) return ret; rma_iov = rx_buf + ft_rx_prefix_size(); *peer_iov = *rma_iov; ret = ft_post_rx(rx_size); } else { ret = ft_get_rx_comp(rx_seq); if (ret) return ret; rma_iov = rx_buf + ft_rx_prefix_size(); *peer_iov = *rma_iov; ret = ft_post_rx(rx_size); if (ret) return ret; rma_iov = tx_buf + ft_tx_prefix_size(); rma_iov->addr = fi->domain_attr->mr_mode == FI_MR_SCALABLE ? 0 : (uintptr_t) rx_buf + ft_rx_prefix_size(); rma_iov->key = fi_mr_key(mr); ret = ft_tx(sizeof *rma_iov); } return ret; }
ssize_t ft_inject(size_t size) { ssize_t ret; if (ft_check_opts(FT_OPT_VERIFY_DATA | FT_OPT_ACTIVE)) ft_fill_buf((char *) tx_buf + ft_tx_prefix_size(), size); ret = ft_post_inject(size); if (ret) return ret; return ret; }
ssize_t ft_tx(struct fid_ep *ep, fi_addr_t fi_addr, size_t size, struct fi_context *ctx) { ssize_t ret; if (ft_check_opts(FT_OPT_VERIFY_DATA | FT_OPT_ACTIVE)) ft_fill_buf((char *) tx_buf + ft_tx_prefix_size(), size); ret = ft_post_tx(ep, fi_addr, size, ctx); if (ret) return ret; ret = ft_get_tx_comp(tx_seq); return ret; }
/* TODO: retry send for unreliable endpoints */ int ft_init_av(void) { size_t addrlen; int ret; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; addrlen = FT_MAX_CTRL_MSG; ret = fi_getname(&ep->fid, (char *) tx_buf + ft_tx_prefix_size(), &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } ret = (int) ft_tx(addrlen); if (ret) return ret; ret = ft_rx(1); } else { ret = (int) ft_rx(FT_MAX_CTRL_MSG); if (ret) return ret; ret = ft_av_insert(av, (char *) rx_buf + ft_rx_prefix_size(), 1, &remote_fi_addr, 0, NULL); if (ret) return ret; ret = (int) ft_tx(1); } return ret; }
static int init_av(void) { int ret; int i; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &addr_array[0], 0, NULL); if (ret) return ret; } for (i = 0; i < ep_cnt; i++) { addrlen = tx_size; ret = fi_getname(&ep_array[i]->fid, tx_buf + ft_tx_prefix_size(), &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } if (opts.dst_addr) { ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx); if (ret) return ret; if (rx_shared_ctx) ret = ft_rx(srx_ctx, rx_size); else ret = ft_rx(ep_array[0], rx_size); if (ret) return ret; /* Skip the first address since we already have it in AV */ if (i) { ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1, &addr_array[i], 0, NULL); if (ret) return ret; } } else { if (rx_shared_ctx) ret = ft_rx(srx_ctx, rx_size); else ret = ft_rx(ep_array[0], rx_size); if (ret) return ret; ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1, &addr_array[i], 0, NULL); if (ret) return ret; ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx); if (ret) return ret; } } /* ACK */ if (opts.dst_addr) { ret = ft_tx(ep_array[0], addr_array[0], 1, &tx_ctx); } else { if (rx_shared_ctx) ret = ft_rx(srx_ctx, rx_size); else ret = ft_rx(ep_array[0], rx_size); } return ret; }
static int run_test_loop(void) { int ret = 0; uint64_t op_data = send_data ? opts.transfer_size : NO_CQ_DATA; uint64_t op_tag = 0x1234; char *op_buf; int i, j; for (i = 0; i < num_iters; i++) { for (j = 0; j < concurrent_msgs; j++) { op_buf = get_tx_buf(j); if (ft_check_opts(FT_OPT_VERIFY_DATA)) ft_fill_buf(op_buf + ft_tx_prefix_size(), opts.transfer_size); ret = ft_post_tx_buf(ep, remote_fi_addr, opts.transfer_size, op_data, &tx_ctx_arr[j], op_buf, mr_desc, op_tag); if (ret) { printf("ERROR send_msg returned %d\n", ret); return ret; } } ret = ft_sync(); if (ret) return ret; for (j = 0; j < concurrent_msgs; j++) { op_buf = get_rx_buf(j); ret = ft_post_rx_buf(ep, opts.transfer_size, &rx_ctx_arr[j], op_buf, mr_desc, op_tag); if (ret) { printf("ERROR recv_msg returned %d\n", ret); return ret; } } for (j = 0; j < concurrent_msgs; j++) { ret = wait_recvs(); if (ret < 1) return ret; } if (ft_check_opts(FT_OPT_VERIFY_DATA)) { for (j = 0; j < concurrent_msgs; j++) { op_buf = get_rx_buf(j); if (ft_check_buf(op_buf + ft_rx_prefix_size(), opts.transfer_size)) return -FI_EOTHER; } } for (j = 0; j < concurrent_msgs; j++) { ret = ft_get_tx_comp(tx_seq); if (ret) return ret; } if (i % 100 == 0) printf("PID %d GOOD iter %d/%ld completed\n", getpid(), i, num_iters); } (void) ft_sync(); printf("PID %d GOOD all done\n", getpid()); return ret; }