int ft_finalize(void)
{
struct iovec iov;
int ret;
struct fi_context ctx;
void *desc = fi_mr_desc(mr);
strcpy(tx_buf + ft_tx_prefix_size(), "fin");
iov.iov_base = tx_buf;
iov.iov_len = 4 + ft_tx_prefix_size();
if (hints->caps & FI_TAGGED) {
struct fi_msg_tagged tmsg;
memset(&tmsg, 0, sizeof tmsg);
tmsg.msg_iov = &iov;
tmsg.desc = &desc;
tmsg.iov_count = 1;
tmsg.addr = remote_fi_addr;
tmsg.tag = tx_seq;
tmsg.ignore = 0;
tmsg.context = &ctx;
ret = fi_tsendmsg(ep, &tmsg, FI_INJECT | FI_TRANSMIT_COMPLETE);
} else {
struct fi_msg msg;
memset(&msg, 0, sizeof msg);
msg.msg_iov = &iov;
msg.desc = &desc;
msg.iov_count = 1;
msg.addr = remote_fi_addr;
msg.context = &ctx;
ret = fi_sendmsg(ep, &msg, FI_INJECT | FI_TRANSMIT_COMPLETE);
}
if (ret) {
FT_PRINTERR("transmit", ret);
return ret;
}
ret = ft_get_tx_comp(++tx_seq);
if (ret)
return ret;
ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;
return 0;
}
ssize_t ft_post_tx(struct fid_ep *ep, fi_addr_t fi_addr, size_t size, struct fi_context* ctx)
{
if (hints->caps & FI_TAGGED) {
FT_POST(fi_tsend, ft_get_tx_comp, tx_seq, "transmit", ep,
tx_buf, size + ft_tx_prefix_size(), fi_mr_desc(mr),
fi_addr, tx_seq, ctx);
} else {
FT_POST(fi_send, ft_get_tx_comp, tx_seq, "transmit", ep,
tx_buf, size + ft_tx_prefix_size(), fi_mr_desc(mr),
fi_addr, ctx);
}
return 0;
}
ssize_t ft_post_inject(struct fid_ep *ep, size_t size)
{
if (hints->caps & FI_TAGGED) {
FT_POST(fi_tinject, ft_get_tx_comp, tx_seq, "inject",
ep, tx_buf, size + ft_tx_prefix_size(),
remote_fi_addr, tx_seq);
} else {
FT_POST(fi_inject, ft_get_tx_comp, tx_seq, "inject",
ep, tx_buf, size + ft_tx_prefix_size(),
remote_fi_addr);
}
tx_cq_cntr++;
return 0;
}
/* Common code will free allocated buffers and MR */
static int alloc_bufs(void)
{
int ret;
tx_size = opts.transfer_size + ft_tx_prefix_size();
rx_size = opts.transfer_size + ft_rx_prefix_size();
buf_size = (tx_size + rx_size) * concurrent_msgs;
buf = malloc(buf_size);
tx_ctx_arr = calloc(concurrent_msgs, sizeof(*tx_ctx_arr));
rx_ctx_arr = calloc(concurrent_msgs, sizeof(*rx_ctx_arr));
if (!buf || !tx_ctx_arr || !rx_ctx_arr)
return -FI_ENOMEM;
rx_buf = buf;
tx_buf = (char *) buf + rx_size * concurrent_msgs;
if (fi->domain_attr->mr_mode & FI_MR_LOCAL) {
ret = fi_mr_reg(domain, buf, buf_size, FI_SEND | FI_RECV,
0, FT_MR_KEY, 0, &mr, NULL);
if (ret)
return ret;
mr_desc = fi_mr_desc(mr);
}
return 0;
}
/*
* Include FI_MSG_PREFIX space in the allocated buffer, and ensure that the
* buffer is large enough for a control message used to exchange addressing
* data.
*/
int ft_alloc_msgs(void)
{
int ret;
long alignment = 1;
/* TODO: support multi-recv tests */
if (fi->rx_attr->op_flags == FI_MULTI_RECV)
return 0;
tx_size = opts.options & FT_OPT_SIZE ?
opts.transfer_size : test_size[TEST_CNT - 1].size;
if (tx_size > fi->ep_attr->max_msg_size)
tx_size = fi->ep_attr->max_msg_size;
rx_size = tx_size + ft_rx_prefix_size();
tx_size += ft_tx_prefix_size();
buf_size = MAX(tx_size, FT_MAX_CTRL_MSG) + MAX(rx_size, FT_MAX_CTRL_MSG);
if (opts.options & FT_OPT_ALIGN) {
alignment = sysconf(_SC_PAGESIZE);
if (alignment < 0)
return -errno;
buf_size += alignment;
ret = posix_memalign(&buf, (size_t) alignment, buf_size);
if (ret) {
FT_PRINTERR("posix_memalign", ret);
return ret;
}
} else {
buf = malloc(buf_size);
if (!buf) {
perror("malloc");
return -FI_ENOMEM;
}
}
memset(buf, 0, buf_size);
rx_buf = buf;
tx_buf = (char *) buf + MAX(rx_size, FT_MAX_CTRL_MSG);
tx_buf = (void *) (((uintptr_t) tx_buf + alignment - 1) &
~(alignment - 1));
remote_cq_data = ft_init_cq_data(fi);
if (!ft_skip_mr && ((fi->mode & FI_LOCAL_MR) ||
(fi->caps & (FI_RMA | FI_ATOMIC)))) {
ret = fi_mr_reg(domain, buf, buf_size, ft_caps_to_mr_access(fi->caps),
0, FT_MR_KEY, 0, &mr, NULL);
if (ret) {
FT_PRINTERR("fi_mr_reg", ret);
return ret;
}
} else {
mr = &no_mr;
}
return 0;
}
ssize_t ft_post_tx(size_t size)
{
ssize_t ret;
if (hints->caps & FI_TAGGED) {
ret = fi_tsend(ep, tx_buf, size + ft_tx_prefix_size(),
fi_mr_desc(mr), remote_fi_addr, tx_seq, &tx_ctx);
} else {
ret = fi_send(ep, tx_buf, size + ft_tx_prefix_size(),
fi_mr_desc(mr), remote_fi_addr, &tx_ctx);
}
if (ret) {
FT_PRINTERR("transmit", ret);
return ret;
}
tx_seq++;
return 0;
}
ssize_t ft_post_inject(size_t size)
{
ssize_t ret;
if (hints->caps & FI_TAGGED) {
ret = fi_tinject(ep, tx_buf, size + ft_tx_prefix_size(),
remote_fi_addr, tx_seq);
} else {
ret = fi_inject(ep, tx_buf, size + ft_tx_prefix_size(),
remote_fi_addr);
}
if (ret) {
FT_PRINTERR("transmit", ret);
return ret;
}
tx_seq++;
tx_cq_cntr++;
return 0;
}
int ft_exchange_keys(struct fi_rma_iov *peer_iov)
{
struct fi_rma_iov *rma_iov;
int ret;
if (opts.dst_addr) {
rma_iov = tx_buf + ft_tx_prefix_size();
rma_iov->addr = fi->domain_attr->mr_mode == FI_MR_SCALABLE ?
0 : (uintptr_t) rx_buf + ft_rx_prefix_size();
rma_iov->key = fi_mr_key(mr);
ret = ft_tx(sizeof *rma_iov);
if (ret)
return ret;
ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;
rma_iov = rx_buf + ft_rx_prefix_size();
*peer_iov = *rma_iov;
ret = ft_post_rx(rx_size);
} else {
ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;
rma_iov = rx_buf + ft_rx_prefix_size();
*peer_iov = *rma_iov;
ret = ft_post_rx(rx_size);
if (ret)
return ret;
rma_iov = tx_buf + ft_tx_prefix_size();
rma_iov->addr = fi->domain_attr->mr_mode == FI_MR_SCALABLE ?
0 : (uintptr_t) rx_buf + ft_rx_prefix_size();
rma_iov->key = fi_mr_key(mr);
ret = ft_tx(sizeof *rma_iov);
}
return ret;
}
ssize_t ft_inject(size_t size)
{
ssize_t ret;
if (ft_check_opts(FT_OPT_VERIFY_DATA | FT_OPT_ACTIVE))
ft_fill_buf((char *) tx_buf + ft_tx_prefix_size(), size);
ret = ft_post_inject(size);
if (ret)
return ret;
return ret;
}
ssize_t ft_tx(struct fid_ep *ep, fi_addr_t fi_addr, size_t size, struct fi_context *ctx)
{
ssize_t ret;
if (ft_check_opts(FT_OPT_VERIFY_DATA | FT_OPT_ACTIVE))
ft_fill_buf((char *) tx_buf + ft_tx_prefix_size(), size);
ret = ft_post_tx(ep, fi_addr, size, ctx);
if (ret)
return ret;
ret = ft_get_tx_comp(tx_seq);
return ret;
}
/* TODO: retry send for unreliable endpoints */
int ft_init_av(void)
{
size_t addrlen;
int ret;
if (opts.dst_addr) {
ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL);
if (ret)
return ret;
addrlen = FT_MAX_CTRL_MSG;
ret = fi_getname(&ep->fid, (char *) tx_buf + ft_tx_prefix_size(),
&addrlen);
if (ret) {
FT_PRINTERR("fi_getname", ret);
return ret;
}
ret = (int) ft_tx(addrlen);
if (ret)
return ret;
ret = ft_rx(1);
} else {
ret = (int) ft_rx(FT_MAX_CTRL_MSG);
if (ret)
return ret;
ret = ft_av_insert(av, (char *) rx_buf + ft_rx_prefix_size(),
1, &remote_fi_addr, 0, NULL);
if (ret)
return ret;
ret = (int) ft_tx(1);
}
return ret;
}
static int init_av(void)
{
int ret;
int i;
if (opts.dst_addr) {
ret = ft_av_insert(av, fi->dest_addr, 1, &addr_array[0], 0, NULL);
if (ret)
return ret;
}
for (i = 0; i < ep_cnt; i++) {
addrlen = tx_size;
ret = fi_getname(&ep_array[i]->fid, tx_buf + ft_tx_prefix_size(),
&addrlen);
if (ret) {
FT_PRINTERR("fi_getname", ret);
return ret;
}
if (opts.dst_addr) {
ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx);
if (ret)
return ret;
if (rx_shared_ctx)
ret = ft_rx(srx_ctx, rx_size);
else
ret = ft_rx(ep_array[0], rx_size);
if (ret)
return ret;
/* Skip the first address since we already have it in AV */
if (i) {
ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1,
&addr_array[i], 0, NULL);
if (ret)
return ret;
}
} else {
if (rx_shared_ctx)
ret = ft_rx(srx_ctx, rx_size);
else
ret = ft_rx(ep_array[0], rx_size);
if (ret)
return ret;
ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1,
&addr_array[i], 0, NULL);
if (ret)
return ret;
ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx);
if (ret)
return ret;
}
}
/* ACK */
if (opts.dst_addr) {
ret = ft_tx(ep_array[0], addr_array[0], 1, &tx_ctx);
} else {
if (rx_shared_ctx)
ret = ft_rx(srx_ctx, rx_size);
else
ret = ft_rx(ep_array[0], rx_size);
}
return ret;
}
static int run_test_loop(void)
{
int ret = 0;
uint64_t op_data = send_data ? opts.transfer_size : NO_CQ_DATA;
uint64_t op_tag = 0x1234;
char *op_buf;
int i, j;
for (i = 0; i < num_iters; i++) {
for (j = 0; j < concurrent_msgs; j++) {
op_buf = get_tx_buf(j);
if (ft_check_opts(FT_OPT_VERIFY_DATA))
ft_fill_buf(op_buf + ft_tx_prefix_size(),
opts.transfer_size);
ret = ft_post_tx_buf(ep, remote_fi_addr,
opts.transfer_size,
op_data, &tx_ctx_arr[j],
op_buf, mr_desc, op_tag);
if (ret) {
printf("ERROR send_msg returned %d\n", ret);
return ret;
}
}
ret = ft_sync();
if (ret)
return ret;
for (j = 0; j < concurrent_msgs; j++) {
op_buf = get_rx_buf(j);
ret = ft_post_rx_buf(ep, opts.transfer_size,
&rx_ctx_arr[j], op_buf,
mr_desc, op_tag);
if (ret) {
printf("ERROR recv_msg returned %d\n", ret);
return ret;
}
}
for (j = 0; j < concurrent_msgs; j++) {
ret = wait_recvs();
if (ret < 1)
return ret;
}
if (ft_check_opts(FT_OPT_VERIFY_DATA)) {
for (j = 0; j < concurrent_msgs; j++) {
op_buf = get_rx_buf(j);
if (ft_check_buf(op_buf + ft_rx_prefix_size(),
opts.transfer_size))
return -FI_EOTHER;
}
}
for (j = 0; j < concurrent_msgs; j++) {
ret = ft_get_tx_comp(tx_seq);
if (ret)
return ret;
}
if (i % 100 == 0)
printf("PID %d GOOD iter %d/%ld completed\n",
getpid(), i, num_iters);
}
(void) ft_sync();
printf("PID %d GOOD all done\n", getpid());
return ret;
}
