ssize_t ft_post_rx(struct fid_ep *ep, size_t size, struct fi_context* ctx)
{
if (hints->caps & FI_TAGGED) {
FT_POST(fi_trecv, ft_get_rx_comp, rx_seq, "receive", ep, rx_buf,
MAX(size, FT_MAX_CTRL_MSG) + ft_rx_prefix_size(),
fi_mr_desc(mr), 0, rx_seq, 0, ctx);
} else {
FT_POST(fi_recv, ft_get_rx_comp, rx_seq, "receive", ep, rx_buf,
MAX(size, FT_MAX_CTRL_MSG) + ft_rx_prefix_size(),
fi_mr_desc(mr), 0, ctx);
}
return 0;
}
/* Common code will free allocated buffers and MR */
static int alloc_bufs(void)
{
int ret;
tx_size = opts.transfer_size + ft_tx_prefix_size();
rx_size = opts.transfer_size + ft_rx_prefix_size();
buf_size = (tx_size + rx_size) * concurrent_msgs;
buf = malloc(buf_size);
tx_ctx_arr = calloc(concurrent_msgs, sizeof(*tx_ctx_arr));
rx_ctx_arr = calloc(concurrent_msgs, sizeof(*rx_ctx_arr));
if (!buf || !tx_ctx_arr || !rx_ctx_arr)
return -FI_ENOMEM;
rx_buf = buf;
tx_buf = (char *) buf + rx_size * concurrent_msgs;
if (fi->domain_attr->mr_mode & FI_MR_LOCAL) {
ret = fi_mr_reg(domain, buf, buf_size, FI_SEND | FI_RECV,
0, FT_MR_KEY, 0, &mr, NULL);
if (ret)
return ret;
mr_desc = fi_mr_desc(mr);
}
return 0;
}
/*
* Include FI_MSG_PREFIX space in the allocated buffer, and ensure that the
* buffer is large enough for a control message used to exchange addressing
* data.
*/
int ft_alloc_msgs(void)
{
int ret;
long alignment = 1;
/* TODO: support multi-recv tests */
if (fi->rx_attr->op_flags == FI_MULTI_RECV)
return 0;
tx_size = opts.options & FT_OPT_SIZE ?
opts.transfer_size : test_size[TEST_CNT - 1].size;
if (tx_size > fi->ep_attr->max_msg_size)
tx_size = fi->ep_attr->max_msg_size;
rx_size = tx_size + ft_rx_prefix_size();
tx_size += ft_tx_prefix_size();
buf_size = MAX(tx_size, FT_MAX_CTRL_MSG) + MAX(rx_size, FT_MAX_CTRL_MSG);
if (opts.options & FT_OPT_ALIGN) {
alignment = sysconf(_SC_PAGESIZE);
if (alignment < 0)
return -errno;
buf_size += alignment;
ret = posix_memalign(&buf, (size_t) alignment, buf_size);
if (ret) {
FT_PRINTERR("posix_memalign", ret);
return ret;
}
} else {
buf = malloc(buf_size);
if (!buf) {
perror("malloc");
return -FI_ENOMEM;
}
}
memset(buf, 0, buf_size);
rx_buf = buf;
tx_buf = (char *) buf + MAX(rx_size, FT_MAX_CTRL_MSG);
tx_buf = (void *) (((uintptr_t) tx_buf + alignment - 1) &
~(alignment - 1));
remote_cq_data = ft_init_cq_data(fi);
if (!ft_skip_mr && ((fi->mode & FI_LOCAL_MR) ||
(fi->caps & (FI_RMA | FI_ATOMIC)))) {
ret = fi_mr_reg(domain, buf, buf_size, ft_caps_to_mr_access(fi->caps),
0, FT_MR_KEY, 0, &mr, NULL);
if (ret) {
FT_PRINTERR("fi_mr_reg", ret);
return ret;
}
} else {
mr = &no_mr;
}
return 0;
}
static int send_recv()
{
struct fi_cq_entry comp;
int ret;
ret = fi_recv(ep, rx_buf, rx_size + ft_rx_prefix_size(),
mr_desc, 0, &rx_ctx);
if (ret)
return ret;
ft_sync();
fprintf(stdout, "Posting a send...\n");
ret = ft_post_tx(ep, remote_fi_addr, tx_size, NO_CQ_DATA, &tx_ctx);
if (ret)
return ret;
while ((tx_cq_cntr < tx_seq) || (rx_cq_cntr < rx_seq)) {
/* Wait for completion events on CQs */
ret = fi_wait(waitset, -1);
if (ret < 0) {
FT_PRINTERR("fi_wait", ret);
return ret;
}
/* Read the send completion entry */
ret = fi_cq_read(txcq, &comp, 1);
if (ret > 0) {
tx_cq_cntr++;
fprintf(stdout, "Received send completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(txcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
/* Read the recv completion entry */
ret = fi_cq_read(rxcq, &comp, 1);
if (ret > 0) {
rx_cq_cntr++;
fprintf(stdout, "Received recv completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(rxcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
}
return 0;
}
ssize_t ft_post_rx(size_t size)
{
ssize_t ret;
if (hints->caps & FI_TAGGED) {
ret = fi_trecv(ep, rx_buf, size + ft_rx_prefix_size(), fi_mr_desc(mr),
0, rx_seq, 0, &rx_ctx);
} else {
ret = fi_recv(ep, rx_buf, size + ft_rx_prefix_size(), fi_mr_desc(mr),
0, &rx_ctx);
}
if (ret) {
FT_PRINTERR("receive", ret);
return ret;
}
rx_seq++;
return 0;
}
int ft_exchange_keys(struct fi_rma_iov *peer_iov)
{
struct fi_rma_iov *rma_iov;
int ret;
if (opts.dst_addr) {
rma_iov = tx_buf + ft_tx_prefix_size();
rma_iov->addr = fi->domain_attr->mr_mode == FI_MR_SCALABLE ?
0 : (uintptr_t) rx_buf + ft_rx_prefix_size();
rma_iov->key = fi_mr_key(mr);
ret = ft_tx(sizeof *rma_iov);
if (ret)
return ret;
ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;
rma_iov = rx_buf + ft_rx_prefix_size();
*peer_iov = *rma_iov;
ret = ft_post_rx(rx_size);
} else {
ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;
rma_iov = rx_buf + ft_rx_prefix_size();
*peer_iov = *rma_iov;
ret = ft_post_rx(rx_size);
if (ret)
return ret;
rma_iov = tx_buf + ft_tx_prefix_size();
rma_iov->addr = fi->domain_attr->mr_mode == FI_MR_SCALABLE ?
0 : (uintptr_t) rx_buf + ft_rx_prefix_size();
rma_iov->key = fi_mr_key(mr);
ret = ft_tx(sizeof *rma_iov);
}
return ret;
}
ssize_t ft_rx(size_t size)
{
ssize_t ret;
ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;
if (ft_check_opts(FT_OPT_VERIFY_DATA | FT_OPT_ACTIVE)) {
ret = ft_check_buf((char *) rx_buf + ft_rx_prefix_size(), size);
if (ret)
return ret;
}
/* TODO: verify CQ data, if available */
ret = ft_post_rx(rx_size);
return ret;
}
ssize_t ft_rx(struct fid_ep *ep, size_t size)
{
ssize_t ret;
ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;
if (ft_check_opts(FT_OPT_VERIFY_DATA | FT_OPT_ACTIVE)) {
ret = ft_check_buf((char *) rx_buf + ft_rx_prefix_size(), size);
if (ret)
return ret;
}
/* TODO: verify CQ data, if available */
/* Ignore the size arg. Post a buffer large enough to handle all message
* sizes. ft_sync() makes use of ft_rx() and gets called in tests just before
* message size is updated. The recvs posted are always for the next incoming
* message */
ret = ft_post_rx(ep, rx_size, &rx_ctx);
return ret;
}
/* TODO: retry send for unreliable endpoints */
int ft_init_av(void)
{
size_t addrlen;
int ret;
if (opts.dst_addr) {
ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL);
if (ret)
return ret;
addrlen = FT_MAX_CTRL_MSG;
ret = fi_getname(&ep->fid, (char *) tx_buf + ft_tx_prefix_size(),
&addrlen);
if (ret) {
FT_PRINTERR("fi_getname", ret);
return ret;
}
ret = (int) ft_tx(addrlen);
if (ret)
return ret;
ret = ft_rx(1);
} else {
ret = (int) ft_rx(FT_MAX_CTRL_MSG);
if (ret)
return ret;
ret = ft_av_insert(av, (char *) rx_buf + ft_rx_prefix_size(),
1, &remote_fi_addr, 0, NULL);
if (ret)
return ret;
ret = (int) ft_tx(1);
}
return ret;
}
static int run(void)
{
int i, ret;
ret = ft_init_fabric();
if (ret)
return ret;
/* Post an extra receive to avoid lacking a posted receive in the
* finalize.
*/
ret = fi_recv(ep, rx_buf, rx_size + ft_rx_prefix_size(), fi_mr_desc(mr),
0, &rx_ctx);
if (!(opts.options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (!ft_use_size(i, opts.sizes_enabled))
continue;
opts.transfer_size = test_size[i].size;
if (opts.transfer_size > fi->ep_attr->max_msg_size)
continue;
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
return ret;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
return ret;
}
return ft_finalize();
}
static int init_av(void)
{
int ret;
int i;
if (opts.dst_addr) {
ret = ft_av_insert(av, fi->dest_addr, 1, &addr_array[0], 0, NULL);
if (ret)
return ret;
}
for (i = 0; i < ep_cnt; i++) {
addrlen = tx_size;
ret = fi_getname(&ep_array[i]->fid, tx_buf + ft_tx_prefix_size(),
&addrlen);
if (ret) {
FT_PRINTERR("fi_getname", ret);
return ret;
}
if (opts.dst_addr) {
ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx);
if (ret)
return ret;
if (rx_shared_ctx)
ret = ft_rx(srx_ctx, rx_size);
else
ret = ft_rx(ep_array[0], rx_size);
if (ret)
return ret;
/* Skip the first address since we already have it in AV */
if (i) {
ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1,
&addr_array[i], 0, NULL);
if (ret)
return ret;
}
} else {
if (rx_shared_ctx)
ret = ft_rx(srx_ctx, rx_size);
else
ret = ft_rx(ep_array[0], rx_size);
if (ret)
return ret;
ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1,
&addr_array[i], 0, NULL);
if (ret)
return ret;
ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx);
if (ret)
return ret;
}
}
/* ACK */
if (opts.dst_addr) {
ret = ft_tx(ep_array[0], addr_array[0], 1, &tx_ctx);
} else {
if (rx_shared_ctx)
ret = ft_rx(srx_ctx, rx_size);
else
ret = ft_rx(ep_array[0], rx_size);
}
return ret;
}
static int run_test_loop(void)
{
int ret = 0;
uint64_t op_data = send_data ? opts.transfer_size : NO_CQ_DATA;
uint64_t op_tag = 0x1234;
char *op_buf;
int i, j;
for (i = 0; i < num_iters; i++) {
for (j = 0; j < concurrent_msgs; j++) {
op_buf = get_tx_buf(j);
if (ft_check_opts(FT_OPT_VERIFY_DATA))
ft_fill_buf(op_buf + ft_tx_prefix_size(),
opts.transfer_size);
ret = ft_post_tx_buf(ep, remote_fi_addr,
opts.transfer_size,
op_data, &tx_ctx_arr[j],
op_buf, mr_desc, op_tag);
if (ret) {
printf("ERROR send_msg returned %d\n", ret);
return ret;
}
}
ret = ft_sync();
if (ret)
return ret;
for (j = 0; j < concurrent_msgs; j++) {
op_buf = get_rx_buf(j);
ret = ft_post_rx_buf(ep, opts.transfer_size,
&rx_ctx_arr[j], op_buf,
mr_desc, op_tag);
if (ret) {
printf("ERROR recv_msg returned %d\n", ret);
return ret;
}
}
for (j = 0; j < concurrent_msgs; j++) {
ret = wait_recvs();
if (ret < 1)
return ret;
}
if (ft_check_opts(FT_OPT_VERIFY_DATA)) {
for (j = 0; j < concurrent_msgs; j++) {
op_buf = get_rx_buf(j);
if (ft_check_buf(op_buf + ft_rx_prefix_size(),
opts.transfer_size))
return -FI_EOTHER;
}
}
for (j = 0; j < concurrent_msgs; j++) {
ret = ft_get_tx_comp(tx_seq);
if (ret)
return ret;
}
if (i % 100 == 0)
printf("PID %d GOOD iter %d/%ld completed\n",
getpid(), i, num_iters);
}
(void) ft_sync();
printf("PID %d GOOD all done\n", getpid());
return ret;
}
