Beispiel #1
0
static int run_test()
{
	int ret, i;

	/* Post recvs */
	for (i = 0; i < ep_cnt; i++) {
		fprintf(stdout, "Posting recv for ctx: %d\n", i);
		ret = fi_recv(srx_ctx, rx_buf, rx_size, fi_mr_desc(mr),
				FI_ADDR_UNSPEC, NULL);
		if (ret) {
			FT_PRINTERR("fi_recv", ret);
			return ret;
		}
		rx_seq++;
	}

	if (opts.dst_addr) {
		/* Post sends addressed to remote EPs */
		for (i = 0; i < ep_cnt; i++) {
			fprintf(stdout, "Posting send to remote ctx: %d\n", i);
			ret = fi_send(ep_array[i], tx_buf, tx_size, fi_mr_desc(mr),
					addr_array[i], NULL);
			if (ret) {
				FT_PRINTERR("fi_send", ret);
				return ret;
			}

			ret = ft_get_tx_comp(++tx_seq);
			if (ret)
				return ret;
		}
	}

	/* Wait for recv completions */
	ret = ft_get_rx_comp(rx_seq);
	if (ret)
		return ret;

	if (!opts.dst_addr) {
		/* Post sends addressed to remote EPs */
		for (i = 0; i < ep_cnt; i++) {
			fprintf(stdout, "Posting send to remote ctx: %d\n", i);
			ret = fi_send(ep_array[i], tx_buf, tx_size, fi_mr_desc(mr),
					addr_array[i], NULL);
			if (ret) {
				FT_PRINTERR("fi_send", ret);
				return ret;
			}

			ret = ft_get_tx_comp(++tx_seq);
			if (ret)
				return ret;
		}
	}

	return 0;
}
Beispiel #2
0
static int init_av(void)
{
	size_t addrlen;
	int ret, i;

	if (opts.dst_addr) {
		ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL);
		if (ret)
			return ret;

		addrlen = FT_MAX_CTRL_MSG;
		ret = fi_getname(&sep->fid, tx_buf, &addrlen);
		if (ret) {
			FT_PRINTERR("fi_getname", ret);
			return ret;
		}

		ret = fi_send(tx_ep[0], tx_buf, addrlen,
				fi_mr_desc(mr), remote_fi_addr, NULL);
		if (ret) {
			FT_PRINTERR("fi_send", ret);
			return ret;
		}

		ret = wait_for_comp(rxcq_array[0]);
		if (ret)
			return ret;
	} else {
		ret = wait_for_comp(rxcq_array[0]);
		if (ret)
			return ret;

		ret = ft_av_insert(av, rx_buf, 1, &remote_fi_addr, 0, NULL);
		if (ret)
			return ret;

		ret = fi_send(tx_ep[0], tx_buf, 1,
				fi_mr_desc(mr), remote_fi_addr, NULL);
		if (ret) {
			FT_PRINTERR("fi_send", ret);
			return ret;
		}
	}

	for (i = 0; i < ctx_cnt; i++)
		remote_rx_addr[i] = fi_rx_addr(remote_fi_addr, i, rx_ctx_bits);

	ret = fi_recv(rx_ep[0], rx_buf, rx_size, fi_mr_desc(mr), 0, NULL);
	if (ret) {
		FT_PRINTERR("fi_recv", ret);
		return ret;
	}

	ret = wait_for_comp(txcq_array[0]);
	return ret;
}
Beispiel #3
0
static int run_test()
{
	int ret, i;

	/* Post recvs */
	for (i = 0; i < ep_cnt; i++) {
		fprintf(stdout, "Posting recv for ctx: %d\n", i);
		ret = fi_recv(srx_ctx, buf, buffer_size, fi_mr_desc(mr),
				FI_ADDR_UNSPEC, NULL);
		if (ret) {
			FT_PRINTERR("fi_recv", ret);
			return ret;
		}
	}

	if (opts.dst_addr) {
		/* Post sends addressed to remote EPs */
		for (i = 0; i < ep_cnt; i++) {
			fprintf(stdout, "Posting send to remote ctx: %d\n", i);
			ret = fi_send(ep[i], buf, transfer_size, fi_mr_desc(mr),
					remote_fi_addr[i], NULL); 
			if (ret) {
				FT_PRINTERR("fi_send", ret);
				return ret;
			}

			wait_for_completion(scq, 1);
		}
	}

	/* Wait for recv completions */
	for (i = 0; i < ep_cnt; i++) {
		wait_for_completion(rcq, 1);
	}

	if (!opts.dst_addr) {
		/* Post sends addressed to remote EPs */
		for (i = 0; i < ep_cnt; i++) {
			fprintf(stdout, "Posting send to remote ctx: %d\n", i);
			ret = fi_send(ep[i], buf, transfer_size, fi_mr_desc(mr),
					remote_fi_addr[i], NULL); 
			if (ret) {
				FT_PRINTERR("fi_send", ret);
				return ret;
			}

			wait_for_completion(scq, 1);
		}
	}

	return 0;
}
Beispiel #4
0
static int send_xfer(int size)
{
	struct fi_cq_entry comp;
	int ret;

	while (!credits) {
		ret = fi_cq_read(scq, &comp, 1);
		if (ret > 0) {
			goto post;
		} else if (ret < 0 && ret != -FI_EAGAIN) {
			if (ret == -FI_EAVAIL) {
				cq_readerr(scq, "scq");
			} else {
				FT_PRINTERR("fi_cq_read", ret);
			}
			return ret;
		}
	}

	credits--;
post:
	ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), remote_fi_addr,
			&fi_ctx_send);
	if (ret)
		FT_PRINTERR("fi_send", ret);

	return ret;
}
Beispiel #5
0
static int run_test()
{
	int ret = 0, i;

	if (opts.dst_addr) {
		for (i = 0; i < ctx_cnt && !ret; i++) {
			fprintf(stdout, "Posting send for ctx: %d\n", i);
			ret = fi_send(tx_ep[i], buf, tx_size, fi_mr_desc(mr),
					remote_rx_addr[i], NULL);
			if (ret) {
				FT_PRINTERR("fi_send", ret);
				return ret;
			}

			ret = wait_for_comp(txcq_array[i]);
		}
	} else {
		for (i = 0; i < ctx_cnt && !ret; i++) {
			fprintf(stdout, "wait for recv completion for ctx: %d\n", i);
			ret = wait_for_comp(rxcq_array[i]);
		}
	}

	return ret;
}
Beispiel #6
0
void cm_basic_send(void)
{
	int ret;
	int source_done = 0, dest_done = 0;
	struct fi_cq_tagged_entry cqe;
	ssize_t sz;
	uint64_t source = 0xa4321234a4321234,
		 target = 0xb5678901b5678901;

	sz = fi_send(cli_ep, &source, 8, 0, 0, &target);
	cr_assert_eq(sz, 0);

	sz = fi_recv(srv_ep, &target, 8, 0, 0, &source);
	cr_assert_eq(sz, 0);

	/* need to progress both CQs simultaneously for rendezvous */
	do {
		ret = fi_cq_read(cli_cq, &cqe, 1);
		if (ret == 1) {
			cr_assert_eq(cqe.op_context, &target);
			source_done = 1;
		}

		ret = fi_cq_read(srv_cq, &cqe, 1);
		if (ret == 1) {
			cr_assert_eq(cqe.op_context, &source);
			dest_done = 1;
		}
	} while (!source_done || !dest_done);

	cr_assert_eq(source, target);
	dbg_printf("Basic send/recv complete! (0x%lx, 0x%lx)\n",
		   source, target);
}
Beispiel #7
0
static int send_xfer(int size)
{
	struct fi_cq_entry comp;
	int ret;

	while (!credits) {
		ret = fi_cq_read(scq, &comp, 1);
		if (ret > 0) {
			goto post;
		} else if (ret < 0) {
			if (ret == -FI_EAVAIL) {
				cq_readerr(scq, "scq");
			} else {
				printf("scq read %d (%s)\n", ret, fi_strerror(-ret));
			}
			return ret;
		}
	}

	credits--;
post:
	ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), NULL);
	if (ret)
		printf("fi_send %d (%s)\n", ret, fi_strerror(-ret));

	return ret;
}
Beispiel #8
0
static int send_xfer(int size)
{
	struct fi_cq_entry comp;
	int ret;

	while (!credits) {
		ret = fi_cq_read(scq, &comp, 1);
		if (ret > 0) {
			goto post;
		} else if (ret < 0) {
			printf("RCQ read %d (%s)\n", ret, fi_strerror(-ret));
			return ret;
		}
	}

	credits--;
post:
	ret = dst_addr ?
		fi_send(ep, buf_ptr, (size_t) size, fi_mr_desc(mr), NULL) :
		fi_sendto(ep, buf_ptr, (size_t) size, fi_mr_desc(mr),
				client_addr, NULL);
	if (ret)
		printf("fi_send %d (%s)\n", ret, fi_strerror(-ret));

	return ret;
}
Beispiel #9
0
/*
 * ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len,
 *		void *desc, fi_addr_t dest_addr, void *context);
 *
 * ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len,
 *		void *desc, fi_addr_t src_addr, void *context);
 */
void api_send_recv(int len)
{
	ssize_t sz;
	uint64_t caps = fi[0]->caps;

	rdm_api_init_data(source, len, 0xab);
	rdm_api_init_data(target, len, 0);

	sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
	if (MSG_SEND_ALLOWED(caps)) {
		cr_assert(sz == 0, "fi_send failed caps:0x%lx err:%ld",
			  caps, sz);
	} else {
		cr_assert(sz < 0, "fi_send should fail caps:0x%lx err:%ld",
			  caps, sz);
	}

	sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
	if (MSG_RECV_ALLOWED(caps)) {
		cr_assert(sz == 0, "fi_recv failed caps:0x%lx err:%ld",
			  caps, sz);
	} else {
		cr_assert(sz < 0, "fi_recv should fail caps:0x%lx err:%ld",
			  caps, sz);
	}
}
Beispiel #10
0
/*
 * ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len,
 *		void *desc, fi_addr_t dest_addr, void *context);
 *
 * ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len,
 *		void *desc, fi_addr_t src_addr, void *context);
 */
void do_send(int len)
{
	int ret;
	ssize_t sz;
	struct fi_cq_entry cqe;

	rdm_sr_init_data(source, len, 0xab);
	rdm_sr_init_data(target, len, 0);

	sz = fi_send(ep[0], source, len, loc_mr, gni_addr[1], target);
	cr_assert_eq(sz, 0);

	sz = fi_recv(ep[1], target, len, rem_mr, gni_addr[0], source);
	cr_assert_eq(sz, 0);

	while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN) {
		pthread_yield();
	}

	cr_assert_eq(ret, 1);
	cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)target);

	dbg_printf("got send context event!\n");

	while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
		pthread_yield();
	}

	cr_assert_eq(ret, 1);
	cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)source);

	dbg_printf("got recv context event!\n");

	cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
Beispiel #11
0
static inline int
rxm_ep_send_atomic_req(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn,
		       struct rxm_tx_atomic_buf *tx_buf, uint64_t len)
{
	int ret;

	/* Atomic request TX completion processing is performed when the
	 * software generated atomic response message is received. */
	tx_buf->hdr.state = RXM_ATOMIC_RESP_WAIT;
	if (len <= rxm_ep->inject_limit)
		ret = fi_inject(rxm_conn->msg_ep, &tx_buf->pkt, len, 0);
	else
		ret = fi_send(rxm_conn->msg_ep, &tx_buf->pkt, len,
			      tx_buf->hdr.desc, 0, tx_buf);
	if (ret == -FI_EAGAIN)
		rxm_ep_do_progress(&rxm_ep->util_ep);

	if (OFI_LIKELY(!ret))
		FI_DBG(&rxm_prov, FI_LOG_EP_DATA, "sent atomic request: op: %"
		       PRIu8 " msg_id: 0x%" PRIx64 "\n", tx_buf->pkt.hdr.op,
		       tx_buf->pkt.ctrl_hdr.msg_id);
	else if (OFI_UNLIKELY(ret != -FI_EAGAIN))
		FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "unable to send atomic "
			"request: op: %" PRIu8 " msg_id: 0x%" PRIx64 "\n",
			tx_buf->pkt.hdr.op, tx_buf->pkt.ctrl_hdr.msg_id);
	return ret;
}
Beispiel #12
0
void do_multirecv(int len)
{
	int i, ret;
	ssize_t sz;
	struct fi_cq_tagged_entry s_cqe, d_cqe;
	struct iovec iov;
	struct fi_msg msg;
	uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
	uint64_t r_e[NUMEPS] = {0};
	uint64_t flags;
	int nrecvs = 3;

	rdm_sr_init_data(source, len, 0xab);
	rdm_sr_init_data(target, len, 0);

	/* Post receives first to force matching in SMSG callback. */
	iov.iov_base = target;
	iov.iov_len = len * nrecvs + 63;

	msg.msg_iov = &iov;
	msg.desc = (void **)rem_mr;
	msg.iov_count = 1;
	msg.addr = gni_addr[0];
	msg.context = source;
	msg.data = (uint64_t)source;

	sz = fi_recvmsg(ep[1], &msg, FI_MULTI_RECV);
	cr_assert_eq(sz, 0);

	for (i = 0; i < nrecvs; i++) {
		sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1],
			     target);
		cr_assert_eq(sz, 0);
	}

	/* need to progress both CQs simultaneously for rendezvous */
	do {
		ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
		if (ret == 1) {
			rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND),
					 0, 0, 0, false);
			s[0]++;
		}
		ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
		flags = (r[1] < (nrecvs -1 )) ? FI_MSG | FI_RECV :
				FI_MSG | FI_RECV | FI_MULTI_RECV;
		if (ret == 1) {
			rdm_sr_check_cqe(&d_cqe, source,
					 flags,
					 target + (r[1] * len), len, 0, true);
			cr_assert(rdm_sr_check_data(source, d_cqe.buf, len),
				  "Data mismatch");
			r[1]++;
		}
	} while (s[0] < nrecvs || r[1] < nrecvs);

	rdm_sr_check_cntrs(s, r, s_e, r_e);

	dbg_printf("got context events!\n");
}
Beispiel #13
0
static int run_test()
{
	int ret = 0, i;
	uint32_t data;
	uint32_t *tb = (uint32_t *)tx_buf;
	uint32_t *rb = (uint32_t *)rx_buf;

	if (opts.dst_addr) {
		for (i = 0; i < ctx_cnt && !ret; i++) {
			fprintf(stdout, "Posting send for ctx: %d\n", i);
			tb[0] = DATA + i;
			ret = fi_send(tx_ep[i], tx_buf, tx_size, mr_desc,
				      remote_rx_addr[i], NULL);
			if (ret) {
				FT_PRINTERR("fi_send", ret);
				return ret;
			}

			ret = wait_for_comp(txcq_array[i]);
		}
	} else {
		for (i = 0; i < ctx_cnt && !ret; i++) {
			fprintf(stdout, "wait for recv completion for ctx: %d\n", i);
			ret = wait_for_comp(rxcq_array[i]);

			data = DATA + i;
			if (memcmp(&data, rx_buf, 4) != 0) {
				fprintf(stdout, "failed compare expected 0x%x,"
					" read 0x%x\n", data, rb[0]);
			}
		}
	}

	return ret;
}
Beispiel #14
0
void do_sendrecvv_alignment(int slen, int dlen, int offset)
{
	int i, ret, iov_cnt;
	ssize_t sz;
	int source_done = 0, dest_done = 0;
	struct fi_cq_tagged_entry s_cqe, d_cqe;
	uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
	uint64_t r_e[NUMEPS] = {0};
	uint64_t iov_s_buf = (uint64_t) iov_src_buf;

	iov_s_buf += offset;

	for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
		for (i = 0; i < iov_cnt; i++) {
			d_iov[i].iov_base = dest_iov[i].iov_base;
			d_iov[i].iov_base = (void *) ((uint64_t)d_iov[i].iov_base + offset);
			rdm_sr_init_data(d_iov[i].iov_base, dlen - offset, 0);
			d_iov[i].iov_len = dlen - offset;
		}

		rdm_sr_init_data((void *) iov_s_buf, (slen - offset) * iov_cnt, 0xab);

		sz = fi_send(ep[0], (void *) iov_s_buf, (slen - offset) * iov_cnt, NULL, gni_addr[1],
			     d_iov);
		cr_assert_eq(sz, 0);

		sz = fi_recvv(ep[1], d_iov, NULL, iov_cnt, gni_addr[0], (void *) iov_s_buf);
		cr_assert_eq(sz, 0);

		/*  need to progress both CQs simultaneously for rendezvous */
		do {
			ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
			if (ret == 1) {
				source_done = 1;
			}
			ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
			if (ret == 1) {
				dest_done = 1;
			}
		} while (!(source_done && dest_done));

		rdm_sr_check_cqe(&s_cqe, d_iov, (FI_MSG|FI_SEND), 0, 0, 0, false);
		rdm_sr_check_cqe(&d_cqe, (void *) iov_s_buf, (FI_MSG|FI_RECV), d_iov,
				 MIN((slen - offset) * iov_cnt, (dlen - offset) * iov_cnt), 0, false);

		s[0] = 1; r[1] = 1;
		rdm_sr_check_cntrs(s, r, s_e, r_e);

		dbg_printf("got context events!\n");

		cr_assert(rdm_sr_check_iov_data(d_iov, (void *) iov_s_buf,
						iov_cnt, (slen - offset) * iov_cnt),
			  "Data mismatch");
		source_done = dest_done = 0;
	}
}
Beispiel #15
0
/*
ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc,
		 size_t count, fi_addr_t src_addr, void *context);
*/
void do_recvv(int len)
{
	int i, ret, iov_cnt;
	ssize_t sz;
	int source_done = 0, dest_done = 0;
	struct fi_cq_tagged_entry s_cqe, d_cqe;
	uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
	uint64_t r_e[NUMEPS] = {0};

	sz = fi_recvv(ep[1], NULL, NULL, IOV_CNT, gni_addr[0], iov_src_buf);
	cr_assert_eq(sz, -FI_EINVAL);

	sz = fi_recvv(ep[1], dest_iov, NULL, IOV_CNT + 1, gni_addr[0], iov_src_buf);
	cr_assert_eq(sz, -FI_EINVAL);

	for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
		rdm_sr_init_data(iov_src_buf, len * iov_cnt, 0xab);

		for (i = 0; i < iov_cnt; i++) {
			rdm_sr_init_data(dest_iov[i].iov_base, len, 0);
			dest_iov[i].iov_len = len;
		}

		sz = fi_send(ep[0], iov_src_buf, len * iov_cnt, NULL, gni_addr[1],
			     dest_iov);
		cr_assert_eq(sz, 0);

		sz = fi_recvv(ep[1], dest_iov, NULL, iov_cnt, gni_addr[0], iov_src_buf);
		cr_assert_eq(sz, 0);

		/*  need to progress both CQs simultaneously for rendezvous */
		do {
			ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
			if (ret == 1) {
				source_done = 1;
			}
			ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
			if (ret == 1) {
				dest_done = 1;
			}
		} while (!(source_done && dest_done));

		rdm_sr_check_cqe(&s_cqe, dest_iov, (FI_MSG|FI_SEND), 0, 0, 0, false);
		rdm_sr_check_cqe(&d_cqe, iov_src_buf, (FI_MSG|FI_RECV), dest_iov,
				 len * iov_cnt, 0, false);

		s[0] = 1; r[1] = 1;
		rdm_sr_check_cntrs(s, r, s_e, r_e);

		dbg_printf("got context events!\n");

		cr_assert(rdm_sr_check_iov_data(dest_iov, iov_src_buf, iov_cnt, len * iov_cnt),
			  "Data mismatch");
		source_done = dest_done = 0;
	}
}
Beispiel #16
0
static int send_msg(int size)
{
	int ret;
	
	ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), remote_fi_addr, 
			&fi_ctx_send);
	if (ret)
		FT_PRINTERR("fi_send", ret);

	return wait_for_completion(scq, 1);
}
Beispiel #17
0
static int send_msg(int size)
{
	int ret;
	
	ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), remote_fi_addr, 
			&fi_ctx_send);
	if (ret)
		fprintf(stderr, "fi_send %d (%s)\n", ret, fi_strerror(-ret));

	return wait_for_completion(scq, 1);
}
Beispiel #18
0
/*
ssize_t (*recvmsg)(struct fid_ep *ep, const struct fi_msg *msg,
		   uint64_t flags);
*/
void do_recvmsg(int len)
{
	int ret;
	ssize_t sz;
	int source_done = 0, dest_done = 0;
	struct fi_cq_tagged_entry s_cqe, d_cqe;
	struct iovec iov;
	struct fi_msg msg;
	uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
	uint64_t r_e[NUMEPS] = {0};

	rdm_sr_init_data(source, len, 0xab);
	rdm_sr_init_data(target, len, 0);

	sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
	cr_assert_eq(sz, 0);

	iov.iov_base = target;
	iov.iov_len = len;

	msg.msg_iov = &iov;
	msg.desc = (void **)rem_mr;
	msg.iov_count = 1;
	msg.addr = gni_addr[0];
	msg.context = source;
	msg.data = (uint64_t)source;

	sz = fi_recvmsg(ep[1], &msg, 0);
	cr_assert_eq(sz, 0);

	/* need to progress both CQs simultaneously for rendezvous */
	do {
		ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
		if (ret == 1) {
			source_done = 1;
		}
		ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
		if (ret == 1) {
			dest_done = 1;
		}
	} while (!(source_done && dest_done));

	rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0, false);
	rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0,
			false);

	s[0] = 1; r[1] = 1;
	rdm_sr_check_cntrs(s, r, s_e, r_e);

	dbg_printf("got context events!\n");

	cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
Beispiel #19
0
/*
 * ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len,
 *		void *desc, fi_addr_t dest_addr, void *context);
 *
 * ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len,
 *		void *desc, fi_addr_t src_addr, void *context);
 */
void do_send(int len)
{
	int ret;
	int source_done = 0, dest_done = 0;
	int scanceled = 0, dcanceled = 0;
	struct fi_cq_tagged_entry s_cqe, d_cqe;
	ssize_t sz;
	uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
	uint64_t r_e[NUMEPS] = {0};

	rdm_sr_init_data(source, len, 0xab);
	rdm_sr_init_data(target, len, 0);

	sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
	cr_assert_eq(sz, 0);

	sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
	cr_assert_eq(sz, 0);

	/* need to progress both CQs simultaneously for rendezvous */
	do {
		ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
		if (ret == 1) {
			source_done = 1;
		}
		if (ret == -FI_EAVAIL) {
			if (rdm_sr_check_canceled(msg_cq[0]))
				scanceled = 1;
		}
		ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
		if (ret == 1) {
			dest_done = 1;
		}
		if (ret == -FI_EAVAIL) {
			if (rdm_sr_check_canceled(msg_cq[1]))
				dcanceled = 1;
		}
	} while (!((source_done || scanceled) && (dest_done || dcanceled)));

	/* no further checking needed */
	if (dgram_should_fail && (scanceled || dcanceled))
		return;

	rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
	rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);

	s[0] = 1; r[1] = 1;
	rdm_sr_check_cntrs(s, r, s_e, r_e);

	dbg_printf("got context events!\n");

	cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
Beispiel #20
0
static int pp_post_send(struct pingpong_context *ctx)
{
	int rc = 0;

	rc = fi_send(ctx->ep, ctx->buf, ctx->size, fi_mr_desc(ctx->mr),
		     0, (void *)(uintptr_t)PINGPONG_SEND_WCID);
	if (rc) {
		FT_PRINTERR("fi_send", rc);
		return 1;
	}

	return 0;
}
Beispiel #21
0
void api_cntr_send_recv(int len)
{
	ssize_t sz;

	api_cntr_init_data(source, len, 0xab);
	api_cntr_init_data(target, len, 0);

	sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
	api_cntr_send_allowed(sz, cntr_bind_flags, "fi_send");

	sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
	api_cntr_recv_allowed(sz, cntr_bind_flags, "fi_recv");
}
Beispiel #22
0
static int send_recv()
{
	struct fi_cq_entry comp;
	int ret;

	if (opts.dst_addr) {
		/* Client */
		fprintf(stdout, "Posting a send...\n");
		sprintf(buf, "Hello from Client!"); 
		ret = fi_send(ep, buf, sizeof("Hello from Client!"), 
				fi_mr_desc(mr), remote_fi_addr, &fi_ctx_send);
		if (ret) {
			FT_PRINTERR("fi_send", ret);
			return ret;
		}

		/* Read send queue */
		do {
			ret = fi_cq_read(scq, &comp, 1);
			if (ret < 0 && ret != -FI_EAGAIN) {
				FT_PRINTERR("fi_cq_read", ret);
				return ret;
			}
		} while (ret == -FI_EAGAIN);

		fprintf(stdout, "Send completion received\n");
	} else {
		/* Server */
		fprintf(stdout, "Posting a recv...\n");
		ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, 
				&fi_ctx_recv);
		if (ret) {
			FT_PRINTERR("fi_recv", ret);
			return ret;
		}

		/* Read recv queue */
		fprintf(stdout, "Waiting for client...\n");
		do {
			ret = fi_cq_read(rcq, &comp, 1);
			if (ret < 0 && ret != -FI_EAGAIN) {
				FT_PRINTERR("fi_cq_read", ret);
				return ret;
			}
		} while (ret == -FI_EAGAIN);

		fprintf(stdout, "Received data from client: %s\n", (char *)buf);
	}

	return 0;
}
Beispiel #23
0
static int send_msg(int size)
{
	int ret;

	ret = fi_send(ep_array[0], buf, (size_t) size, fi_mr_desc(mr),
			addr_array[0], &tx_ctx);
	if (ret) {
		FT_PRINTERR("fi_send", ret);
		return ret;
	}

	ret = ft_get_tx_comp(++tx_seq);
	return ret;
}
static int send_msg(int size)
{
	int ret;

	ret = fi_send(ep_array[0], buf, (size_t) size, fi_mr_desc(mr),
			remote_fi_addr[0], &fi_ctx_send);
	if (ret) {
		FT_PRINTERR("fi_send", ret);
		return ret;
	}

	ret = ft_wait_for_comp(txcq, 1);

	return ret;
}
Beispiel #25
0
static int send_msg(int size)
{
	int ret;

	ret = fi_send(tx_ep[0], buf, (size_t) size, fi_mr_desc(mr),
			remote_rx_addr[0], &fi_ctx_send);
	if (ret) {
		FT_PRINTERR("fi_send", ret);
		return ret;
	}

	ret = wait_for_completion(scq[0], 1);

	return ret;
}
Beispiel #26
0
/*
ssize_t (*recvmsg)(struct fid_ep *ep, const struct fi_msg *msg,
		uint64_t flags);
 */
void do_recvmsg(int len)
{
	int ret;
	ssize_t sz;
	struct fi_cq_entry cqe;
	struct fi_msg msg;
	struct iovec iov;

	rdm_sr_init_data(source, len, 0xab);
	rdm_sr_init_data(target, len, 0);

	sz = fi_send(ep[0], source, len, loc_mr, gni_addr[1], target);
	cr_assert_eq(sz, 0);

	iov.iov_base = target;
	iov.iov_len = len;

	msg.msg_iov = &iov;
	msg.desc = (void **)&rem_mr;
	msg.iov_count = 1;
	msg.addr = gni_addr[0];
	msg.context = source;
	msg.data = (uint64_t)source;

	sz = fi_recvmsg(ep[1], &msg, 0);
	cr_assert_eq(sz, 0);

	while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN) {
		pthread_yield();
	}

	cr_assert_eq(ret, 1);
	cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)target);

	dbg_printf("got send context event!\n");

	while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
		pthread_yield();
	}

	cr_assert_eq(ret, 1);
	cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)source);

	dbg_printf("got recv context event!\n");

	cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
Beispiel #27
0
/*
 * ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len,
 *		void *desc, fi_addr_t dest_addr, void *context);
 *
 * ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len,
 *		void *desc, fi_addr_t src_addr, void *context);
 */
static void do_send(int len)
{
	int ret;
	int source_done = 0, dest_done = 0;
	struct fi_cq_tagged_entry s_cqe = { (void *) -1, UINT_MAX, UINT_MAX,
					    (void *) -1, UINT_MAX, UINT_MAX };
	struct fi_cq_tagged_entry d_cqe = { (void *) -1, UINT_MAX, UINT_MAX,
					    (void *) -1, UINT_MAX, UINT_MAX };
	struct fi_cq_err_entry d_err_cqe;
	struct fi_cq_err_entry s_err_cqe;

	ssize_t sz;

	memset(&d_err_cqe, -1, sizeof(struct fi_cq_err_entry));
	memset(&s_err_cqe, -1, sizeof(struct fi_cq_err_entry));

	rdm_str_addr_sr_init_data(source, len, 0xab);
	rdm_str_addr_sr_init_data(target, len, 0);

	sz = fi_send(ep[0], source, len, NULL, gni_addr[1], target);
	cr_assert_eq(sz, 0);

	sz = fi_recv(ep[1], target, len, NULL, FI_ADDR_UNSPEC, source);
	cr_assert_eq(sz, 0);

	/* need to progress both CQs simultaneously for rendezvous */
	do {
		ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
		if (ret == 1) {
			source_done = 1;
		}

		ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
		if (ret == 1) {
			dest_done = 1;
		}

		if (ret == -FI_EAVAIL) {
			ret = rdm_str_addr_sr_check_err_cqe(msg_cq[1]);
			cr_assert((ret == FI_SUCCESS), "Err CQE processing failed");
			dest_done = 1;
		}
	} while (!(source_done && dest_done));

	cr_assert(rdm_str_addr_sr_check_data(source, target, len), "Data mismatch");
}
Beispiel #28
0
static int post_sends(struct cma_node *node)
{
	int i, ret = 0;

	if (!node->connected || !hints->tx_attr->size)
		return 0;

	for (i = 0; i < hints->tx_attr->size && !ret; i++) {
		ret = fi_send(node->ep, node->mem, hints->ep_attr->max_msg_size,
				node->mrdesc, 0, node);
		if (ret) {
			FT_PRINTERR("fi_send", ret);
			break;
		}
	}
	return ret;
}
Beispiel #29
0
Test(rdm_sr, send_readfrom)
{
	int ret;
	int source_done = 0, dest_done = 0;
	struct fi_cq_tagged_entry s_cqe, d_cqe;
	ssize_t sz;
	fi_addr_t src_addr;
	int len = 64;
	uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
	uint64_t r_e[NUMEPS] = {0};

	rdm_sr_init_data(source, len, 0xab);
	rdm_sr_init_data(target, len, 0);

	sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
	cr_assert_eq(sz, 0);

	sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
	cr_assert_eq(sz, 0);

	/* need to progress both CQs simultaneously for rendezvous */
	do {
		ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
		if (ret == 1) {
			source_done = 1;
		}
		ret = fi_cq_readfrom(msg_cq[1], &d_cqe, 1, &src_addr);
		if (ret == 1) {
			dest_done = 1;
		}
	} while (!(source_done && dest_done));

	rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
	rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);

	s[0] = 1; r[1] = 1;
	rdm_sr_check_cntrs(s, r, s_e, r_e);

	cr_assert(src_addr == gni_addr[0], "src_addr mismatch");

	dbg_printf("got context events!\n");

	cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
Beispiel #30
0
ssize_t ft_post_tx(size_t size)
{
	ssize_t ret;

	if (hints->caps & FI_TAGGED) {
		ret = fi_tsend(ep, tx_buf, size + ft_tx_prefix_size(),
				fi_mr_desc(mr), remote_fi_addr, tx_seq, &tx_ctx);
	} else {
		ret = fi_send(ep, tx_buf, size + ft_tx_prefix_size(),
				fi_mr_desc(mr), remote_fi_addr, &tx_ctx);
	}
	if (ret) {
		FT_PRINTERR("transmit", ret);
		return ret;
	}

	tx_seq++;
	return 0;
}