Example #1
0
File: rpc.c Project: CCI/cci
static int
transfer_data(int cookie)
{
	int ret = 0;

	assert(is_server);

	ret = cci_rma(connection, ack, opts.ack_size, local, 0,
			remote, 0, opts.transfer_size,
			(void*)((uintptr_t)cookie), opts.rma_flags);
	check_return(endpoint, "cci_rma", ret, 0);

	return ret;
}
Example #2
0
static void do_client(void)
{
	int ret;
	uint32_t min = 0, max;
	struct timeval start, end;
	char *func;
	char *header = "Done";

	/* initiate connect */
	ret =
	    cci_connect(endpoint, server_uri, &opts, sizeof(opts), attr, NULL,
			0, NULL);
	check_return(endpoint, "cci_connect", ret, 1);

	/* poll for connect completion */
	while (!connect_done)
		poll_events();

	if (!connection) {
		fprintf(stderr, "no connection\n");
		return;
	}

	while (!ready)
		poll_events();

	if (opts.method == MSGS) {
		func = "cci_send";
		max = connection->max_send_size;
	} else {
		func = "cci_rma";
		max = opts.max_rma_size;
	}

	ret = posix_memalign((void **)&buffer, 4096, max);
	check_return(endpoint, "memalign buffer", ret, 1);

	memset(buffer, 'b', max);

	if (opts.method != MSGS) {
		int flags = 0;

		/* for the client, we want the opposite of the opts.method.
		 * when testing RMA WRITE, we only need READ access.
		 * when testing RMA READ, we need WRITE access.
		 */

		if (opts.method == RMA_WRITE)
			flags = CCI_FLAG_READ;
		else if (opts.method == RMA_READ)
			flags = CCI_FLAG_WRITE;

		ret = cci_rma_register(endpoint, buffer, max, flags,
				       &local_rma_handle);
		check_return(endpoint, "cci_rma_register", ret, 1);
		fprintf(stderr, "local_rma_handle is %p\n",
			(void*)local_rma_handle);
		min = 1;
		if (opts.method == RMA_WRITE)
			opts.flags |= CCI_FLAG_WRITE;
		else
			opts.flags |= CCI_FLAG_READ;
	}

	if (remote_completion) {
		rmt_comp_msg = header;
		rmt_comp_len = 4;
	}

	if (opts.method == MSGS)
		printf("Bytes\t\tLatency (one-way)\tThroughput\n");
	else
		printf("Bytes\t\tLatency (round-trip)\tThroughput\n");

	/* begin communication with server */
	for (current_size = min; current_size <= max;) {
		double lat = 0.0;
		double bw = 0.0;

		if (opts.method == MSGS)
			ret =
			    cci_send(connection, buffer, current_size, NULL,
				     opts.flags);
		else
			ret = cci_rma(connection, rmt_comp_msg, rmt_comp_len,
				      local_rma_handle, 0,
				      &opts.rma_handle, 0,
				      current_size, (void *)1, opts.flags);
		check_return(endpoint, func, ret, 1);

		while (count < warmup)
			poll_events();

		gettimeofday(&start, NULL);

		while (count < warmup + iters)
			poll_events();

		gettimeofday(&end, NULL);

		if (opts.method == MSGS)
			lat = usecs(start, end) / (double)iters / 2.0;
		else
			lat = usecs(start, end) / (double)iters;

		bw = (double)current_size / lat;
		printf("%8d\t%8.2f us\t\t%8.2f MB/s\n", current_size, lat,
		       bw);

		count = 0;

		if (current_size == 0)
			current_size++;
		else
			current_size *= 2;

		if (current_size >= 64 * 1024) {
			if (iters >= 32)
				iters /= 2;
			if (warmup >= 4)
				warmup /= 2;
		}
	}

	ret = cci_send(connection, "bye", 3, (void *)0xdeadbeef, opts.flags);
	check_return(endpoint, "cci_send", ret, 0);

	while (!done)
		poll_events();

	if (opts.method != MSGS) {
		ret = cci_rma_deregister(endpoint, local_rma_handle);
		check_return(endpoint, "cci_rma_deregister", ret, 1);
	}

	printf("client done\n");
	sleep(1);

	return;
}
Example #3
0
static void do_client(void)
{
	int ret;
	uint32_t min = 1;

	/* initiate connect */
	msg.request.type = MSG_CONTROL;

	ret =
	    cci_connect(endpoint, server_uri, &msg, sizeof(msg.generic), attr,
			(void*)(uintptr_t)CONTROL, 0, NULL);
	check_return(endpoint, "cci_connect", ret, 1);

	msg.request.type = MSG_CONN_REQ;
	msg.request.opts = opts;

	ret =
	    cci_connect(endpoint, server_uri, &msg, sizeof(msg.request), attr,
			(void*)(uintptr_t)TEST, 0, NULL);
	check_return(endpoint, "cci_connect", ret, 1);
	/* poll for connect completion */
	while (!connect_done)
		poll_events();

	if (!test) {
		fprintf(stderr, "no connection\n");
		return;
	}

	while (!ready)
		poll_events();

	ret = posix_memalign((void **)&buffer, 4096, opts.reg_len);
	check_return(endpoint, "memalign buffer", ret, 1);

	memset(buffer, 0xaa, opts.reg_len);

	init_buffer(1);
	print_buffer(buffer, (int) opts.reg_len);

	/* for the client, we do not need remote access flags */

	ret = cci_rma_register(endpoint, buffer, opts.reg_len, 0, &local_rma_handle);
	check_return(endpoint, "cci_rma_register", ret, 1);

	if (opts.method == RMA_WRITE)
		opts.flags = CCI_FLAG_WRITE;
	else
		opts.flags = CCI_FLAG_READ;

	/* begin communication with server */
	for (current_size = min; current_size <= length;) {
		void *ptr = (void*)((uintptr_t)buffer + local_offset);

		msg.check.type = MSG_RMA_CHK;
		msg.check.offset = remote_offset;
		msg.check.len = current_size;
		/* Compute the CRC only on a valid buffer */
		if (current_size + local_offset <= opts.reg_len)
			msg.check.crc = crc32(0, ptr, current_size);
		else
			msg.check.crc = 0;
		msg_len = sizeof(msg.check);
		print_buffer(ptr, current_size);

		fprintf(stderr, "Testing length %9u ... ", current_size);

		ret = cci_rma(test, &msg, msg_len,
			      local_rma_handle, local_offset,
			      &remote_rma_handle, remote_offset,
			      current_size, NULL, opts.flags);
		check_return(endpoint, "cci_rma", ret, 1);

		while (count < iters)
			poll_events();

		if (test)
			fprintf(stderr, "success.\n");
		else
			goto out;

		count = 0;
		current_size *= 2;

		if (current_size >= 64 * 1024) {
			if (iters >= 32)
				iters /= 2;
		}
	}

out:
	ret = cci_send(control, "bye", 3, (void *)0xdeadbeef, 0);
	check_return(endpoint, "cci_send", ret, 0);

	while (!done)
		poll_events();

	ret = cci_rma_deregister(endpoint, local_rma_handle);
	check_return(endpoint, "cci_rma_deregister", ret, 1);

	printf("client done\n");
	sleep(1);

	return;
}
Example #4
0
static void poll_events(void)
{
	int ret;
	cci_event_t *event;

	if (blocking) {
		FD_ZERO(&rfds);
		FD_SET(fd, &rfds);

		ret = select(nfds, &rfds, NULL, NULL, NULL);
		if (!ret)
			return;
	}

	ret = cci_get_event(endpoint, &event);
	if (ret == CCI_SUCCESS) {
		assert(event);
		switch (event->type) {
		case CCI_EVENT_SEND:
			assert(event->send.status == CCI_SUCCESS);
			if (opts.method != MSGS) {
				if (!is_server
				    && event->send.context == (void *)1) {
					count++;
					if (count < warmup + iters) {
						ret =
						    cci_rma(connection,
							    rmt_comp_msg,
							    rmt_comp_len,
							    local_rma_handle, 0,
							    &opts.rma_handle,
							    0, current_size,
							    (void *)1,
							    opts.flags);
						check_return(endpoint, "cci_rma", ret, 1);
					}
				}
			}
			if (!is_server
			    && event->send.context == (void *)0xdeadbeef)
				done = 1;
			break;
		case CCI_EVENT_RECV:
			{
				if (!is_server && opts.method != MSGS
				    && event->recv.ptr == (void *)1) {
					count++;
					if (count < warmup + iters) {
						ret =
						    cci_rma(connection,
							    rmt_comp_msg,
							    rmt_comp_len,
							    local_rma_handle, 0,
							    &opts.rma_handle,
							    0, current_size,
							    (void *)1,
							    opts.flags);
						check_return(endpoint, "cci_rma", ret, 1);
					}
				}
				if (!ready) {
					ready = 1;
					if (opts.method != MSGS && !is_server) {
						/* get server_rma_handle */
						opts =
						    *((options_t *) event->recv.
						      ptr);
					}
				} else if (is_server && event->recv.len == 3) {
					done = 1;
					break;
				} else if (opts.method == MSGS) {
					if (is_server) {
						count++;
						if (event->recv.len >
						    current_size) {
							current_size =
							    event->recv.len;
							count = 1;
						}
					} else {
						if (event->recv.len ==
						    current_size)
							count++;
					}
					if (is_server || count < warmup + iters) {
						ret =
						    cci_send(connection, buffer,
							     current_size, NULL,
							     opts.flags);
						if (ret)
							fprintf(stderr,
								"%s: %s: send returned %s\n",
								__func__,
								is_server ?
								"server" :
								"client",
								cci_strerror
								(endpoint, ret));
						check_return(endpoint, "cci_send", ret,
							     1);
					}
				}
				break;
			}
		case CCI_EVENT_CONNECT:
			if (!is_server) {
				connect_done = 1;
				connection = event->connect.connection;
			}
			break;
		default:
			fprintf(stderr, "ignoring event type %d\n",
				event->type);
		}
		cci_return_event(event);
	}
	return;
}
Example #5
0
static void poll_events(void)
{
	int ret;
	cci_event_t *event;

	if (blocking) {
		FD_ZERO(&rfds);
		FD_SET(fd, &rfds);

		ret = select(nfds, &rfds, NULL, NULL, NULL);
		if (!ret)
			return;
	}

	ret = cci_get_event(endpoint, &event);
	if (ret == CCI_SUCCESS) {
		assert(event);
		switch (event->type) {
		case CCI_EVENT_SEND:
			if (event->send.status != CCI_SUCCESS) {
				fprintf(stderr, "RMA failed with %s.\n",
					cci_strerror(endpoint, event->send.status));
				cci_disconnect(test);
				test = NULL;
				done = 1;
			}
			if (is_server)
				break;
			/* Client */
			if (event->send.context == (void *)0xdeadbeef) {
				done = 1;
				break;
			}
			break;
		case CCI_EVENT_RECV:
			if (is_client) {
				hdr_t *h = (void*)event->recv.ptr;

				if (!ready) {
					ready = 1;
					memcpy((void*)&remote_rma_handle,
						&h->reply.handle,
						sizeof(remote_rma_handle));
				} else {
					/* RMA status msg */
					if (opts.method == RMA_WRITE) {
						if (h->status.crc != msg.check.crc) {
							fprintf(stderr, "Server reported "
								"CRC failed.\n"
								"Local CRC 0x%x != "
								"remote CRC 0x%x.\n"
								"count=%d current_size=%u\n",
								msg.check.crc, h->status.crc,
								count, current_size);
						}
					} else {
						uint32_t crc = 0;
						void *ptr = (void*)((uintptr_t)buffer
							+ local_offset);

						/* Compute the CRC only on a valid buffer */
						if (current_size + local_offset <= opts.reg_len)
							crc = crc32(0, ptr, current_size);
						else
							crc = 0;
						if (crc != h->status.crc) {
							fprintf(stderr, "Server reported "
								"CRC failed.\n"
								"Local CRC 0x%x != "
								"remote CRC 0x%x.\n"
								"count=%d current_size=%u\n",
								crc, h->status.crc,
								count, current_size);
						}
					}
					/* RMA completed */
					count++;
					if (count < iters) {
						ret = cci_rma(test,
								&msg,
								msg_len,
								local_rma_handle,
								local_offset,
								&remote_rma_handle,
								remote_offset,
								current_size,
								NULL,
								opts.flags);
						check_return(endpoint, "cci_rma", ret, 1);
					}
				}
			} else {
				hdr_t *h = (void*)event->recv.ptr;

				/* is_server */
				if (event->recv.len == 3) {
					done = 1;
				} else {
					uint32_t crc = 0;
					void *ptr = (void*)((uintptr_t)buffer
							+ h->check.offset);

					/* RMA check request */
					if ((h->check.len + h->check.offset) <= opts.reg_len)
						crc = crc32(0, ptr, h->check.len);
					else
						crc = 0;
					msg.status.type = MSG_RMA_STATUS;
					msg.status.crc = crc;
					if (opts.method == RMA_WRITE) {
						fprintf(stderr, "server: client crc=0x%x "
							"server crc=0x%x\n", h->check.crc,
							crc);
					}
					print_buffer(ptr, h->check.len);
					ret = cci_send(test, &msg, sizeof(msg.status),
							NULL, CCI_FLAG_SILENT);
					check_return(endpoint, "cci_send", ret, 1);
				}
			}
			break;
		case CCI_EVENT_CONNECT:
			if (event->connect.status != CCI_SUCCESS)
			{
				fprintf(stderr, "Connection rejected.\n");
				exit(0);
			}
			if ((uintptr_t)event->connect.context == (uintptr_t)CONTROL) {
				control = event->connect.connection;
			} else {
				test = event->connect.connection;
			}
			if (control && test)
				connect_done = 1;
			break;
		case CCI_EVENT_CONNECT_REQUEST:
			fprintf(stderr, "Peer is reconnecting? Rejecting.\n");
			cci_reject(event);
			break;
		default:
			fprintf(stderr, "ignoring event type %s\n",
				cci_event_type_str(event->type));
		}
		cci_return_event(event);
	}
	return;
}