Exemple #1
0
Fichier : rpc.c Projet : CCI/cci
static void do_server(void)
{
	int ret = 0, i = 0;

	accept_connection();

	if (connection->max_send_size < opts.ack_size)
		opts.ack_size = connection->max_send_size;

	while (!done)
		progress_server();

	for (i = 0; i < 1000; i++)
		progress_server();

	ret = cci_rma_deregister(endpoint, local);
	check_return(endpoint, "cci_rma_deregister", ret, 1);

	if (!suppress)
		printf("server done\n");

	return;
}
Exemple #2
0
Fichier : rpc.c Projet : CCI/cci
static void do_client(void)
{
	int ret, i = 0;
	struct timeval start, end;
	double lat = 0.0;
	double bw = 0.0;

	ret = posix_memalign((void **)&request, 4096, opts.req_size);
	check_return(endpoint, "memalign buffer", ret, 1);

	msg = (int*) request;

	ret = posix_memalign((void **)&buffer, 4096, opts.transfer_size);
	check_return(endpoint, "memalign buffer", ret, 1);

	memset(buffer, 'b', opts.transfer_size);

	ret = cci_rma_register(endpoint, buffer, opts.transfer_size,
				opts.rma_flags, &local);
	check_return(endpoint, "cci_rma_register", ret, 1);

	memcpy(&opts.rma_handle, local, sizeof(*local));

	connect_to_server();

	if (connection->max_send_size < opts.req_size)
		opts.req_size = connection->max_send_size;

	if (!suppress)
		printf("Bytes\t\tLatency (per rpc)\tThroughput (per rpc)\n");

	/* begin communication with server */
	ret = send_request(count);
	check_return(endpoint, "send first request", ret, 1);
	if (!ret) {
		count++;
		inflight++;
	}

	progress_client(opts.warmup);

	count = 0;
	completed = 0;

	gettimeofday(&start, NULL);

	for (i = 0; i < concurrent; i++) {
		ret = send_request(count);
		if (!ret) {
			count++;
			inflight++;
		}
		check_return(endpoint, "send first request", ret, 0);
	}

	progress_client(opts.iters);

	gettimeofday(&end, NULL);

	lat = usecs(start, end) / (double)opts.iters;

	bw = (double)opts.transfer_size / lat;
	printf("%8d\t%8.2f us\t\t%8.2f MB/s\n", opts.transfer_size, lat, bw);

	ret = cci_send(connection, "bye", 3, (void *)0xdeadbeef, 0);
	check_return(endpoint, "cci_send", ret, 1);

	progress_client(0);

	ret = cci_rma_deregister(endpoint, local);
	check_return(endpoint, "cci_rma_deregister", ret, 1);

	if (!suppress)
		printf("client done\n");

	return;
}
Exemple #3
0
int main(int argc, char *argv[])
{
	int c, ret;
	int dereg = 0, prefault = 0;
	uint32_t pagesize = 0, offset = 0;
	uint64_t regsize = REGSIZE, totalsize = TOTALSIZE, count, i;
	uint32_t caps;
	cci_device_t * const *devices;
	cci_endpoint_t *endpoint;
	void *base, *ptr;
	uint64_t length;
	cci_rma_handle_t **handles = NULL;
	struct timeval start, end;
	uint64_t usecs = 0;

	pagesize = sysconf(_SC_PAGESIZE);

	while ((c = getopt(argc, argv, "dfo:s:t:")) != -1) {
		switch (c) {
		case 'd':
			dereg = 1;
			break;
		case 'f':
			prefault = 1;
			break;
		case 'o':
			offset = strtoul(optarg, NULL, 0);
			if (offset >= pagesize) {
				fprintf(stderr,
					"offset larger than pagesize (%u)\n",
					pagesize);
				usage(argv[0]);
			}
			break;
		case 's':
			regsize = strtoull(optarg, NULL, 0);
			if (regsize < pagesize) {
				printf("regsize (%" PRIu64
				       ") < pagesize (%u) - increasing to pagesize\n",
				       regsize, pagesize);
				regsize = pagesize;
			}
			break;
		case 't':
			totalsize = strtoull(optarg, NULL, 0);
			break;
		default:
			usage(argv[0]);
			break;
		}
	}

	count = totalsize / regsize;

	ret = posix_memalign(&base, pagesize, totalsize + offset);
	check_return(NULL, "posix_memalign", ret);

	ptr = (void*)((uintptr_t)base + (uintptr_t) offset);
	length = regsize;

	handles = calloc(count, sizeof(*handles));
	check_return(NULL, "calloc", handles ? 0 : CCI_ENOMEM);

	if (prefault) {
		for (i = 0; i < totalsize; i += pagesize) {
			char *c = (char *)ptr + (uintptr_t) i;
			*c = '1';
		}
	}

	ret = cci_init(CCI_ABI_VERSION, 0, &caps);
	check_return(NULL, "cci_init", ret);

	ret = cci_get_devices(&devices);
	check_return(NULL, "cci_get_devices", ret);

	ret = cci_create_endpoint(NULL, 0, &endpoint, NULL);
	check_return(NULL, "cci_create_endpoint", ret);

	/* register */
	if (!dereg)
		gettimeofday(&start, NULL);

	for (i = 0; i < count; i++) {
		void *p = (void*)((uintptr_t)ptr + ((uintptr_t) i * (uintptr_t)length));

		ret = cci_rma_register(endpoint, p, length, CCI_FLAG_READ|CCI_FLAG_WRITE, &handles[i]);
		check_return(endpoint, "cci_rma_register", ret);
	}

	if (!dereg)
		gettimeofday(&end, NULL);

	/* deregister */
	if (dereg)
		gettimeofday(&start, NULL);

	for (i = 0; i < count; i++) {
		ret = cci_rma_deregister(endpoint, handles[i]);
		check_return(endpoint, "cci_rma_register", ret);
	}

	if (dereg)
		gettimeofday(&end, NULL);

	usecs = (end.tv_sec - start.tv_sec) * 1000000 +
	    end.tv_usec - start.tv_usec;
	printf("%10s%10s%10s%10s\n", "RegSize", "Count", "usecs", "us/page");
	printf("%10" PRIu64 "%10" PRIu64 "%10" PRIu64 "%10.2f\n",
	       regsize, count, usecs,
	       ((double)usecs / (double) count) / ((double)regsize / (double)pagesize));

	ret = cci_destroy_endpoint(endpoint);
	check_return(endpoint, "cci_destroy_endpoint", ret);

        ret = cci_finalize();
	check_return(NULL, "cci_finalize", ret);

	return 0;
}
Exemple #4
0
static void do_server(void)
{
	int ret;

	while (!ready) {
		cci_event_t *event;

		if (blocking) {
			FD_ZERO(&rfds);
			FD_SET(fd, &rfds);

			ret = select(nfds, &rfds, NULL, NULL, NULL);
			if (!ret)
				return;
		}

		ret = cci_get_event(endpoint, &event);
		if (ret == CCI_SUCCESS) {
			switch (event->type) {
			case CCI_EVENT_CONNECT_REQUEST:
				if (accept) {
					opts =
					    *((options_t *) event->request.
					      data_ptr);
					ret = cci_accept(event, NULL);
					check_return(endpoint, "cci_accept", ret, 1);
				} else {
					ret = cci_reject(event);
					check_return(endpoint, "cci_reject", ret, 1);
				}
				break;
			case CCI_EVENT_ACCEPT:
				{
					int len;

					ready = 1;
					connection = event->accept.connection;

					if (opts.method == MSGS)
						len = connection->max_send_size;
					else
						len = opts.max_rma_size;

					ret =
					    posix_memalign((void **)&buffer,
							   4096, len);
					check_return(endpoint, "memalign buffer", ret, 1);

					memset(buffer, 'a', len);

					if (opts.method != MSGS) {
						ret =
						    cci_rma_register(endpoint,
								     buffer,
								     opts.
								     max_rma_size,
								     opts.method == RMA_WRITE ? CCI_FLAG_WRITE : CCI_FLAG_READ,
								     &server_rma_handle);
						check_return(endpoint, "cci_rma_register",
							     ret, 1);
						memcpy(&opts.rma_handle,
								server_rma_handle,
								sizeof(*server_rma_handle));
					}
					ret =
					    cci_send(connection, &opts,
						     sizeof(opts), NULL, 0);
					check_return(endpoint, "cci_send", ret, 1);
					break;
				}
			default:
				fprintf(stderr,
					"%s: ignoring unexpected event %d\n",
					__func__, event->type);
				break;
			}
			ret = cci_return_event(event);
			if (ret)
				fprintf(stderr, "cci_return_event() failed with %s\n",
						cci_strerror(endpoint, ret));
		}
	}

	while (!done)
		poll_events();

	if (opts.method != MSGS) {
		ret = cci_rma_deregister(endpoint, server_rma_handle);
		check_return(endpoint, "cci_rma_deregister", ret, 1);
	}

	printf("server done\n");
	sleep(1);

	return;
}
Exemple #5
0
static void do_client(void)
{
	int ret;
	uint32_t min = 0, max;
	struct timeval start, end;
	char *func;
	char *header = "Done";

	/* initiate connect */
	ret =
	    cci_connect(endpoint, server_uri, &opts, sizeof(opts), attr, NULL,
			0, NULL);
	check_return(endpoint, "cci_connect", ret, 1);

	/* poll for connect completion */
	while (!connect_done)
		poll_events();

	if (!connection) {
		fprintf(stderr, "no connection\n");
		return;
	}

	while (!ready)
		poll_events();

	if (opts.method == MSGS) {
		func = "cci_send";
		max = connection->max_send_size;
	} else {
		func = "cci_rma";
		max = opts.max_rma_size;
	}

	ret = posix_memalign((void **)&buffer, 4096, max);
	check_return(endpoint, "memalign buffer", ret, 1);

	memset(buffer, 'b', max);

	if (opts.method != MSGS) {
		int flags = 0;

		/* for the client, we want the opposite of the opts.method.
		 * when testing RMA WRITE, we only need READ access.
		 * when testing RMA READ, we need WRITE access.
		 */

		if (opts.method == RMA_WRITE)
			flags = CCI_FLAG_READ;
		else if (opts.method == RMA_READ)
			flags = CCI_FLAG_WRITE;

		ret = cci_rma_register(endpoint, buffer, max, flags,
				       &local_rma_handle);
		check_return(endpoint, "cci_rma_register", ret, 1);
		fprintf(stderr, "local_rma_handle is %p\n",
			(void*)local_rma_handle);
		min = 1;
		if (opts.method == RMA_WRITE)
			opts.flags |= CCI_FLAG_WRITE;
		else
			opts.flags |= CCI_FLAG_READ;
	}

	if (remote_completion) {
		rmt_comp_msg = header;
		rmt_comp_len = 4;
	}

	if (opts.method == MSGS)
		printf("Bytes\t\tLatency (one-way)\tThroughput\n");
	else
		printf("Bytes\t\tLatency (round-trip)\tThroughput\n");

	/* begin communication with server */
	for (current_size = min; current_size <= max;) {
		double lat = 0.0;
		double bw = 0.0;

		if (opts.method == MSGS)
			ret =
			    cci_send(connection, buffer, current_size, NULL,
				     opts.flags);
		else
			ret = cci_rma(connection, rmt_comp_msg, rmt_comp_len,
				      local_rma_handle, 0,
				      &opts.rma_handle, 0,
				      current_size, (void *)1, opts.flags);
		check_return(endpoint, func, ret, 1);

		while (count < warmup)
			poll_events();

		gettimeofday(&start, NULL);

		while (count < warmup + iters)
			poll_events();

		gettimeofday(&end, NULL);

		if (opts.method == MSGS)
			lat = usecs(start, end) / (double)iters / 2.0;
		else
			lat = usecs(start, end) / (double)iters;

		bw = (double)current_size / lat;
		printf("%8d\t%8.2f us\t\t%8.2f MB/s\n", current_size, lat,
		       bw);

		count = 0;

		if (current_size == 0)
			current_size++;
		else
			current_size *= 2;

		if (current_size >= 64 * 1024) {
			if (iters >= 32)
				iters /= 2;
			if (warmup >= 4)
				warmup /= 2;
		}
	}

	ret = cci_send(connection, "bye", 3, (void *)0xdeadbeef, opts.flags);
	check_return(endpoint, "cci_send", ret, 0);

	while (!done)
		poll_events();

	if (opts.method != MSGS) {
		ret = cci_rma_deregister(endpoint, local_rma_handle);
		check_return(endpoint, "cci_rma_deregister", ret, 1);
	}

	printf("client done\n");
	sleep(1);

	return;
}
Exemple #6
0
static void do_server(void)
{
	int ret = 0;
	hdr_t *h = NULL;

	while (!ready) {
		cci_event_t *event;

		if (blocking) {
			FD_ZERO(&rfds);
			FD_SET(fd, &rfds);

			ret = select(nfds, &rfds, NULL, NULL, NULL);
			if (!ret)
				return;
		}

		ret = cci_get_event(endpoint, &event);
		if (ret == CCI_SUCCESS) {
			switch (event->type) {
			case CCI_EVENT_CONNECT_REQUEST:
			{
				int which = 0;

				h = (void*)event->request.data_ptr;
				if (h->generic.type == MSG_CONN_REQ) {
					opts = h->request.opts;
					which = TEST;
				} else {
					which = CONTROL;
				}
				ret = cci_accept(event, (void*)((uintptr_t)which));
				check_return(endpoint, "cci_accept", ret, 1);
				break;
			}
			case CCI_EVENT_ACCEPT:
			{
				if ((uintptr_t)event->accept.context == (uintptr_t)CONTROL) {
					control = event->accept.connection;
				} else {
					int len;

					test = event->accept.connection;

					len = opts.reg_len;

					ret =
					    posix_memalign((void **)&buffer,
							   4096, len);
					check_return(endpoint, "memalign buffer", ret, 1);

					init_buffer(0);
					print_buffer(buffer, opts.reg_len);

					ret = cci_rma_register(endpoint,
							     buffer,
							     opts.reg_len,
							     opts.method == RMA_WRITE ? CCI_FLAG_WRITE : CCI_FLAG_READ,
							     &local_rma_handle);
					check_return(endpoint, "cci_rma_register",
							     ret, 1);
				}
				if (test && control) {
					hdr_t msg;

					ready = 1;
					msg.reply.type = MSG_CONN_REPLY;
					msg.reply.handle = *local_rma_handle;

					ret = cci_send(test, &msg,
						     sizeof(msg.reply), NULL, 0);
					check_return(endpoint, "cci_send", ret, 1);
				}
				break;
			}
			default:
				fprintf(stderr,
					"%s: ignoring unexpected event %s\n",
					__func__, cci_event_type_str(event->type));
				break;
			}
			ret = cci_return_event(event);
			if (ret)
				fprintf(stderr, "cci_return_event() failed with %s\n",
						cci_strerror(endpoint, ret));
		}
	}

	while (!done)
		poll_events();

	ret = cci_rma_deregister(endpoint, local_rma_handle);
	check_return(endpoint, "cci_rma_deregister", ret, 1);

	printf("server done\n");
	sleep(1);

	return;
}
Exemple #7
0
static void do_client(void)
{
	int ret;
	uint32_t min = 1;

	/* initiate connect */
	msg.request.type = MSG_CONTROL;

	ret =
	    cci_connect(endpoint, server_uri, &msg, sizeof(msg.generic), attr,
			(void*)(uintptr_t)CONTROL, 0, NULL);
	check_return(endpoint, "cci_connect", ret, 1);

	msg.request.type = MSG_CONN_REQ;
	msg.request.opts = opts;

	ret =
	    cci_connect(endpoint, server_uri, &msg, sizeof(msg.request), attr,
			(void*)(uintptr_t)TEST, 0, NULL);
	check_return(endpoint, "cci_connect", ret, 1);
	/* poll for connect completion */
	while (!connect_done)
		poll_events();

	if (!test) {
		fprintf(stderr, "no connection\n");
		return;
	}

	while (!ready)
		poll_events();

	ret = posix_memalign((void **)&buffer, 4096, opts.reg_len);
	check_return(endpoint, "memalign buffer", ret, 1);

	memset(buffer, 0xaa, opts.reg_len);

	init_buffer(1);
	print_buffer(buffer, (int) opts.reg_len);

	/* for the client, we do not need remote access flags */

	ret = cci_rma_register(endpoint, buffer, opts.reg_len, 0, &local_rma_handle);
	check_return(endpoint, "cci_rma_register", ret, 1);

	if (opts.method == RMA_WRITE)
		opts.flags = CCI_FLAG_WRITE;
	else
		opts.flags = CCI_FLAG_READ;

	/* begin communication with server */
	for (current_size = min; current_size <= length;) {
		void *ptr = (void*)((uintptr_t)buffer + local_offset);

		msg.check.type = MSG_RMA_CHK;
		msg.check.offset = remote_offset;
		msg.check.len = current_size;
		/* Compute the CRC only on a valid buffer */
		if (current_size + local_offset <= opts.reg_len)
			msg.check.crc = crc32(0, ptr, current_size);
		else
			msg.check.crc = 0;
		msg_len = sizeof(msg.check);
		print_buffer(ptr, current_size);

		fprintf(stderr, "Testing length %9u ... ", current_size);

		ret = cci_rma(test, &msg, msg_len,
			      local_rma_handle, local_offset,
			      &remote_rma_handle, remote_offset,
			      current_size, NULL, opts.flags);
		check_return(endpoint, "cci_rma", ret, 1);

		while (count < iters)
			poll_events();

		if (test)
			fprintf(stderr, "success.\n");
		else
			goto out;

		count = 0;
		current_size *= 2;

		if (current_size >= 64 * 1024) {
			if (iters >= 32)
				iters /= 2;
		}
	}

out:
	ret = cci_send(control, "bye", 3, (void *)0xdeadbeef, 0);
	check_return(endpoint, "cci_send", ret, 0);

	while (!done)
		poll_events();

	ret = cci_rma_deregister(endpoint, local_rma_handle);
	check_return(endpoint, "cci_rma_deregister", ret, 1);

	printf("client done\n");
	sleep(1);

	return;
}