int main(
int argc,
char **argv)
{
struct options *user_opts = NULL;
struct server_request *my_req = NULL;
struct server_ack *my_ack = NULL;
int ret = -1;
PVFS_BMI_addr_t client_addr;
void *recv_buffer1 = NULL;
void *recv_buffer2 = NULL;
bmi_op_id_t server_ops[2];
bmi_error_code_t error_code;
int outcount = 0;
struct BMI_unexpected_info request_info;
bmi_size_t actual_size;
bmi_size_t size_list[2];
void *buffer_list[2];
int last = 0;
int i = 0;
bmi_context_id context;
char method[24], *cp;
int len;
/* grab any command line options */
user_opts = parse_args(argc, argv);
if (!user_opts)
{
return (-1);
}
/* set debugging stuff */
gossip_enable_stderr();
gossip_set_debug_mask(0, GOSSIP_BMI_DEBUG_ALL);
/* convert address to bmi method type by prefixing bmi_ */
cp = strchr(user_opts->hostid, ':');
if (!cp)
return 1;
len = cp - user_opts->hostid;
strcpy(method, "bmi_");
strncpy(method + 4, user_opts->hostid, len);
method[4+len] = '\0';
/* initialize local interface (default options) */
ret = BMI_initialize(method, user_opts->hostid, BMI_INIT_SERVER);
if (ret < 0)
{
errno = -ret;
perror("BMI_initialize");
return (-1);
}
ret = BMI_open_context(&context);
if (ret < 0)
{
errno = -ret;
perror("BMI_open_context()");
return (-1);
}
/* wait for an initial request */
do
{
ret = BMI_testunexpected(1, &outcount, &request_info, 10);
} while (ret == 0 && outcount == 0);
if (ret < 0)
{
fprintf(stderr, "Request recv failure (bad state).\n");
errno = -ret;
perror("BMI_testunexpected");
return (-1);
}
if (request_info.error_code != 0)
{
fprintf(stderr, "Request recv failure (bad state).\n");
return (-1);
}
printf("Received a new request.\n");
if (request_info.size != sizeof(struct server_request))
{
fprintf(stderr, "Bad Request!\n");
exit(-1);
}
my_req = (struct server_request *) request_info.buffer;
client_addr = request_info.addr;
/* create an ack */
my_ack = (struct server_ack *) BMI_memalloc(client_addr,
sizeof(struct server_ack),
BMI_SEND);
if (!my_ack)
{
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
memset(my_ack, 0, sizeof(struct server_ack));
/* create 2 buffers to recv into */
recv_buffer1 = BMI_memalloc(client_addr, (my_req->size / 2), BMI_RECV);
recv_buffer2 = BMI_memalloc(client_addr,
(my_req->size - (my_req->size / 2)), BMI_RECV);
if (!recv_buffer1 || !recv_buffer2)
{
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
buffer_list[0] = recv_buffer1;
buffer_list[1] = recv_buffer2;
size_list[0] = my_req->size / 2;
size_list[1] = my_req->size - (my_req->size / 2);
/* post the ack */
ret = BMI_post_send(&(server_ops[1]), client_addr, my_ack,
sizeof(struct server_ack), BMI_PRE_ALLOC, 0, NULL,
context, NULL);
if (ret < 0)
{
fprintf(stderr, "BMI_post_send_failure.\n");
return (-1);
}
if (ret == 0)
{
/* turning this into a blocking call for testing :) */
/* check for completion of ack send */
do
{
ret = BMI_test(server_ops[1], &outcount, &error_code,
&actual_size, NULL, 10, context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0)
{
fprintf(stderr, "ack send failed.\n");
return (-1);
}
}
/* post the recv */
ret = BMI_post_recv_list(&(server_ops[0]), client_addr, buffer_list,
size_list, 2, my_req->size, &actual_size,
BMI_PRE_ALLOC, 0, NULL, context, NULL);
if (ret < 0)
{
fprintf(stderr, "BMI_post_recv_failure.\n");
return (-1);
}
if (ret == 0)
{
/* turning this into a blocking call for testing :) */
/* check for completion of data payload recv */
do
{
ret = BMI_test(server_ops[0], &outcount, &error_code,
&actual_size, NULL, 10, context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0)
{
fprintf(stderr, "data recv failed.\n");
return (-1);
}
}
if (actual_size != my_req->size)
{
printf("Short recv.\n");
printf("diff: %d\n", (int) (my_req->size - actual_size));
}
/* check validity of received message */
for (i = 0; i < ((my_req->size / 2) / sizeof(int)); i++)
{
if (((int *) recv_buffer1)[i] != i)
{
fprintf(stderr, "Validation failure, offset %d.\n", i);
}
}
last = i;
for (i = last;
i < (last + ((my_req->size - (my_req->size / 2)) / sizeof(int))); i++)
{
if (((int *) recv_buffer2)[i - last] != i)
{
fprintf(stderr, "Validation failure, offset %d.\n", i);
}
}
/* free up the message buffers */
BMI_memfree(client_addr, recv_buffer1, (my_req->size / 2), BMI_RECV);
BMI_memfree(client_addr, recv_buffer2, (my_req->size -
(my_req->size / 2)), BMI_RECV);
BMI_memfree(client_addr, my_ack, sizeof(struct server_ack), BMI_SEND);
BMI_unexpected_free(client_addr, my_req);
/* shutdown the local interface */
BMI_close_context(context);
ret = BMI_finalize();
if (ret < 0)
{
errno = -ret;
perror("BMI_finalize");
return (-1);
}
/* turn off debugging stuff */
gossip_disable();
free(user_opts->hostid);
free(user_opts);
return (0);
}
int main(
int argc,
char **argv)
{
int ret = -1;
int outcount = 0, count;
struct BMI_unexpected_info request_info;
flow_descriptor *flow_d = NULL;
double time1, time2;
int i;
PINT_Request *req;
char path_name[PATH_SIZE];
TROVE_op_id op_id;
TROVE_coll_id coll_id;
TROVE_handle file_handle, parent_handle;
TROVE_ds_state state;
char *file_name;
TROVE_keyval_s key, val;
bmi_context_id context;
TROVE_context_id trove_context;
PVFS_handle_extent cur_extent;
PVFS_handle_extent_array extent_array;
/*************************************************************/
/* initialization stuff */
/* set debugging level */
gossip_enable_stderr();
gossip_set_debug_mask(0, GOSSIP_FLOW_PROTO_DEBUG | GOSSIP_BMI_DEBUG_TCP );
/* start up BMI */
ret = BMI_initialize("bmi_tcp", "tcp://NULL:3335", BMI_INIT_SERVER);
if (ret < 0)
{
fprintf(stderr, "BMI init failure.\n");
return (-1);
}
ret = BMI_open_context(&context);
if (ret < 0)
{
fprintf(stderr, "BMI_open_context() failure.\n");
return (-1);
}
ret = trove_initialize(
TROVE_METHOD_DBPF, NULL, storage_space, 0);
if (ret < 0)
{
fprintf(stderr, "initialize failed: run trove-mkfs first.\n");
return -1;
}
/* initialize the flow interface. protocol specific */
ret = PINT_flow_initialize("flowproto_bmi_cache", 0);
if (ret < 0)
{
fprintf(stderr, "flow init failure.\n");
return (-1);
}
/* try to look up collection used to store file system */
ret = trove_collection_lookup(
TROVE_METHOD_DBPF, file_system, &coll_id, NULL, &op_id);
if (ret < 0)
{
fprintf(stderr, "collection lookup failed.\n");
return -1;
}
ret = trove_open_context(coll_id, &trove_context);
if (ret < 0)
{
fprintf(stderr, "TROVE_open_context() failure.\n");
return (-1);
}
/* find the parent directory name */
strcpy(path_name, path_to_file);
for (i = strlen(path_name); i >= 0; i--)
{
if (path_name[i] != '/')
path_name[i] = '\0';
else
break;
}
file_name = path_to_file + strlen(path_name);
printf("path is %s\n", path_name);
printf("file is %s\n", file_name);
/* find the parent directory handle */
ret = path_lookup(coll_id, trove_context, path_name, &parent_handle);
if (ret < 0)
{
return -1;
}
file_handle = 0;
cur_extent.first = cur_extent.last = requested_file_handle;
extent_array.extent_count = 1;
extent_array.extent_array = &cur_extent;
ret = trove_dspace_create(coll_id,
&extent_array,
&file_handle,
TROVE_TEST_FILE,
NULL,
TROVE_FORCE_REQUESTED_HANDLE,
NULL, trove_context, &op_id, NULL);
while (ret == 0)
ret =
trove_dspace_test(coll_id, op_id, trove_context, &count, NULL, NULL,
&state, TROVE_DEFAULT_TEST_TIMEOUT);
if (ret < 0)
{
fprintf(stderr, "dspace create failed.\n");
return -1;
}
/* TODO: set attributes of file? */
/* add new file name/handle pair to parent directory */
key.buffer = file_name;
key.buffer_sz = strlen(file_name) + 1;
val.buffer = &file_handle;
val.buffer_sz = sizeof(file_handle);
ret =
trove_keyval_write(coll_id, parent_handle, &key, &val, 0, NULL, NULL,
trove_context, &op_id, NULL);
while (ret == 0)
ret =
trove_dspace_test(coll_id, op_id, trove_context, &count, NULL, NULL,
&state, TROVE_DEFAULT_TEST_TIMEOUT);
if (ret < 0)
{
fprintf(stderr, "keyval write failed.\n");
return -1;
}
/* wait for an initial communication via BMI */
/* we don't give a crap about that message except that it tells us
* where to find the client
*/
do
{
ret = BMI_testunexpected(1, &outcount, &request_info, 10);
} while (ret == 0 && outcount == 0);
if (ret < 0 || request_info.error_code != 0)
{
fprintf(stderr, "waitunexpected failure.\n");
return (-1);
}
BMI_unexpected_free(request_info.addr, request_info.buffer);
/******************************************************/
/* setup request/dist stuff */
/* request description */
/* just want one contiguous region */
ret = PVFS_Request_contiguous(TEST_SIZE, PVFS_BYTE, &req);
if (ret < 0)
{
fprintf(stderr, "PVFS_Request_contiguous() failure.\n");
return (-1);
}
/******************************************************/
/* setup communicaton stuff */
/* create a flow descriptor */
flow_d = PINT_flow_alloc();
if (!flow_d)
{
fprintf(stderr, "flow_alloc failed.\n");
return (-1);
}
/* file data */
flow_d->file_data.fsize = TEST_SIZE;
flow_d->file_data.server_nr = 0;
flow_d->file_data.server_ct = 1;
flow_d->file_data.extend_flag = 1;
flow_d->file_data.dist = PINT_dist_create("basic_dist");
if (!flow_d->file_data.dist)
{
fprintf(stderr, "Error: failed to create dist.\n");
return (-1);
}
ret = PINT_dist_lookup(flow_d->file_data.dist);
if (ret != 0)
{
fprintf(stderr, "Error: failed to lookup dist.\n");
return (-1);
}
flow_d->file_req = req;
flow_d->tag = 0;
flow_d->user_ptr = NULL;
flow_d->aggregate_size = TEST_SIZE;
/* fill in flow details */
flow_d->src.endpoint_id = BMI_ENDPOINT;
flow_d->src.u.bmi.address = request_info.addr;
flow_d->dest.endpoint_id = TROVE_ENDPOINT;
flow_d->dest.u.trove.handle = file_handle;
flow_d->dest.u.trove.coll_id = coll_id;
/***************************************************
* test bmi to file (analogous to a client side write)
*/
time1 = Wtime();
ret = block_on_flow(flow_d);
if (ret < 0)
{
return (-1);
}
time2 = Wtime();
#if 0
printf("Server bw (recv): %f MB/sec\n",
((TEST_SIZE) / ((time2 - time1) * 1000000.0)));
#endif
/*******************************************************/
/* final cleanup and output */
PINT_flow_free(flow_d);
/* shut down flow interface */
ret = PINT_flow_finalize();
if (ret < 0)
{
fprintf(stderr, "flow finalize failure.\n");
return (-1);
}
/* shut down BMI */
BMI_close_context(context);
BMI_finalize();
trove_close_context(coll_id, trove_context);
trove_finalize(TROVE_METHOD_DBPF);
gossip_disable();
return (0);
}
static int do_server(struct options *opts, bmi_context_id *context)
{
int ret = 0;
int i = 0;
PVFS_BMI_addr_t peer_addr;
PVFS_BMI_addr_t server_addr;
void *recv_buffer = NULL;
void *send_buffer = NULL;
bmi_op_id_t op_id[2];
bmi_error_code_t error_code;
int outcount = 0;
struct BMI_unexpected_info request_info;
bmi_size_t actual_size;
struct msg *tx_msg = NULL;
struct msg *rx_msg = NULL;
int bytes = MIN_BYTES;
int max_bytes = MAX_BYTES;
int warmup = 1;
int iterations = 0;
int msg_len = 0;
int run = 0;
/* wait for an initial request to get size */
do {
ret = BMI_testunexpected(1, &outcount, &request_info, 10);
} while (ret == 0 && outcount == 0);
if (ret < 0) {
fprintf(stderr, "Request recv failure (bad state).\n");
errno = -ret;
perror("BMI_testunexpected");
return ret;
}
if (request_info.error_code != 0) {
fprintf(stderr, "Request recv failure (bad state).\n");
return ret;
}
if (request_info.size != sizeof(struct msg)) {
fprintf(stderr, "Bad Request! Received %d bytes\n",
(int) request_info.size);
return ret;
}
rx_msg = (struct msg *) request_info.buffer;
opts->test = ntohl(rx_msg->test);
printf("Starting %s test%s\n", opts->test == EXPECTED ? "expected" : "unexpected",
opts->crc ? " with checksums" : "");
peer_addr = request_info.addr;
BMI_unexpected_free(peer_addr, request_info.buffer);
#ifdef WIN32
server_addr = (PVFS_BMI_addr_t) 0;
#endif
ret = BMI_get_info(server_addr, BMI_CHECK_MAXSIZE,
(void *)&max_bytes);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
if (max_bytes > MAX_BYTES) max_bytes = MAX_BYTES;
if (opts->test == UNEXPECTED) {
ret = BMI_addr_lookup(&server_addr, opts->hostid);
if (ret < 0) {
errno = -ret;
perror("BMI_addr_lookup");
return (-1);
}
ret = BMI_get_info(server_addr, BMI_GET_UNEXP_SIZE,
(void *)&max_bytes);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
} else {
int maxsize = 0;
ret = BMI_get_info(server_addr, BMI_CHECK_MAXSIZE,
(void *)&maxsize);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
if (maxsize < max_bytes) max_bytes = maxsize;
}
msg_len = sizeof(struct msg);
/* create an ack */
send_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_SEND);
if (!send_buffer) {
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
memset(send_buffer, 0, max_bytes);
tx_msg = (struct msg *) send_buffer;
tx_msg->test = htonl(opts->test);
/* create a buffer to recv into */
recv_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_RECV);
if (!recv_buffer) {
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
/* post the ack */
ret = BMI_post_send(&(op_id[SEND]), peer_addr, tx_msg,
msg_len, BMI_PRE_ALLOC, 0, NULL,
*context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_send failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[SEND], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "ack send failed.\n");
return (-1);
}
if (actual_size != (bmi_size_t) msg_len) {
fprintf(stderr, "Expected %d but received %llu\n",
msg_len, llu(actual_size));
}
}
/* start iterations */
while (bytes <= max_bytes) {
iterations = bytes_to_iterations(bytes);
for (i=0; i < iterations; i++) {
/* receive the ping */
if (opts->test == EXPECTED) {
ret = BMI_post_recv(&(op_id[RECV]), peer_addr, recv_buffer,
bytes, &actual_size, BMI_PRE_ALLOC, i, NULL,
*context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_recv_failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[RECV], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "data recv failed.\n");
return (-1);
}
if (actual_size != bytes) {
fprintf(stderr, "Expected %d but received %llu\n",
bytes, llu(actual_size));
return (-1);
}
}
} else { /* UNEXPECTED */
do {
ret = BMI_testunexpected(1, &outcount, &request_info, 10);
} while (ret == 0 && outcount == 0);
if (ret < 0) {
fprintf(stderr, "Request recv failure (bad state).\n");
errno = -ret;
perror("BMI_testunexpected");
return ret;
}
if (request_info.error_code != 0) {
fprintf(stderr, "Request recv failure (bad state).\n");
return ret;
}
if (request_info.size != bytes) {
fprintf(stderr, "Bad Request! Received %d bytes\n",
(int) request_info.size);
return ret;
}
}
/* send the pong */
ret = BMI_post_send(&(op_id[SEND]), peer_addr,
(opts->test == EXPECTED ?
recv_buffer : request_info.buffer),
bytes, BMI_PRE_ALLOC, i, NULL, *context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_send failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[SEND], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "ack send failed.\n");
return (-1);
}
if (actual_size != bytes) {
fprintf(stderr, "Expected %d but received %llu\n",
bytes, llu(actual_size));
return (-1);
}
}
}
if (!warmup) {
bytes *= 2;
run++;
}
else warmup = 0;
}
/* free up the message buffers */
BMI_memfree(peer_addr, recv_buffer, max_bytes, BMI_RECV);
BMI_memfree(peer_addr, send_buffer, max_bytes, BMI_SEND);
return ret;
}
