static void *_9p_rdma_cleanup_conn_thread(void *arg) {
msk_trans_t * trans = arg;
_9p_rdma_priv * priv = _9p_rdma_priv_of(trans) ;
int i ;
if( priv )
{
if( priv->pconn )
{
LogDebug(COMPONENT_9P,
"9P/RDMA: waiting till we're done with all requests on trans [%p]", trans) ;
while( atomic_fetch_uint32_t( &priv->pconn->refcount ) != 0 ) {
sleep( 1 );
}
}
LogDebug(COMPONENT_9P,
"9P/RDMA: Freeing data associated with trans [%p]", trans) ;
if( priv->pconn )
{
_9p_cleanup_fids( priv->pconn );
}
if( priv->datalock )
{
if( priv->datalock->data && priv->datalock->data->mr ) msk_dereg_mr( priv->datalock->data->mr ) ;
gsh_free( priv->datalock ) ;
}
if( priv->rdata )
{
gsh_free( priv->rdata ) ;
}
if( priv->rdmabuf ) gsh_free( priv->rdmabuf ) ;
if( priv->pconn ) gsh_free( priv->pconn ) ;
gsh_free( priv ) ;
}
msk_destroy_trans( &trans ) ;
pthread_exit( NULL ) ;
}
int main(int argc, char **argv) {
msk_trans_t *trans;
uint8_t *mrbuf;
struct ibv_mr *mr;
msk_data_t *wdata;
msk_trans_attr_t attr;
memset(&attr, 0, sizeof(msk_trans_attr_t));
attr.server = -1; // put an incorrect value to check if we're either client or server
// sane values for optional or non-configurable elements
attr.rq_depth = RECV_NUM+2;
attr.sq_depth = RECV_NUM+2;
attr.max_recv_sge = NUM_SGE;
attr.max_send_sge = NUM_SGE;
attr.port = "1235";
// attr.disconnect_callback = callback_disconnect;
// argument handling
static struct option long_options[] = {
{ "client", required_argument, 0, 'c' },
{ "port", required_argument, 0, 'p' },
{ "server", no_argument, 0, 's' },
{ "help", no_argument, 0, 'h' },
{ 0, 0, 0, 0 }
};
int option_index = 0;
int op;
while ((op = getopt_long(argc, argv, "@hvsS:c:p:", long_options, &option_index)) != -1) { /* */
switch(op) {
case '@':
printf("%s compiled on %s at %s\n", argv[0], __DATE__, __TIME__);
printf("Release = %s\n", VERSION);
printf("Release comment = %s\n", VERSION_COMMENT);
printf("Git HEAD = %s\n", _GIT_HEAD_COMMIT ) ;
printf("Git Describe = %s\n", _GIT_DESCRIBE ) ;
exit(0);
case 'h':
print_help(argv);
exit(0);
case 'v':
attr.debug = attr.debug * 2 + 1;
break;
case 'c':
attr.server = 0;
attr.node = optarg;
break;
case 's':
attr.server = 10;
attr.node = "::";
break;
case 'S':
attr.server = 10;
attr.node = optarg;
break;
case 'p':
attr.port = optarg;
break;
default:
ERROR_LOG("Failed to parse arguments");
print_help(argv);
exit(EINVAL);
}
}
if (attr.server == -1) {
ERROR_LOG("must be either a client or a server!");
print_help(argv);
exit(EINVAL);
}
TEST_Z(msk_init(&trans, &attr));
if (!trans)
exit(-1);
if (trans->server) {
TEST_Z(msk_bind_server(trans));
TEST_NZ(trans = msk_accept_one(trans));
} else { //client
TEST_Z(msk_connect(trans));
TEST_NZ(trans);
}
TEST_NZ(mrbuf = malloc((RECV_NUM*NUM_SGE+1)*CHUNK_SIZE));
memset(mrbuf, 0, (RECV_NUM*NUM_SGE+1)*CHUNK_SIZE);
TEST_NZ(mr = msk_reg_mr(trans, mrbuf, (RECV_NUM*NUM_SGE+1)*CHUNK_SIZE, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ));
pthread_mutex_t lock;
pthread_cond_t cond;
pthread_mutex_init(&lock, NULL);
pthread_cond_init(&cond, NULL);
msk_data_t *rdata;
struct datalock datalock;
TEST_NZ(rdata = malloc(RECV_NUM*NUM_SGE*sizeof(msk_data_t)));
int i;
for (i=0; i<RECV_NUM*NUM_SGE; i++) {
rdata[i].data=mrbuf+i*CHUNK_SIZE;
rdata[i].size = 0;
rdata[i].max_size=CHUNK_SIZE;
rdata[i].mr = mr;
if ((i-1) % NUM_SGE != 0)
rdata[i].next = &rdata[i+1];
else
rdata[i].next = NULL;
}
datalock.lock = &lock;
datalock.cond = &cond;
TEST_NZ(wdata = malloc(sizeof(msk_data_t)));
wdata->data = mrbuf+RECV_NUM*NUM_SGE*CHUNK_SIZE;
wdata->mr = mr;
wdata->max_size = CHUNK_SIZE;
pthread_mutex_lock(&lock);
if (trans->server) // server receives, client sends
TEST_Z(msk_post_n_recv(trans, rdata, NUM_SGE, callback_recv, NULL, &datalock));
if (trans->server) {
TEST_Z(msk_finalize_accept(trans));
} else {
TEST_Z(msk_finalize_connect(trans));
}
if (trans->server) {
TEST_Z(pthread_cond_wait(&cond, &lock));
printf("Got something:\n %s (%d), %s (%d)\n", rdata[0].data, rdata[0].size, rdata[1].data, rdata[1].size);
} else {
memcpy(rdata[0].data, "012345678", 10);
rdata[0].size = 10;
memcpy(rdata[1].data, "0123456", 8);
rdata[1].size = 8;
TEST_Z(msk_post_n_send(trans, rdata, NUM_SGE, callback_recv, NULL, &datalock));
TEST_Z(pthread_cond_wait(&cond, &lock));
printf("Done with send\n");
}
pthread_mutex_unlock(&lock);
msk_dereg_mr(mr);
msk_destroy_trans(&trans);
free(rdata);
free(wdata);
free(mrbuf);
return 0;
}
int main(int argc, char **argv) {
msk_trans_t *trans;
uint8_t *rdmabuf;
struct ibv_mr *mr;
msk_data_t *wdata;
msk_trans_attr_t attr;
memset(&attr, 0, sizeof(msk_trans_attr_t));
attr.server = -1; // put an incorrect value to check if we're either client or server
// sane values for optional or non-configurable elements
attr.rq_depth = RECV_NUM+2;
attr.port = "1235";
// attr.disconnect_callback = callback_disconnect;
// argument handling
static struct option long_options[] = {
{ "client", required_argument, 0, 'c' },
{ "server", required_argument, 0, 's' },
{ "port", required_argument, 0, 'p' },
{ "help", no_argument, 0, 'h' },
{ 0, 0, 0, 0 }
};
int option_index = 0;
int op;
while ((op = getopt_long(argc, argv, "@hvsS:c:p:", long_options, &option_index)) != -1) {
switch(op) {
case '@':
printf("%s compiled on %s at %s\n", argv[0], __DATE__, __TIME__);
printf("Release = %s\n", VERSION);
printf("Release comment = %s\n", VERSION_COMMENT);
printf("Git HEAD = %s\n", _GIT_HEAD_COMMIT ) ;
printf("Git Describe = %s\n", _GIT_DESCRIBE ) ;
exit(0);
case 'h':
print_help(argv);
exit(0);
case 'v':
attr.debug = attr.debug * 2 + 1;
break;
case 'c':
attr.server = 0;
attr.node = optarg;
break;
case 's':
attr.server = 10;
attr.node = "::";
break;
case 'S':
attr.server = 10;
attr.node = optarg;
break;
case 'p':
attr.port = optarg;
break;
default:
ERROR_LOG("Failed to parse arguments");
print_help(argv);
exit(EINVAL);
}
}
if (attr.server == -1) {
ERROR_LOG("must be either a client or a server!");
print_help(argv);
exit(EINVAL);
}
TEST_Z(msk_init(&trans, &attr));
if (!trans)
exit(-1);
if (trans->server) {
TEST_Z(msk_bind_server(trans));
TEST_NZ(trans = msk_accept_one(trans));
} else { //client
TEST_Z(msk_connect(trans));
TEST_NZ(trans);
}
TEST_NZ(rdmabuf = malloc((RECV_NUM+2)*CHUNK_SIZE*sizeof(char)));
memset(rdmabuf, 0, (RECV_NUM+2)*CHUNK_SIZE*sizeof(char));
TEST_NZ(mr = msk_reg_mr(trans, rdmabuf, (RECV_NUM+2)*CHUNK_SIZE*sizeof(char), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ));
msk_data_t *ackdata;
TEST_NZ(ackdata = malloc(sizeof(msk_data_t)));
ackdata->data = rdmabuf+(RECV_NUM+1)*CHUNK_SIZE*sizeof(char);
ackdata->max_size = CHUNK_SIZE*sizeof(char);
ackdata->size = 1;
ackdata->data[0] = 0;
pthread_mutex_t lock;
pthread_cond_t cond;
pthread_mutex_init(&lock, NULL);
pthread_cond_init(&cond, NULL);
msk_data_t *rdata;
struct datalock datalock;
TEST_NZ(rdata = malloc(sizeof(msk_data_t)));
rdata->data=rdmabuf; //+i*CHUNK_SIZE*sizeof(char);
rdata->max_size=CHUNK_SIZE*sizeof(char);
rdata->mr = mr;
datalock.ackdata = ackdata;
datalock.lock = &lock;
datalock.cond = &cond;
pthread_mutex_lock(&lock);
TEST_Z(msk_post_recv(trans, rdata, callback_recv, NULL, &datalock));
if (trans->server) {
TEST_Z(msk_finalize_accept(trans));
} else {
TEST_Z(msk_finalize_connect(trans));
}
TEST_NZ(wdata = malloc(sizeof(msk_data_t)));
wdata->data = rdmabuf+RECV_NUM*CHUNK_SIZE*sizeof(char);
wdata->mr = mr;
wdata->max_size = CHUNK_SIZE*sizeof(char);
msk_rloc_t *rloc;
if (trans->server) {
printf("wait for rloc\n");
TEST_Z(pthread_cond_wait(&cond, &lock)); // receive rloc
TEST_NZ(rloc = malloc(sizeof(msk_rloc_t)));
memcpy(rloc, rdata->data, sizeof(msk_rloc_t));
printf("got rloc! key: %u, addr: %"PRIu64", size: %d\n",
rloc->rkey, rloc->raddr, rloc->size);
memcpy(wdata->data, "roses are red", 14);
wdata->size = 14;
TEST_Z(msk_post_write(trans, wdata, rloc, callback_recv, NULL, &datalock));
printf("waiting for write to finish\n");
TEST_Z(pthread_cond_wait(&cond, &lock)); // write done
TEST_Z(msk_post_recv(trans, rdata, callback_recv, NULL, &datalock));
TEST_Z(msk_post_send(trans, wdata, NULL, NULL, NULL)); // ack to say we're done
printf("waiting for something to be ready to read\n");
TEST_Z(pthread_cond_wait(&cond, &lock));
wdata->size=17;
TEST_Z(msk_post_read(trans, wdata, rloc, callback_recv, NULL, &datalock));
printf("wait for read to finish\n");
TEST_Z(pthread_cond_wait(&cond, &lock));
printf("%s\n", wdata->data);
TEST_Z(msk_wait_send(trans, wdata)); // ack - other can quit
} else {
TEST_NZ(rloc = msk_make_rloc(mr, (uint64_t)(uintptr_t)ackdata->data, ackdata->max_size));
memcpy(wdata->data, rloc, sizeof(msk_rloc_t));
wdata->size = sizeof(msk_rloc_t);
TEST_Z(msk_post_send(trans, wdata, NULL, NULL, NULL));
printf("sent rloc, waiting for server to say they're done\n");
TEST_Z(pthread_cond_wait(&cond, &lock)); // receive server ack (they wrote stuff)
printf("%s\n", ackdata->data);
TEST_Z(msk_post_recv(trans, rdata, callback_recv, NULL, &datalock));
memcpy(ackdata->data, "violets are blue", 17);
TEST_Z(msk_post_send(trans, wdata, NULL, NULL, NULL)); // say we've got something to read
printf("waiting for server to be done\n");
TEST_Z(pthread_cond_wait(&cond, &lock));
}
pthread_mutex_unlock(&lock);
msk_dereg_mr(mr);
msk_destroy_trans(&trans);
msk_destroy_trans(&trans); // check that double_destroy works
free(rloc);
free(ackdata);
free(rdata);
free(wdata);
free(rdmabuf);
return 0;
}