static void *_9p_rdma_cleanup_conn_thread(void *arg) {
msk_trans_t * trans = arg;
_9p_rdma_priv * priv = _9p_rdma_priv_of(trans) ;
int i ;
if( priv )
{
if( priv->pconn )
{
LogDebug(COMPONENT_9P,
"9P/RDMA: waiting till we're done with all requests on trans [%p]", trans) ;
while( atomic_fetch_uint32_t( &priv->pconn->refcount ) != 0 ) {
sleep( 1 );
}
}
LogDebug(COMPONENT_9P,
"9P/RDMA: Freeing data associated with trans [%p]", trans) ;
if( priv->pconn )
{
_9p_cleanup_fids( priv->pconn );
}
if( priv->datalock )
{
if( priv->datalock->data && priv->datalock->data->mr ) msk_dereg_mr( priv->datalock->data->mr ) ;
gsh_free( priv->datalock ) ;
}
if( priv->rdata )
{
gsh_free( priv->rdata ) ;
}
if( priv->rdmabuf ) gsh_free( priv->rdmabuf ) ;
if( priv->pconn ) gsh_free( priv->pconn ) ;
gsh_free( priv ) ;
}
msk_destroy_trans( &trans ) ;
pthread_exit( NULL ) ;
}
void *_9p_socket_thread(void *Arg)
{
long int tcp_sock = (long int)Arg;
int rc = -1;
struct pollfd fds[1];
int fdcount = 1;
static char my_name[MAXNAMLEN + 1];
socklen_t addrpeerlen = 0;
struct sockaddr_storage addrpeer;
char strcaller[INET6_ADDRSTRLEN];
request_data_t *req = NULL;
int tag;
unsigned long sequence = 0;
unsigned int i = 0;
char *_9pmsg = NULL;
uint32_t msglen;
struct _9p_conn _9p_conn;
int readlen = 0;
int total_readlen = 0;
snprintf(my_name, MAXNAMLEN, "9p_sock_mgr#fd=%ld", tcp_sock);
SetNameFunction(my_name);
/* Init the struct _9p_conn structure */
memset(&_9p_conn, 0, sizeof(_9p_conn));
pthread_mutex_init(&_9p_conn.sock_lock, NULL);
_9p_conn.trans_type = _9P_TCP;
_9p_conn.trans_data.sockfd = tcp_sock;
for (i = 0; i < FLUSH_BUCKETS; i++) {
pthread_mutex_init(&_9p_conn.flush_buckets[i].lock, NULL);
glist_init(&_9p_conn.flush_buckets[i].list);
}
atomic_store_uint32_t(&_9p_conn.refcount, 0);
/* Init the fids pointers array */
memset(&_9p_conn.fids, 0, _9P_FID_PER_CONN * sizeof(struct _9p_fid *));
/* Set initial msize.
* Client may request a lower value during TVERSION */
_9p_conn.msize = _9p_param._9p_tcp_msize;
if (gettimeofday(&_9p_conn.birth, NULL) == -1)
LogFatal(COMPONENT_9P, "Cannot get connection's time of birth");
addrpeerlen = sizeof(addrpeer);
rc = getpeername(tcp_sock, (struct sockaddr *)&addrpeer,
&addrpeerlen);
if (rc == -1) {
LogMajor(COMPONENT_9P,
"Cannot get peername to tcp socket for 9p, error %d (%s)",
errno, strerror(errno));
/* XXX */
strncpy(strcaller, "(unresolved)", INET6_ADDRSTRLEN);
strcaller[12] = '\0';
} else {
switch (addrpeer.ss_family) {
case AF_INET:
inet_ntop(addrpeer.ss_family,
&((struct sockaddr_in *)&addrpeer)->
sin_addr, strcaller, INET6_ADDRSTRLEN);
break;
case AF_INET6:
inet_ntop(addrpeer.ss_family,
&((struct sockaddr_in6 *)&addrpeer)->
sin6_addr, strcaller, INET6_ADDRSTRLEN);
break;
default:
snprintf(strcaller, INET6_ADDRSTRLEN, "BAD ADDRESS");
break;
}
LogEvent(COMPONENT_9P, "9p socket #%ld is connected to %s",
tcp_sock, strcaller);
printf("9p socket #%ld is connected to %s\n", tcp_sock,
strcaller);
}
_9p_conn.client = get_gsh_client(&addrpeer, false);
/* Set up the structure used by poll */
memset((char *)fds, 0, sizeof(struct pollfd));
fds[0].fd = tcp_sock;
fds[0].events =
POLLIN | POLLPRI | POLLRDBAND | POLLRDNORM | POLLRDHUP | POLLHUP |
POLLERR | POLLNVAL;
for (;;) {
total_readlen = 0; /* new message */
rc = poll(fds, fdcount, -1);
if (rc == -1) {
/* timeout = -1 => Wait indefinitely for events */
/* Interruption if not an issue */
if (errno == EINTR)
continue;
LogCrit(COMPONENT_9P,
"Got error %u (%s) on fd %ld connect to %s while polling on socket",
errno, strerror(errno), tcp_sock, strcaller);
}
if (fds[0].revents & POLLNVAL) {
LogEvent(COMPONENT_9P,
"Client %s on socket %lu produced POLLNVAL",
strcaller, tcp_sock);
goto end;
}
if (fds[0].revents & (POLLERR | POLLHUP | POLLRDHUP)) {
LogEvent(COMPONENT_9P,
"Client %s on socket %lu has shut down and closed",
strcaller, tcp_sock);
goto end;
}
if (!(fds[0].revents & (POLLIN | POLLRDNORM)))
continue;
/* Prepare to read the message */
_9pmsg = gsh_malloc(_9p_conn.msize);
if (_9pmsg == NULL) {
LogCrit(COMPONENT_9P,
"Could not allocate 9pmsg buffer for client %s on socket %lu",
strcaller, tcp_sock);
goto end;
}
/* An incoming 9P request: the msg has a 4 bytes header
showing the size of the msg including the header */
readlen = recv(fds[0].fd, _9pmsg,
_9P_HDR_SIZE, MSG_WAITALL);
if (readlen != _9P_HDR_SIZE)
goto badmsg;
msglen = *(uint32_t *) _9pmsg;
if (msglen > _9p_conn.msize) {
LogCrit(COMPONENT_9P,
"Message size too big! got %u, max = %u",
msglen, _9p_conn.msize);
goto end;
}
LogFullDebug(COMPONENT_9P,
"Received 9P/TCP message of size %u from client %s on socket %lu",
msglen, strcaller, tcp_sock);
total_readlen += readlen;
while (total_readlen < msglen) {
readlen = recv(fds[0].fd,
_9pmsg + total_readlen,
msglen - total_readlen,
0);
if (readlen > 0) {
total_readlen += readlen;
continue;
}
if (readlen == 0 ||
(readlen < 0 && errno != EINTR))
goto badmsg;
} /* while */
server_stats_transport_done(_9p_conn.client,
total_readlen, 1, 0,
0, 0, 0);
/* Message is good. */
req = pool_alloc(request_pool, NULL);
req->rtype = _9P_REQUEST;
req->r_u._9p._9pmsg = _9pmsg;
req->r_u._9p.pconn = &_9p_conn;
/* Add this request to the request list,
* should it be flushed later. */
tag = *(u16 *) (_9pmsg + _9P_HDR_SIZE +
_9P_TYPE_SIZE);
_9p_AddFlushHook(&req->r_u._9p, tag,
sequence++);
LogFullDebug(COMPONENT_9P,
"Request tag is %d\n", tag);
/* Message was OK push it */
DispatchWork9P(req);
/* Not our buffer anymore */
_9pmsg = NULL;
continue;
badmsg:
if (readlen == 0)
LogEvent(COMPONENT_9P,
"Premature end for Client %s on socket %lu, total read = %u",
strcaller, tcp_sock, total_readlen);
else if (readlen < 0) {
LogEvent(COMPONENT_9P,
"Read error client %s on socket %lu errno=%d, total read = %u",
strcaller, tcp_sock,
errno, total_readlen);
} else
LogEvent(COMPONENT_9P,
"Header too small! for client %s on socket %lu: readlen=%u expected=%u",
strcaller, tcp_sock, readlen,
_9P_HDR_SIZE);
/* Either way, we close the connection.
* It is not possible to survive
* once we get out of sync in the TCP stream
* with the client
*/
break; /* bail out */
} /* for( ;; ) */
end:
LogEvent(COMPONENT_9P, "Closing connection on socket %lu", tcp_sock);
close(tcp_sock);
/* Free buffer if we encountered an error
* before we could give it to a worker */
if (_9pmsg)
gsh_free(_9pmsg);
while (atomic_fetch_uint32_t(&_9p_conn.refcount)) {
LogEvent(COMPONENT_9P, "Waiting for workers to release pconn");
sleep(1);
}
_9p_cleanup_fids(&_9p_conn);
if (_9p_conn.client != NULL)
put_gsh_client(_9p_conn.client);
pthread_exit(NULL);
} /* _9p_socket_thread */