/*! \brief
* handles a new connection, called internally by tcp_main_loop/handle_io.
* \param si - pointer to one of the tcp socket_info structures on which
* an io event was detected (connection attempt)
* \return handle_* return convention: -1 on error, 0 on EAGAIN (no more
* io events queued), >0 on success. success/error refer only to
* the accept.
*/
static inline int handle_new_connect(struct socket_info* si)
{
union sockaddr_union su;
struct tcp_connection* tcpconn;
socklen_t su_len;
int new_sock;
int id;
/* got a connection on r */
su_len=sizeof(su);
new_sock=accept(si->socket, &(su.s), &su_len);
if (new_sock==-1){
if ((errno==EAGAIN)||(errno==EWOULDBLOCK))
return 0;
LM_ERR("failed to accept connection(%d): %s\n", errno, strerror(errno));
return -1;
}
if (tcp_connections_no>=tcp_max_connections){
LM_ERR("maximum number of connections exceeded: %d/%d\n",
tcp_connections_no, tcp_max_connections);
close(new_sock);
return 1; /* success, because the accept was succesfull */
}
if (tcp_init_sock_opt(new_sock)<0){
LM_ERR("tcp_init_sock_opt failed\n");
close(new_sock);
return 1; /* success, because the accept was succesfull */
}
/* add socket to list */
tcpconn=tcpconn_new(new_sock, &su, si, S_CONN_OK, F_CONN_ACCEPTED);
if (tcpconn){
tcpconn->refcnt++; /* safe, not yet available to the
outside world */
tcpconn_add(tcpconn);
LM_DBG("new connection: %p %d flags: %04x\n",
tcpconn, tcpconn->s, tcpconn->flags);
/* pass it to a child */
if(send2child(tcpconn,IO_WATCH_READ)<0){
LM_ERR("no children available\n");
id = tcpconn->id;
TCPCONN_LOCK(id);
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
_tcpconn_rm(tcpconn);
close(new_sock/*same as tcpconn->s*/);
}else tcpconn->lifetime=0; /* force expire */
TCPCONN_UNLOCK(id);
}
}else{ /*tcpconn==0 */
LM_ERR("tcpconn_new failed, closing socket\n");
close(new_sock);
}
return 1; /* accept() was succesfull */
}
/* handle a new connection, called internally by tcp_main_loop */
static inline void handle_new_connect(struct socket_info* si,
fd_set* sel_set, int* n)
{
union sockaddr_union su;
struct tcp_connection* tcpconn;
socklen_t su_len;
int new_sock;
if ((FD_ISSET(si->socket, sel_set))){
/* got a connection on r */
su_len=sizeof(su);
new_sock=accept(si->socket, &(su.s), &su_len);
(*n)--;
if (new_sock==-1){
LOG(L_ERR, "WARNING: tcp_main_loop: error while accepting"
" connection(%d): %s\n", errno, strerror(errno));
return;
}
if (init_sock_opt(new_sock)<0){
LOG(L_ERR, "ERROR: tcp_main_loop: init_sock_opt failed\n");
close(new_sock);
return;
}
/* add socket to list */
tcpconn=tcpconn_new(new_sock, &su, si, si->proto, S_CONN_ACCEPT);
if (tcpconn){
tcpconn->refcnt++; /* safe, not yet available to the
outside world */
tcpconn_add(tcpconn);
DBG("tcp_main_loop: new connection: %p %d\n",
tcpconn, tcpconn->s);
/* pass it to a child */
if(send2child(tcpconn)<0){
LOG(L_ERR,"ERROR: tcp_main_loop: no children "
"available\n");
TCPCONN_LOCK;
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
close(tcpconn->s);
_tcpconn_rm(tcpconn);
}else tcpconn->timeout=0; /* force expire */
TCPCONN_UNLOCK;
}
}else{ /*tcpconn==0 */
LOG(L_ERR, "ERROR: tcp_main_loop: tcpconn_new failed, "
"closing socket\n");
close(new_sock);
}
}
}
void tcp_main_loop()
{
int r;
int n;
fd_set master_set;
fd_set sel_set;
int maxfd;
struct tcp_connection* tcpconn;
unsigned h;
long response[2];
int cmd;
int bytes;
struct timeval timeout;
int fd;
/*init */
maxfd=0;
FD_ZERO(&master_set);
/* set all the listen addresses */
for (r=0; r<sock_no; r++){
if ((tcp_info[r].proto==PROTO_TCP) &&(tcp_info[r].socket!=-1)){
FD_SET(tcp_info[r].socket, &master_set);
if (tcp_info[r].socket>maxfd) maxfd=tcp_info[r].socket;
}
#ifdef USE_TLS
if ((!tls_disable)&&(tls_info[r].proto==PROTO_TLS) &&
(tls_info[r].socket!=-1)){
FD_SET(tls_info[r].socket, &master_set);
if (tls_info[r].socket>maxfd) maxfd=tls_info[r].socket;
}
#endif
}
/* set all the unix sockets used for child comm */
for (r=1; r<process_no; r++){
if (pt[r].unix_sock>0){ /* we can't have 0, we never close it!*/
FD_SET(pt[r].unix_sock, &master_set);
if (pt[r].unix_sock>maxfd) maxfd=pt[r].unix_sock;
}
}
for (r=0; r<tcp_children_no; r++){
if (tcp_children[r].unix_sock>0){ /* we can't have 0,
we never close it!*/
FD_SET(tcp_children[r].unix_sock, &master_set);
if (tcp_children[r].unix_sock>maxfd)
maxfd=tcp_children[r].unix_sock;
}
}
/* main loop*/
while(1){
sel_set=master_set;
timeout.tv_sec=TCP_MAIN_SELECT_TIMEOUT;
timeout.tv_usec=0;
n=select(maxfd+1, &sel_set, 0 ,0 , &timeout);
if (n<0){
if (errno==EINTR) continue; /* just a signal */
/* errors */
LOG(L_ERR, "ERROR: tcp_main_loop: select:(%d) %s\n", errno,
strerror(errno));
n=0;
}
for (r=0; r<sock_no && n; r++){
handle_new_connect(&tcp_info[r], &sel_set, &n);
#ifdef USE_TLS
if (!tls_disable)
handle_new_connect(&tls_info[r], &sel_set, &n);
#endif
}
/* check all the read fds (from the tcpconn_addr_hash ) */
for (h=0; h<TCP_ADDR_HASH_SIZE; h++){
for(tcpconn=tcpconn_addr_hash[h]; tcpconn && n;
tcpconn=tcpconn->next){
/* FIXME: is refcnt==0 really necessary? */
if ((tcpconn->refcnt==0)&&(FD_ISSET(tcpconn->s, &sel_set))){
/* new data available */
n--;
/* pass it to child, so remove it from select list */
DBG("tcp_main_loop: data available on %p [h:%d] %d\n",
tcpconn, h, tcpconn->s);
FD_CLR(tcpconn->s, &master_set);
tcpconn_ref(tcpconn); /* refcnt ++ */
if (send2child(tcpconn)<0){
LOG(L_ERR,"ERROR: tcp_main_loop: no "
"children available\n");
TCPCONN_LOCK;
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
fd=tcpconn->s;
_tcpconn_rm(tcpconn);
close(fd);
}else tcpconn->timeout=0; /* force expire*/
TCPCONN_UNLOCK;
}
}
}
}
/* check unix sockets & listen | destroy connections */
/* tcp_children readers first */
for (r=0; r<tcp_children_no && n; r++){
if ( (tcp_children[r].unix_sock>0) &&
FD_ISSET(tcp_children[r].unix_sock, &sel_set)){
/* (we can't have a fd==0, 0 is never closed )*/
n--;
/* read until sizeof(response)
* (this is a SOCK_STREAM so read is not atomic */
bytes=recv_all(tcp_children[r].unix_sock, response,
sizeof(response));
if (bytes==0){
/* EOF -> bad, child has died */
DBG("DBG: tcp_main_loop: dead tcp child %d"
" (shutting down?)\n", r);
/* don't listen on it any more */
FD_CLR(tcp_children[r].unix_sock, &master_set);
/*exit(-1);*/
continue; /* skip this and try the next one */
}else if (bytes<0){
LOG(L_CRIT, "ERROR: tcp_main_loop: read from tcp child %d "
"%s\n", r, strerror(errno));
/* try to ignore ? */
continue; /* skip this and try the next one */
}
DBG("tcp_main_loop: reader response= %lx, %ld from %d \n",
response[0], response[1], r);
cmd=response[1];
tcpconn=(struct tcp_connection*)response[0];
switch(cmd){
case CONN_RELEASE:
tcp_children[r].busy--;
if (tcpconn){
if (tcpconn->state==S_CONN_BAD){
tcpconn_destroy(tcpconn);
break;
}
FD_SET(tcpconn->s, &master_set);
if (maxfd<tcpconn->s) maxfd=tcpconn->s;
/* update the timeout*/
tcpconn->timeout=get_ticks()+TCP_CON_TIMEOUT;
tcpconn_put(tcpconn);
DBG("tcp_main_loop: CONN_RELEASE %p"
" refcnt= %d\n",
tcpconn, tcpconn->refcnt);
}
break;
case CONN_ERROR:
case CONN_DESTROY:
case CONN_EOF:
/* WARNING: this will auto-dec. refcnt! */
tcp_children[pt[r].idx].busy--;
if (tcpconn){
if (tcpconn->s!=-1)
FD_CLR(tcpconn->s, &master_set);
tcpconn_destroy(tcpconn);
}
break;
default:
LOG(L_CRIT, "BUG: tcp_main_loop: unknown cmd %d"
" from tcp reader %d\n",
cmd, r);
}
}
}
/* check "send" unix sockets & listen | destroy connections */
/* start from 1, the "main" process does not transmit anything*/
for (r=1; r<process_no && n; r++){
if ( (pt[r].unix_sock>0) && FD_ISSET(pt[r].unix_sock, &sel_set)){
/* (we can't have a fd==0, 0 is never closed )*/
n--;
/* read until sizeof(response)
* (this is a SOCK_STREAM so read is not atomic */
bytes=recv_all(pt[r].unix_sock, response, sizeof(response));
if (bytes==0){
/* EOF -> bad, child has died */
DBG("DBG: tcp_main_loop: dead child %d"
" (shutting down?)\n", r);
/* don't listen on it any more */
FD_CLR(pt[r].unix_sock, &master_set);
/*exit(-1);*/
continue; /* skip this and try the next one */
}else if (bytes<0){
LOG(L_CRIT, "ERROR: tcp_main_loop: read from child: %s\n",
strerror(errno));
/* try to ignore ? */
continue; /* skip this and try the next one */
}
DBG("tcp_main_loop: read response= %lx, %ld from %d (%d)\n",
response[0], response[1], r, pt[r].pid);
cmd=response[1];
tcpconn=(struct tcp_connection*)response[0];
switch(cmd){
case CONN_ERROR:
if (tcpconn){
if (tcpconn->s!=-1)
FD_CLR(tcpconn->s, &master_set);
tcpconn_destroy(tcpconn);
}
break;
case CONN_GET_FD:
/* send the requested FD */
/* WARNING: take care of setting refcnt properly to
* avoid race condition */
if (tcpconn){
if (send_fd(pt[r].unix_sock, &tcpconn,
sizeof(tcpconn), tcpconn->s)<=0){
LOG(L_ERR, "ERROR: tcp_main_loop:"
"send_fd failed\n");
}
}else{
LOG(L_CRIT, "BUG: tcp_main_loop: null pointer\n");
}
break;
case CONN_NEW:
/* update the fd in the requested tcpconn*/
/* WARNING: take care of setting refcnt properly to
* avoid race condition */
if (tcpconn){
bytes=receive_fd(pt[r].unix_sock, &tcpconn,
sizeof(tcpconn), &tcpconn->s);
if (bytes<sizeof(tcpconn)){
if (bytes<0){
LOG(L_CRIT, "BUG: tcp_main_loop:"
" CONN_NEW: receive_fd "
"failed\n");
}else{
LOG(L_CRIT, "BUG: tcp_main_loop:"
" CONN_NEW: to few bytes "
"received (%d)\n", bytes );
}
break; /* try to ignore */
}
/* add tcpconn to the list*/
tcpconn_add(tcpconn);
FD_SET(tcpconn->s, &master_set);
if (maxfd<tcpconn->s) maxfd=tcpconn->s;
/* update the timeout*/
tcpconn->timeout=get_ticks()+TCP_CON_TIMEOUT;
}else{
LOG(L_CRIT, "BUG: tcp_main_loop: null pointer\n");
}
break;
default:
LOG(L_CRIT, "BUG: tcp_main_loop: unknown cmd %d\n",
cmd);
}
}
} /* for */
/* remove old connections */
tcpconn_timeout(&master_set);
}
}
/*! \brief handles io from a "generic" ser process (get fd or new_fd from a tcp_send)
*
* \param p - pointer in the ser processes array (pt[]), to the entry for
* which an io event was detected
* \param fd_i - fd index in the fd_array (usefull for optimizing
* io_watch_deletes)
* \return handle_* return convention:
* - -1 on error reading from the fd,
* - 0 on EAGAIN or when no more io events are queued
* (receive buffer empty),
* - >0 on successfull reads from the fd (the receive buffer might
* be non-empty).
*/
inline static int handle_ser_child(struct process_table* p, int fd_i)
{
struct tcp_connection* tcpconn;
long response[2];
int cmd;
int bytes;
int ret;
int fd;
ret=-1;
if (p->unix_sock<=0){
/* (we can't have a fd==0, 0 is never closed )*/
LM_CRIT("fd %d for %d (pid %d)\n",
p->unix_sock, (int)(p-&pt[0]), p->pid);
goto error;
}
/* get all bytes and the fd (if transmitted)
* (this is a SOCK_STREAM so read is not atomic) */
bytes=receive_fd(p->unix_sock, response, sizeof(response), &fd,
MSG_DONTWAIT);
if (bytes<(int)sizeof(response)){
/* too few bytes read */
if (bytes==0){
/* EOF -> bad, child has died */
LM_DBG("dead child %d, pid %d"
" (shutting down?)\n", (int)(p-&pt[0]), p->pid);
/* don't listen on it any more */
io_watch_del(&io_h, p->unix_sock, fd_i, 0);
goto error; /* child dead => no further io events from it */
}else if (bytes<0){
/* EAGAIN is ok if we try to empty the buffer
* e.g: SIGIO_RT overflow mode or EPOLL ET */
if ((errno!=EAGAIN) && (errno!=EWOULDBLOCK)){
LM_CRIT("read from child %d (pid %d): %s [%d]\n",
(int)(p-&pt[0]), p->pid, strerror(errno), errno);
ret=-1;
}else{
ret=0;
}
/* try to ignore ? */
goto end;
}else{
/* should never happen */
LM_CRIT("too few bytes received (%d)\n", bytes );
ret=0; /* something was read so there is no error; otoh if
receive_fd returned less then requested => the receive
buffer is empty => no more io queued on this fd */
goto end;
}
}
ret=1; /* something was received, there might be more queued */
LM_DBG("read response= %lx, %ld, fd %d from %d (%d)\n",
response[0], response[1], fd, (int)(p-&pt[0]), p->pid);
cmd=response[1];
tcpconn=(struct tcp_connection*)response[0];
if (tcpconn==0){
LM_CRIT("null tcpconn pointer received from child %d (pid %d)"
"%lx, %lx\n", (int)(p-&pt[0]), p->pid, response[0], response[1]) ;
goto end;
}
switch(cmd){
case CONN_ERROR:
if (!(tcpconn->flags & F_CONN_REMOVED) && (tcpconn->s!=-1)){
io_watch_del(&io_h, tcpconn->s, -1, IO_FD_CLOSING);
tcpconn->flags|=F_CONN_REMOVED;
}
tcpconn_destroy(tcpconn); /* will close also the fd */
break;
case CONN_GET_FD:
/* send the requested FD */
/* WARNING: take care of setting refcnt properly to
* avoid race condition */
if (send_fd(p->unix_sock, &tcpconn, sizeof(tcpconn),
tcpconn->s)<=0){
LM_ERR("send_fd failed\n");
}
break;
case CONN_NEW:
/* update the fd in the requested tcpconn*/
/* WARNING: take care of setting refcnt properly to
* avoid race condition */
if (fd==-1){
LM_CRIT(" cmd CONN_NEW: no fd received\n");
break;
}
tcpconn->s=fd;
/* add tcpconn to the list*/
tcpconn_add(tcpconn);
/* update the timeout*/
tcpconn->timeout=get_ticks()+tcp_con_lifetime;
io_watch_add(&io_h, tcpconn->s, F_TCPCONN, tcpconn);
tcpconn->flags&=~F_CONN_REMOVED;
break;
default:
LM_CRIT("unknown cmd %d\n", cmd);
}
end:
return ret;
error:
return -1;
}
/*! \brief handles io from a "generic" ser process (get fd or new_fd from a tcp_send)
*
* \param p - pointer in the ser processes array (pt[]), to the entry for
* which an io event was detected
* \param fd_i - fd index in the fd_array (useful for optimizing
* io_watch_deletes)
* \return handle_* return convention:
* - -1 on error reading from the fd,
* - 0 on EAGAIN or when no more io events are queued
* (receive buffer empty),
* - >0 on successfull reads from the fd (the receive buffer might
* be non-empty).
*/
inline static int handle_worker(struct process_table* p, int fd_i)
{
struct tcp_connection* tcpconn;
long response[2];
int cmd;
int bytes;
int ret;
int fd;
ret=-1;
if (p->unix_sock<=0){
/* (we can't have a fd==0, 0 is never closed )*/
LM_CRIT("fd %d for %d (pid %d)\n",
p->unix_sock, (int)(p-&pt[0]), p->pid);
goto error;
}
/* get all bytes and the fd (if transmitted)
* (this is a SOCK_STREAM so read is not atomic) */
bytes=receive_fd(p->unix_sock, response, sizeof(response), &fd,
MSG_DONTWAIT);
if (bytes<(int)sizeof(response)){
/* too few bytes read */
if (bytes==0){
/* EOF -> bad, child has died */
LM_DBG("dead child %d, pid %d"
" (shutting down?)\n", (int)(p-&pt[0]), p->pid);
/* don't listen on it any more */
reactor_del_reader( p->unix_sock, fd_i, 0/*flags*/);
goto error; /* child dead => no further io events from it */
}else if (bytes<0){
/* EAGAIN is ok if we try to empty the buffer
* e.g: SIGIO_RT overflow mode or EPOLL ET */
if ((errno!=EAGAIN) && (errno!=EWOULDBLOCK)){
LM_CRIT("read from child %d (pid %d): %s [%d]\n",
(int)(p-&pt[0]), p->pid, strerror(errno), errno);
ret=-1;
}else{
ret=0;
}
/* try to ignore ? */
goto end;
}else{
/* should never happen */
LM_CRIT("too few bytes received (%d)\n", bytes );
ret=0; /* something was read so there is no error; otoh if
receive_fd returned less then requested => the receive
buffer is empty => no more io queued on this fd */
goto end;
}
}
ret=1; /* something was received, there might be more queued */
LM_DBG("read response= %lx, %ld, fd %d from %d (%d)\n",
response[0], response[1], fd, (int)(p-&pt[0]), p->pid);
cmd=response[1];
tcpconn=(struct tcp_connection*)response[0];
if (tcpconn==0){
LM_CRIT("null tcpconn pointer received from child %d (pid %d)"
"%lx, %lx\n", (int)(p-&pt[0]), p->pid, response[0], response[1]) ;
goto end;
}
switch(cmd){
case CONN_ERROR:
if (!(tcpconn->flags & F_CONN_REMOVED) && (tcpconn->s!=-1)){
reactor_del_all( tcpconn->s, -1, IO_FD_CLOSING);
tcpconn->flags|=F_CONN_REMOVED;
}
tcpconn_destroy(tcpconn); /* will close also the fd */
break;
case CONN_GET_FD:
/* send the requested FD */
/* WARNING: take care of setting refcnt properly to
* avoid race condition */
if (send_fd(p->unix_sock, &tcpconn, sizeof(tcpconn),
tcpconn->s)<=0){
LM_ERR("send_fd failed\n");
}
break;
case CONN_NEW:
/* update the fd in the requested tcpconn*/
/* WARNING: take care of setting refcnt properly to
* avoid race condition */
if (fd==-1){
LM_CRIT(" cmd CONN_NEW: no fd received\n");
break;
}
tcpconn->s=fd;
/* add tcpconn to the list*/
tcpconn_add(tcpconn);
reactor_add_reader( tcpconn->s, F_TCPCONN, RCT_PRIO_NET, tcpconn);
tcpconn->flags&=~F_CONN_REMOVED;
break;
case ASYNC_CONNECT:
/* connection is not yet linked to hash = not yet
* available to the outside world */
if (fd==-1){
LM_CRIT(" cmd CONN_NEW: no fd received\n");
break;
}
tcpconn->s=fd;
/* add tcpconn to the list*/
tcpconn_add(tcpconn);
/* FIXME - now we have lifetime==default_lifetime - should we
* set a shorter one when waiting for a connect ??? */
/* only maintain the socket in the IO_WATCH_WRITE watcher
* while we have stuff to write - otherwise we're going to get
* useless events */
reactor_add_writer( tcpconn->s, F_TCPCONN, RCT_PRIO_NET, tcpconn);
tcpconn->flags&=~F_CONN_REMOVED;
break;
case ASYNC_WRITE:
if (tcpconn->state==S_CONN_BAD){
tcpconn->lifetime=0;
break;
}
/* must be after the de-ref*/
reactor_add_writer( tcpconn->s, F_TCPCONN, RCT_PRIO_NET, tcpconn);
tcpconn->flags&=~F_CONN_REMOVED;
break;
default:
LM_CRIT("unknown cmd %d\n", cmd);
}
end:
return ret;
error:
return -1;
}
