/*! \brief used internally by tcp_main_loop() */
static void tcpconn_destroy(struct tcp_connection* tcpconn)
{
int fd;
TCPCONN_LOCK; /*avoid races w/ tcp_send*/
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
LM_DBG("destroying connection %p, flags %04x\n",
tcpconn, tcpconn->flags);
fd=tcpconn->s;
#ifdef USE_TLS
/*FIXME: lock ->writelock ? */
if (tcpconn->type==PROTO_TLS)
tls_close(tcpconn, fd);
#endif
_tcpconn_rm(tcpconn);
close(fd);
tcp_connections_no--;
}else{
/* force timeout */
tcpconn->timeout=0;
tcpconn->state=S_CONN_BAD;
LM_DBG("delaying (%p, flags %04x) ...\n",
tcpconn, tcpconn->flags);
}
TCPCONN_UNLOCK;
}
/* used internally by tcp_main_loop() */
static void tcpconn_destroy(struct tcp_connection* tcpconn)
{
int fd;
TCPCONN_LOCK; /*avoid races w/ tcp_send*/
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
DBG("tcp_main_loop: destroying connection\n");
fd=tcpconn->s;
#ifdef USE_TLS
/*FIXME: lock ->writelock ? */
if (tcpconn->type==PROTO_TLS)
tls_close(tcpconn, fd);
#endif
_tcpconn_rm(tcpconn);
close(fd);
}else{
/* force timeout */
tcpconn->timeout=0;
tcpconn->state=S_CONN_BAD;
DBG("tcp_main_loop: delaying ...\n");
}
TCPCONN_UNLOCK;
}
/* very ineficient for now - FIXME*/
void tcpconn_timeout(fd_set* set)
{
struct tcp_connection *c, *next;
int ticks;
unsigned h;
int fd;
ticks=get_ticks();
TCPCONN_LOCK; /* fixme: we can lock only on delete IMO */
for(h=0; h<TCP_ADDR_HASH_SIZE; h++){
c=tcpconn_addr_hash[h];
while(c){
next=c->next;
if ((c->refcnt==0) && (ticks>c->timeout)) {
DBG("tcpconn_timeout: timeout for hash=%d - %p (%d > %d)\n",
h, c, ticks, c->timeout);
fd=c->s;
#ifdef USE_TLS
if (c->type==PROTO_TLS)
tls_close(c, fd);
#endif
_tcpconn_rm(c);
if (fd>0) {
FD_CLR(fd, set);
close(fd);
}
}
c=next;
}
}
TCPCONN_UNLOCK;
}
static inline void tcpconn_destroy(struct tcp_connection* tcpconn)
{
int fd;
int id = tcpconn->id;
TCPCONN_LOCK(id); /*avoid races w/ tcp_send*/
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
LM_DBG("destroying connection %p, flags %04x\n",
tcpconn, tcpconn->flags);
fd=tcpconn->s;
_tcpconn_rm(tcpconn);
if (fd >= 0)
close(fd);
tcp_connections_no--;
}else{
/* force timeout */
tcpconn->lifetime=0;
tcpconn->state=S_CONN_BAD;
LM_DBG("delaying (%p, flags %04x) ref = %d ...\n",
tcpconn, tcpconn->flags, tcpconn->refcnt);
}
TCPCONN_UNLOCK(id);
}
int tcp_conn_send(struct tcp_connection *c)
{
long response[2];
int n;
/* inform TCP main about this new connection */
if (c->state==S_CONN_CONNECTING) {
response[0]=(long)c;
response[1]=ASYNC_CONNECT;
n=send_fd(unix_tcp_sock, response, sizeof(response), c->s);
if (n<=0) {
LM_ERR("Failed to send the socket to main for async connection\n");
goto error;
}
close(c->s);
} else {
response[0]=(long)c;
response[1]=CONN_NEW;
n=send_fd(unix_tcp_sock, response, sizeof(response), c->s);
if (n<=0){
LM_ERR("failed send_fd: %s (%d)\n", strerror(errno), errno);
goto error;
}
}
return 0;
error:
_tcpconn_rm(c);
tcp_connections_no--;
return -1;
}
/*! \brief
* handles a new connection, called internally by tcp_main_loop/handle_io.
* \param si - pointer to one of the tcp socket_info structures on which
* an io event was detected (connection attempt)
* \return handle_* return convention: -1 on error, 0 on EAGAIN (no more
* io events queued), >0 on success. success/error refer only to
* the accept.
*/
static inline int handle_new_connect(struct socket_info* si)
{
union sockaddr_union su;
struct tcp_connection* tcpconn;
socklen_t su_len;
int new_sock;
int id;
/* got a connection on r */
su_len=sizeof(su);
new_sock=accept(si->socket, &(su.s), &su_len);
if (new_sock==-1){
if ((errno==EAGAIN)||(errno==EWOULDBLOCK))
return 0;
LM_ERR("failed to accept connection(%d): %s\n", errno, strerror(errno));
return -1;
}
if (tcp_connections_no>=tcp_max_connections){
LM_ERR("maximum number of connections exceeded: %d/%d\n",
tcp_connections_no, tcp_max_connections);
close(new_sock);
return 1; /* success, because the accept was succesfull */
}
if (tcp_init_sock_opt(new_sock)<0){
LM_ERR("tcp_init_sock_opt failed\n");
close(new_sock);
return 1; /* success, because the accept was succesfull */
}
/* add socket to list */
tcpconn=tcpconn_new(new_sock, &su, si, S_CONN_OK, F_CONN_ACCEPTED);
if (tcpconn){
tcpconn->refcnt++; /* safe, not yet available to the
outside world */
tcpconn_add(tcpconn);
LM_DBG("new connection: %p %d flags: %04x\n",
tcpconn, tcpconn->s, tcpconn->flags);
/* pass it to a child */
if(send2child(tcpconn,IO_WATCH_READ)<0){
LM_ERR("no children available\n");
id = tcpconn->id;
TCPCONN_LOCK(id);
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
_tcpconn_rm(tcpconn);
close(new_sock/*same as tcpconn->s*/);
}else tcpconn->lifetime=0; /* force expire */
TCPCONN_UNLOCK(id);
}
}else{ /*tcpconn==0 */
LM_ERR("tcpconn_new failed, closing socket\n");
close(new_sock);
}
return 1; /* accept() was succesfull */
}
/* handle a new connection, called internally by tcp_main_loop */
static inline void handle_new_connect(struct socket_info* si,
fd_set* sel_set, int* n)
{
union sockaddr_union su;
struct tcp_connection* tcpconn;
socklen_t su_len;
int new_sock;
if ((FD_ISSET(si->socket, sel_set))){
/* got a connection on r */
su_len=sizeof(su);
new_sock=accept(si->socket, &(su.s), &su_len);
(*n)--;
if (new_sock==-1){
LOG(L_ERR, "WARNING: tcp_main_loop: error while accepting"
" connection(%d): %s\n", errno, strerror(errno));
return;
}
if (init_sock_opt(new_sock)<0){
LOG(L_ERR, "ERROR: tcp_main_loop: init_sock_opt failed\n");
close(new_sock);
return;
}
/* add socket to list */
tcpconn=tcpconn_new(new_sock, &su, si, si->proto, S_CONN_ACCEPT);
if (tcpconn){
tcpconn->refcnt++; /* safe, not yet available to the
outside world */
tcpconn_add(tcpconn);
DBG("tcp_main_loop: new connection: %p %d\n",
tcpconn, tcpconn->s);
/* pass it to a child */
if(send2child(tcpconn)<0){
LOG(L_ERR,"ERROR: tcp_main_loop: no children "
"available\n");
TCPCONN_LOCK;
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
close(tcpconn->s);
_tcpconn_rm(tcpconn);
}else tcpconn->timeout=0; /* force expire */
TCPCONN_UNLOCK;
}
}else{ /*tcpconn==0 */
LOG(L_ERR, "ERROR: tcp_main_loop: tcpconn_new failed, "
"closing socket\n");
close(new_sock);
}
}
}
/*! \brief
* handles an io event on one of the watched tcp connections
*
* \param tcpconn - pointer to the tcp_connection for which we have an io ev.
* \param fd_i - index in the fd_array table (needed for delete)
* \return handle_* return convention, but on success it always returns 0
* (because it's one-shot, after a succesfull execution the fd is
* removed from tcp_main's watch fd list and passed to a child =>
* tcp_main is not interested in further io events that might be
* queued for this fd)
*/
inline static int handle_tcpconn_ev(struct tcp_connection* tcpconn, int fd_i)
{
int fd;
/* is refcnt!=0 really necessary?
* No, in fact it's a bug: I can have the following situation: a send only
* tcp connection used by n processes simultaneously => refcnt = n. In
* the same time I can have a read event and this situation is perfectly
* valid. -- andrei
*/
#if 0
if ((tcpconn->refcnt!=0)){
/* FIXME: might be valid for sigio_rt iff fd flags are not cleared
* (there is a short window in which it could generate a sig
* that would be catched by tcp_main) */
LM_CRIT("io event on referenced tcpconn (%p), refcnt=%d, fd=%d\n",
tcpconn, tcpconn->refcnt, tcpconn->s);
return -1;
}
#endif
/* pass it to child, so remove it from the io watch list */
LM_DBG("data available on %p %d\n", tcpconn, tcpconn->s);
if (io_watch_del(&io_h, tcpconn->s, fd_i, 0)==-1) goto error;
tcpconn->flags|=F_CONN_REMOVED;
tcpconn_ref(tcpconn); /* refcnt ++ */
if (send2child(tcpconn)<0){
LM_ERR("no children available\n");
TCPCONN_LOCK;
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
fd=tcpconn->s;
_tcpconn_rm(tcpconn);
close(fd);
}else tcpconn->timeout=0; /* force expire*/
TCPCONN_UNLOCK;
}
return 0; /* we are not interested in possibly queued io events,
the fd was either passed to a child, or closed */
error:
return -1;
}
/*! \brief very inefficient for now - FIXME
* keep in sync with tcpconn_destroy, the "delete" part should be
* the same except for io_watch_del..
* \todo FIXME (very inefficient for now)
*/
static inline void __tcpconn_lifetime(int force)
{
struct tcp_connection *c, *next;
unsigned int ticks,part;
unsigned h;
int fd;
if (have_ticks())
ticks=get_ticks();
else
ticks=0;
for( part=0 ; part<TCP_PARTITION_SIZE ; part++ ) {
TCPCONN_LOCK(part); /* fixme: we can lock only on delete IMO */
for(h=0; h<TCP_ID_HASH_SIZE; h++){
c=TCP_PART(part).tcpconn_id_hash[h];
while(c){
next=c->id_next;
if (force ||((c->refcnt==0) && (ticks>c->lifetime))) {
if (!force)
LM_DBG("timeout for hash=%d - %p"
" (%d > %d)\n", h, c, ticks, c->lifetime);
fd=c->s;
_tcpconn_rm(c);
if ((!force)&&(fd>0)&&(c->refcnt==0)) {
if (!(c->flags & F_CONN_REMOVED)){
reactor_del_all( fd, -1, IO_FD_CLOSING);
c->flags|=F_CONN_REMOVED;
}
close(fd);
}
tcp_connections_no--;
}
c=next;
}
}
TCPCONN_UNLOCK(part);
}
}
/*! \brief very inefficient for now - FIXME
* keep in sync with tcpconn_destroy, the "delete" part should be
* the same except for io_watch_del..
* \todo FIXME (very inefficient for now)
*/
static inline void tcpconn_timeout(int force)
{
struct tcp_connection *c, *next;
unsigned int ticks;
unsigned h;
int fd;
ticks=get_ticks();
TCPCONN_LOCK; /* fixme: we can lock only on delete IMO */
for(h=0; h<TCP_ID_HASH_SIZE; h++){
c=tcpconn_id_hash[h];
while(c){
next=c->id_next;
if (force ||((c->refcnt==0) && (ticks>c->timeout))) {
if (!force)
LM_DBG("timeout for hash=%d - %p"
" (%d > %d)\n", h, c, ticks, c->timeout);
fd=c->s;
#ifdef USE_TLS
if (c->type==PROTO_TLS)
tls_close(c, fd);
#endif
_tcpconn_rm(c);
if ((!force)&&(fd>0)&&(c->refcnt==0)) {
if (!(c->flags & F_CONN_REMOVED)){
io_watch_del(&io_h, fd, -1, IO_FD_CLOSING);
c->flags|=F_CONN_REMOVED;
}
close(fd);
}
tcp_connections_no--;
}
c=next;
}
}
TCPCONN_UNLOCK;
}
void tcp_main_loop()
{
int r;
int n;
fd_set master_set;
fd_set sel_set;
int maxfd;
struct tcp_connection* tcpconn;
unsigned h;
long response[2];
int cmd;
int bytes;
struct timeval timeout;
int fd;
/*init */
maxfd=0;
FD_ZERO(&master_set);
/* set all the listen addresses */
for (r=0; r<sock_no; r++){
if ((tcp_info[r].proto==PROTO_TCP) &&(tcp_info[r].socket!=-1)){
FD_SET(tcp_info[r].socket, &master_set);
if (tcp_info[r].socket>maxfd) maxfd=tcp_info[r].socket;
}
#ifdef USE_TLS
if ((!tls_disable)&&(tls_info[r].proto==PROTO_TLS) &&
(tls_info[r].socket!=-1)){
FD_SET(tls_info[r].socket, &master_set);
if (tls_info[r].socket>maxfd) maxfd=tls_info[r].socket;
}
#endif
}
/* set all the unix sockets used for child comm */
for (r=1; r<process_no; r++){
if (pt[r].unix_sock>0){ /* we can't have 0, we never close it!*/
FD_SET(pt[r].unix_sock, &master_set);
if (pt[r].unix_sock>maxfd) maxfd=pt[r].unix_sock;
}
}
for (r=0; r<tcp_children_no; r++){
if (tcp_children[r].unix_sock>0){ /* we can't have 0,
we never close it!*/
FD_SET(tcp_children[r].unix_sock, &master_set);
if (tcp_children[r].unix_sock>maxfd)
maxfd=tcp_children[r].unix_sock;
}
}
/* main loop*/
while(1){
sel_set=master_set;
timeout.tv_sec=TCP_MAIN_SELECT_TIMEOUT;
timeout.tv_usec=0;
n=select(maxfd+1, &sel_set, 0 ,0 , &timeout);
if (n<0){
if (errno==EINTR) continue; /* just a signal */
/* errors */
LOG(L_ERR, "ERROR: tcp_main_loop: select:(%d) %s\n", errno,
strerror(errno));
n=0;
}
for (r=0; r<sock_no && n; r++){
handle_new_connect(&tcp_info[r], &sel_set, &n);
#ifdef USE_TLS
if (!tls_disable)
handle_new_connect(&tls_info[r], &sel_set, &n);
#endif
}
/* check all the read fds (from the tcpconn_addr_hash ) */
for (h=0; h<TCP_ADDR_HASH_SIZE; h++){
for(tcpconn=tcpconn_addr_hash[h]; tcpconn && n;
tcpconn=tcpconn->next){
/* FIXME: is refcnt==0 really necessary? */
if ((tcpconn->refcnt==0)&&(FD_ISSET(tcpconn->s, &sel_set))){
/* new data available */
n--;
/* pass it to child, so remove it from select list */
DBG("tcp_main_loop: data available on %p [h:%d] %d\n",
tcpconn, h, tcpconn->s);
FD_CLR(tcpconn->s, &master_set);
tcpconn_ref(tcpconn); /* refcnt ++ */
if (send2child(tcpconn)<0){
LOG(L_ERR,"ERROR: tcp_main_loop: no "
"children available\n");
TCPCONN_LOCK;
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
fd=tcpconn->s;
_tcpconn_rm(tcpconn);
close(fd);
}else tcpconn->timeout=0; /* force expire*/
TCPCONN_UNLOCK;
}
}
}
}
/* check unix sockets & listen | destroy connections */
/* tcp_children readers first */
for (r=0; r<tcp_children_no && n; r++){
if ( (tcp_children[r].unix_sock>0) &&
FD_ISSET(tcp_children[r].unix_sock, &sel_set)){
/* (we can't have a fd==0, 0 is never closed )*/
n--;
/* read until sizeof(response)
* (this is a SOCK_STREAM so read is not atomic */
bytes=recv_all(tcp_children[r].unix_sock, response,
sizeof(response));
if (bytes==0){
/* EOF -> bad, child has died */
DBG("DBG: tcp_main_loop: dead tcp child %d"
" (shutting down?)\n", r);
/* don't listen on it any more */
FD_CLR(tcp_children[r].unix_sock, &master_set);
/*exit(-1);*/
continue; /* skip this and try the next one */
}else if (bytes<0){
LOG(L_CRIT, "ERROR: tcp_main_loop: read from tcp child %d "
"%s\n", r, strerror(errno));
/* try to ignore ? */
continue; /* skip this and try the next one */
}
DBG("tcp_main_loop: reader response= %lx, %ld from %d \n",
response[0], response[1], r);
cmd=response[1];
tcpconn=(struct tcp_connection*)response[0];
switch(cmd){
case CONN_RELEASE:
tcp_children[r].busy--;
if (tcpconn){
if (tcpconn->state==S_CONN_BAD){
tcpconn_destroy(tcpconn);
break;
}
FD_SET(tcpconn->s, &master_set);
if (maxfd<tcpconn->s) maxfd=tcpconn->s;
/* update the timeout*/
tcpconn->timeout=get_ticks()+TCP_CON_TIMEOUT;
tcpconn_put(tcpconn);
DBG("tcp_main_loop: CONN_RELEASE %p"
" refcnt= %d\n",
tcpconn, tcpconn->refcnt);
}
break;
case CONN_ERROR:
case CONN_DESTROY:
case CONN_EOF:
/* WARNING: this will auto-dec. refcnt! */
tcp_children[pt[r].idx].busy--;
if (tcpconn){
if (tcpconn->s!=-1)
FD_CLR(tcpconn->s, &master_set);
tcpconn_destroy(tcpconn);
}
break;
default:
LOG(L_CRIT, "BUG: tcp_main_loop: unknown cmd %d"
" from tcp reader %d\n",
cmd, r);
}
}
}
/* check "send" unix sockets & listen | destroy connections */
/* start from 1, the "main" process does not transmit anything*/
for (r=1; r<process_no && n; r++){
if ( (pt[r].unix_sock>0) && FD_ISSET(pt[r].unix_sock, &sel_set)){
/* (we can't have a fd==0, 0 is never closed )*/
n--;
/* read until sizeof(response)
* (this is a SOCK_STREAM so read is not atomic */
bytes=recv_all(pt[r].unix_sock, response, sizeof(response));
if (bytes==0){
/* EOF -> bad, child has died */
DBG("DBG: tcp_main_loop: dead child %d"
" (shutting down?)\n", r);
/* don't listen on it any more */
FD_CLR(pt[r].unix_sock, &master_set);
/*exit(-1);*/
continue; /* skip this and try the next one */
}else if (bytes<0){
LOG(L_CRIT, "ERROR: tcp_main_loop: read from child: %s\n",
strerror(errno));
/* try to ignore ? */
continue; /* skip this and try the next one */
}
DBG("tcp_main_loop: read response= %lx, %ld from %d (%d)\n",
response[0], response[1], r, pt[r].pid);
cmd=response[1];
tcpconn=(struct tcp_connection*)response[0];
switch(cmd){
case CONN_ERROR:
if (tcpconn){
if (tcpconn->s!=-1)
FD_CLR(tcpconn->s, &master_set);
tcpconn_destroy(tcpconn);
}
break;
case CONN_GET_FD:
/* send the requested FD */
/* WARNING: take care of setting refcnt properly to
* avoid race condition */
if (tcpconn){
if (send_fd(pt[r].unix_sock, &tcpconn,
sizeof(tcpconn), tcpconn->s)<=0){
LOG(L_ERR, "ERROR: tcp_main_loop:"
"send_fd failed\n");
}
}else{
LOG(L_CRIT, "BUG: tcp_main_loop: null pointer\n");
}
break;
case CONN_NEW:
/* update the fd in the requested tcpconn*/
/* WARNING: take care of setting refcnt properly to
* avoid race condition */
if (tcpconn){
bytes=receive_fd(pt[r].unix_sock, &tcpconn,
sizeof(tcpconn), &tcpconn->s);
if (bytes<sizeof(tcpconn)){
if (bytes<0){
LOG(L_CRIT, "BUG: tcp_main_loop:"
" CONN_NEW: receive_fd "
"failed\n");
}else{
LOG(L_CRIT, "BUG: tcp_main_loop:"
" CONN_NEW: to few bytes "
"received (%d)\n", bytes );
}
break; /* try to ignore */
}
/* add tcpconn to the list*/
tcpconn_add(tcpconn);
FD_SET(tcpconn->s, &master_set);
if (maxfd<tcpconn->s) maxfd=tcpconn->s;
/* update the timeout*/
tcpconn->timeout=get_ticks()+TCP_CON_TIMEOUT;
}else{
LOG(L_CRIT, "BUG: tcp_main_loop: null pointer\n");
}
break;
default:
LOG(L_CRIT, "BUG: tcp_main_loop: unknown cmd %d\n",
cmd);
}
}
} /* for */
/* remove old connections */
tcpconn_timeout(&master_set);
}
}
/*! \brief
* handles an io event on one of the watched tcp connections
*
* \param tcpconn - pointer to the tcp_connection for which we have an io ev.
* \param fd_i - index in the fd_array table (needed for delete)
* \return handle_* return convention, but on success it always returns 0
* (because it's one-shot, after a succesfull execution the fd is
* removed from tcp_main's watch fd list and passed to a child =>
* tcp_main is not interested in further io events that might be
* queued for this fd)
*/
inline static int handle_tcpconn_ev(struct tcp_connection* tcpconn, int fd_i,
int event_type)
{
int fd;
int err;
int id;
unsigned int err_len;
if (event_type == IO_WATCH_READ) {
/* pass it to child, so remove it from the io watch list */
LM_DBG("data available on %p %d\n", tcpconn, tcpconn->s);
if (reactor_del_reader(tcpconn->s, fd_i, 0)==-1)
return -1;
tcpconn->flags|=F_CONN_REMOVED;
tcpconn_ref(tcpconn); /* refcnt ++ */
if (send2child(tcpconn,IO_WATCH_READ)<0){
LM_ERR("no children available\n");
id = tcpconn->id;
TCPCONN_LOCK(id);
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
fd=tcpconn->s;
_tcpconn_rm(tcpconn);
close(fd);
}else tcpconn->lifetime=0; /* force expire*/
TCPCONN_UNLOCK(id);
}
return 0; /* we are not interested in possibly queued io events,
the fd was either passed to a child, or closed */
} else {
LM_DBG("connection %p fd %d is now writable\n", tcpconn, tcpconn->s);
/* we received a write event */
if (tcpconn->state==S_CONN_CONNECTING) {
/* we're coming from an async connect & write
* let's see if we connected successfully*/
err_len=sizeof(err);
if (getsockopt(tcpconn->s, SOL_SOCKET, SO_ERROR, &err, &err_len) < 0 || \
err != 0) {
LM_DBG("Failed connection attempt\n");
tcpconn_ref(tcpconn);
reactor_del_all(tcpconn->s, fd_i, IO_FD_CLOSING);
tcpconn->flags|=F_CONN_REMOVED;
tcpconn_destroy(tcpconn);
return 0;
}
/* we successfully connected - further treat this case as if we
* were coming from an async write */
tcpconn->state = S_CONN_OK;
LM_DBG("Successfully completed previous async connect\n");
goto async_write;
} else {
/* we're coming from an async write -
* just pass to child and have it write
* our TCP chunks */
async_write:
/* no more write events for now */
if (reactor_del_writer( tcpconn->s, fd_i, 0)==-1)
return -1;
tcpconn->flags|=F_CONN_REMOVED;
tcpconn_ref(tcpconn); /* refcnt ++ */
if (send2child(tcpconn,IO_WATCH_WRITE)<0){
LM_ERR("no children available\n");
id = tcpconn->id;
TCPCONN_LOCK(id);
tcpconn->refcnt--;
if (tcpconn->refcnt==0){
fd=tcpconn->s;
_tcpconn_rm(tcpconn);
close(fd);
}else tcpconn->lifetime=0; /* force expire*/
TCPCONN_UNLOCK(id);
}
return 0;
}
}
}