/**
* Application connect callback
The retcode contains the result of the connect() operation started
in asynchronous mode. That value is one of the following:
- RUC_OK : success
- RUC_NOK : failure(see errnum for details
@param userRef : pointer to a load balancer entry
@param socket_context_ref : index of the socket context
@param retcode : connection status
@param errnum : value of the errno in case of error
@retval none
*/
void north_lbg_connect_cbk (void *userRef,uint32_t socket_context_ref,int retcode,int errnum)
{
north_lbg_entry_ctx_t *entry_p = (north_lbg_entry_ctx_t*)userRef;
uint8_t fake_buf[16];
int len;
int status;
/*
** in case of success starts sending the messages
*/
if (retcode != RUC_OK)
{
// printf("error on connect() %s\n",strerror(errnum));
/*
** restart the delay timer and then retry
*/
north_lbg_entry_start_timer(entry_p,ROZOFS_TMR_GET(TMR_TCP_RECONNECT));
return;
}
/**
* do a fake read to be sure that the socket is up
*/
af_unix_ctx_generic_t *sock_p = af_unix_getObjCtx_p(socket_context_ref);
status = af_unix_recv_stream_sock_recv(sock_p,fake_buf,2,MSG_PEEK,&len);
switch(status)
{
case RUC_OK:
case RUC_WOULDBLOCK:
case RUC_PARTIAL:
north_lbg_entry_stop_timer(entry_p);
sock_p->stats.totalUpDownTransition++;
north_lbg_entry_state_change(entry_p,NORTH_LBG_UP);
/*
** attempt the fill the xmit queue of that entry
*/
north_lbg_poll_xmit_queue((north_lbg_ctx_t*)entry_p->parent,entry_p,NORTH_LBG_MAX_XMIT_ON_UP_TRANSITION+2);
// entry_p->state = NORTH_LBG_UP ;
// entry_p->stats.totalUpDownTransition++;
// printf("Successful reconnection!!!\n");
return;
case RUC_DISC:
default:
/*
** still deconnected-> remove from socket controller and restart the timer
*/
af_unix_disconnect_from_socketCtrl(socket_context_ref);
north_lbg_entry_start_timer(entry_p,ROZOFS_TMR_GET(TMR_TCP_RECONNECT));
}
}
void north_lbg_entry_timeout_CBK (void *opaque)
{
int ret;
north_lbg_entry_ctx_t *entry_p = (north_lbg_entry_ctx_t*)opaque;
// entry_p->rpc_guard_timer_flg = TRUE;
entry_p->stats.totalConnectAttempts++;
ret = af_unix_sock_client_reconnect(entry_p->sock_ctx_ref);
if (ret < 0)
{
// printf("north_lbg_entry_timeout_CBK-->fatal error on reconnect\n");
/*
** restart the timer
*/
north_lbg_entry_start_timer(entry_p,ROZOFS_TMR_GET(TMR_TCP_RECONNECT));
}
}
/**
* create a north load balancing object with AF_INET
@param @name : name of the load balancer
@param basename_p : Base name of the remote sunpath
@param @family of the load balancer
@retval >= reference of the load balancer object
@retval < 0 error (out of context ??)
*/
int north_lbg_create_af_inet(char *name,
uint32_t src_ipaddr_host,
uint16_t src_port_host,
north_remote_ip_list_t *remote_ip_p,
int family,int nb_instances,af_unix_socket_conf_t *usr_conf_p)
{
north_lbg_ctx_t *lbg_p;
int i;
north_lbg_entry_ctx_t *entry_p;
af_unix_socket_conf_t *conf_p;
if (nb_instances == 0)
{
/*
** no instances!!
*/
warning("north_lbg_create_af_inet: no instances");
return -1;
}
if (nb_instances >= NORTH__LBG_MAX_ENTRY)
{
/*
** to many instances!!
*/
warning("north_lbg_create_af_inet: to many instances : %d max %d ",nb_instances,NORTH__LBG_MAX_ENTRY);
return -1;
}
/*
** allocate a load balancer context
*/
lbg_p = north_lbg_alloc();
if (lbg_p == NULL)
{
/*
** out of context
*/
fatal("north_lbg_create_af_inet: out of load balancing group context");
return -1;
}
/*
** save the configuration of the lbg
*/
conf_p = &lbg_p->lbg_conf;
memcpy(conf_p,usr_conf_p,sizeof(af_unix_socket_conf_t));
lbg_p->family = family;
lbg_p->nb_entries_conf = nb_instances;
strcpy(lbg_p->name,name);
/*
** install the callbacks of the load balancer
*/
/*
** install the load balancer callback but keep in the context the
** user callback for deconnection
*/
lbg_p->userDiscCallBack = conf_p->userDiscCallBack;
conf_p->userDiscCallBack = north_lbg_userDiscCallBack;
lbg_p->userRcvCallBack = conf_p->userRcvCallBack;
conf_p->userRcvCallBack = north_lbg_userRecvCallBack;
conf_p->userConnectCallBack = north_lbg_connect_cbk;
conf_p->userXmitDoneCallBack = north_lbg_userXmiDoneCallBack;
entry_p = lbg_p->entry_tb;
for (i = 0; i < nb_instances ; i++,entry_p++,remote_ip_p++)
{
conf_p->userRef = entry_p;
entry_p->sock_ctx_ref = af_inet_sock_client_create(name,
src_ipaddr_host,
src_port_host,
remote_ip_p->remote_ipaddr_host,
remote_ip_p->remote_port_host,
conf_p);
if (entry_p->sock_ctx_ref >= 0)
{
north_lbg_entry_start_timer(entry_p,ROZOFS_TMR_GET(TMR_TCP_RECONNECT));
north_lbg_entry_state_change(entry_p,NORTH_LBG_DOWN);
// entry_p->state = NORTH_LBG_DOWN;
}
}
/*
** attach the application callback if any is declared
*/
north_lbg_attach_app_sup_cbk_on_entries(lbg_p);
return (lbg_p->index);
}
void north_lbg_userDiscCallBack(void *userRef,uint32_t socket_context_ref,void *bufRef,int err_no)
{
// int len;
int ret;
int8_t retry_counter;
north_lbg_entry_ctx_t *entry_p = (north_lbg_entry_ctx_t*)userRef;
north_lbg_ctx_t *lbg_p = (north_lbg_ctx_t*)entry_p->parent;
ruc_obj_desc_t *pnext = (ruc_obj_desc_t*)NULL;
int up2down_transition = 0;
/*
** change the state to DOWN
*/
if (entry_p->state != NORTH_LBG_DOWN)
{
north_lbg_entry_state_change(entry_p,NORTH_LBG_DOWN);
warning("north_lbg_userDiscCallBack : LBG %s entry %d \n",lbg_p->name, entry_p->index);
up2down_transition = 1;
}
/*
**get the pointer to the destination stored in the buffer
*/
while (bufRef != NULL)
{
/*
** get the retry counter of the buffer
*/
retry_counter = ruc_buf_get_retryCounter(bufRef);
if ((lbg_p->state == NORTH_LBG_DOWN) && (lbg_p->rechain_when_lbg_gets_down == 1)) {
/*
** The lbg is down but we re-send on it hoping it will comme back up soon
*/
if (retry_counter < lbg_p->nb_entries_conf) {
retry_counter +=1;
ruc_buf_set_retryCounter(bufRef,retry_counter);
/*
** resend by selecting a new destination
*/
lbg_p->stats.totalXmitRetries++;
//info("rechain on LBG %d",lbg_p->index);
north_lbg_send(lbg_p->index,bufRef);
break;
}
}
// if ((retry_counter >= NORTH_LBG_MAX_RETRY) || (lbg_p->state == NORTH_LBG_DOWN))
if ((retry_counter >= lbg_p->nb_entries_conf) || (lbg_p->state == NORTH_LBG_DOWN))
{
/*
** inform the application or release the buffer
*/
lbg_p->stats.totalXmitAborts++;
if (lbg_p->userDiscCallBack!= NULL)
{
(lbg_p->userDiscCallBack)(NULL,lbg_p->index,bufRef, err_no);
break;
}
/*
** release the buffer
*/
//#warning Need to check the in_use counter of the buffer before the release
ruc_buf_freeBuffer(bufRef);
break;
}
/*
** the lbg is still up and there the retry count is not exhausted
*/
retry_counter +=1;
ruc_buf_set_retryCounter(bufRef,retry_counter);
/*
** resend by selecting a new destination
*/
lbg_p->stats.totalXmitRetries++;
north_lbg_send(lbg_p->index,bufRef);
break;
}
/*
** OK, now go the buffer that might be queued on that entry and do the same
*/
while ((bufRef = (void*) ruc_objGetNext((ruc_obj_desc_t*)&entry_p->xmitList,
&pnext))
!=NULL)
{
/*
** remove it from the list because it might be queued afterwards on a new queue
*/
ruc_objRemove((ruc_obj_desc_t*)bufRef);
while (1)
{
/*
** get the retry counter of the buffer
*/
retry_counter = ruc_buf_get_retryCounter(bufRef);
if ((lbg_p->state == NORTH_LBG_DOWN) && (lbg_p->rechain_when_lbg_gets_down == 1)) {
/*
** The lbg is down but we re-send on it hoping it will comme back up soon
*/
if (retry_counter < lbg_p->nb_entries_conf) {
retry_counter +=1;
ruc_buf_set_retryCounter(bufRef,retry_counter);
/*
** resend by selecting a new destination
*/
lbg_p->stats.totalXmitRetries++;
#
//info("rechain on LBG %d",lbg_p->index);
north_lbg_send(lbg_p->index,bufRef);
break;
}
}
// if ((retry_counter >= NORTH_LBG_MAX_RETRY) || (lbg_p->state == NORTH_LBG_DOWN))
if ((retry_counter >= lbg_p->nb_entries_conf) || (lbg_p->state == NORTH_LBG_DOWN))
{
/*
** inform the application or release the buffer
*/
lbg_p->stats.totalXmitAborts++;
if (lbg_p->userDiscCallBack!= NULL)
{
(lbg_p->userDiscCallBack)(NULL,lbg_p->index,bufRef, err_no);
break;
}
/*
** release the buffer
*/
ruc_buf_freeBuffer(bufRef);
break;
}
/*
** the lbg is still up and there the retry count is not exhausted
*/
retry_counter +=1;
ruc_buf_set_retryCounter(bufRef,retry_counter);
/*
** resend by selecting a new destination
*/
lbg_p->stats.totalXmitRetries++;
north_lbg_send(lbg_p->index,bufRef);
break;
}
}
/*
** do the reconnect on UP 2 DOWN transition only
*/
if (up2down_transition)
{
entry_p->stats.totalConnectAttempts++;
ret = af_unix_sock_client_reconnect(socket_context_ref);
if (ret < 0)
{
// printf("north_lbg_userDiscCallBack->fatal error on reconnect\n");
north_lbg_entry_start_timer(entry_p,ROZOFS_TMR_GET(TMR_TCP_FIRST_RECONNECT));
}
}
}
/**
* API to configure a load balancing group.
The load balancing group must have been created previously with north_lbg_create_no_conf()
@param none
@retval >= reference of the load balancer object
@retval < 0 error (out of context ??)
*/
int north_lbg_configure_af_inet(int lbg_idx,char *name,
uint32_t src_ipaddr_host,
uint16_t src_port_host,
north_remote_ip_list_t *remote_ip_p,
int family,int nb_instances,af_unix_socket_conf_t *usr_conf_p, int local)
{
north_lbg_ctx_t *lbg_p;
int i;
north_lbg_entry_ctx_t *entry_p;
af_unix_socket_conf_t *conf_p;
lbg_p = north_lbg_getObjCtx_p(lbg_idx);
if (lbg_p == NULL)
{
warning("north_lbg_configure_af_inet: no such instance %d ",lbg_idx);
return -1;
}
lbg_p->local = local;
conf_p = &lbg_p->lbg_conf;
memcpy(conf_p,usr_conf_p,sizeof(af_unix_socket_conf_t));
if (lbg_p->state != NORTH_LBG_DEPENDENCY)
{
warning("north_lbg_configure_af_inet: unexpected state %d ",lbg_p->state);
return -1;
}
if (nb_instances == 0)
{
/*
** no instances!!
*/
warning("north_lbg_configure_af_inet: no instance for lbg %d ",lbg_idx);
return -1;
}
if (nb_instances >= NORTH__LBG_MAX_ENTRY)
{
/*
** to many instances!!
*/
warning("north_lbg_configure_af_inet: too many instances (%d max %d) for lbg %d ",nb_instances,NORTH__LBG_MAX_ENTRY,lbg_idx);
return -1;
}
/*
** restore the init state
*/
lbg_p->state = NORTH_LBG_DOWN;
lbg_p->family = family;
lbg_p->nb_entries_conf = nb_instances;
strcpy(lbg_p->name,name);
/*
** install the callbacks of the load balancer
*/
/*
** install the load balancer callback but keep in the context the
** user callback for deconnection
*/
lbg_p->userDiscCallBack = conf_p->userDiscCallBack;
conf_p->userDiscCallBack = north_lbg_userDiscCallBack;
lbg_p->userRcvCallBack = conf_p->userRcvCallBack;
conf_p->userRcvCallBack = north_lbg_userRecvCallBack;
conf_p->userConnectCallBack = north_lbg_connect_cbk;
conf_p->userXmitDoneCallBack = north_lbg_userXmiDoneCallBack;
entry_p = lbg_p->entry_tb;
for (i = 0; i < nb_instances ; i++,entry_p++,remote_ip_p++)
{
conf_p->userRef = entry_p;
entry_p->sock_ctx_ref = af_inet_sock_client_create(name,
src_ipaddr_host,
src_port_host,
remote_ip_p->remote_ipaddr_host,
remote_ip_p->remote_port_host,
conf_p);
if (entry_p->sock_ctx_ref >= 0)
{
north_lbg_entry_start_timer(entry_p,ROZOFS_TMR_GET(TMR_TCP_RECONNECT));
north_lbg_entry_state_change(entry_p,NORTH_LBG_DOWN);
// entry_p->state = NORTH_LBG_DOWN;
}
}
/*
** attach the application callback if any is declared
*/
north_lbg_attach_app_sup_cbk_on_entries(lbg_p);
return (lbg_p->index);
}
