/**
* Attach the application call back on each entries of the lbg
Note : the callback must be previously stored in the lbg context
@param lbg_p : context of the load balancing group
retval 0 : success
retval -1 : error
*/
int north_lbg_attach_app_sup_cbk_on_entries(north_lbg_ctx_t *lbg_p)
{
int i;
north_lbg_entry_ctx_t *entry_p;
af_unix_ctx_generic_t *sock_p ;
if (lbg_p->userPollingCallBack == NULL) return 0;
/* configure each stream connection */
entry_p = lbg_p->entry_tb;
for (i = 0; i < lbg_p->nb_entries_conf ; i++,entry_p++) {
/*
** get the reference from idx
*/
sock_p = af_unix_getObjCtx_p(entry_p->sock_ctx_ref);
if (sock_p == NULL)
{
/*
** socket reference is out of range
*/
severe("north_lbg_attach_app_sup_cbk_on_entries socket index %d does not exist",entry_p->sock_ctx_ref);
return -1;
}
af_inet_attach_application_supervision_callback(sock_p,lbg_p->userPollingCallBack, lbg_p->active_standby_mode);
/*
** attach the callabck of the lbg for availability supervision
*/
af_inet_attach_application_availability_callback(sock_p,north_lbg_update_available_state,lbg_p->index);
}
return 0;
}
/**
* Set the lbg entry to use when sending
@param lbg_idx : reference of the load balancing group
@retval none
*/
void north_lbg_set_active_entry(int lbg_idx, int sock_idx_in_lbg)
{
north_lbg_ctx_t *lbg_p;
lbg_p = north_lbg_getObjCtx_p(lbg_idx);
if (lbg_p == NULL)
{
severe("north_lbg_set_active_entry: no such instance %d ",lbg_idx);
return;
}
//info("JPM lbg %d %s Set active %d",lbg_idx,lbg_p->name,sock_idx_in_lbg);
lbg_p->active_lbg_entry = sock_idx_in_lbg;
/*
** When the socket becomes active, stop the polling
*/
if (lbg_p->active_lbg_entry != -1) {
af_unix_ctx_generic_t *sock_p = af_unix_getObjCtx_p(lbg_p->entry_tb[sock_idx_in_lbg].sock_ctx_ref);
if (sock_p == NULL)
{
severe("north_lbg_set_active_entry: no such socket %d in lbg %d entry %d",
lbg_p->entry_tb[sock_idx_in_lbg].sock_ctx_ref,
lbg_idx,sock_idx_in_lbg);
return;
}
sock_p->cnx_supevision.s.check_cnx_rq = 0;
}
return;
}
/**
af_unix_createIndex
create a AF UNIX context given by index
That function tries to allocate a free PDP
context. In case of success, it returns the
index of the Transaction context.
@param : af_unix_ctx_id is the reference of the context
@retval : MS controller reference (if OK)
retval -1 if out of context.
*/
uint32_t af_unix_createIndex(uint32_t af_unix_ctx_id) {
af_unix_ctx_generic_t *p;
/*
** Get the first free context
*/
p = af_unix_getObjCtx_p(af_unix_ctx_id);
if (p == NULL) {
severe( "MS ref out of range: %u", af_unix_ctx_id );
return RUC_NOK;
}
/*
** return an error if the context is not free
*/
if (p->free == FALSE) {
severe( "the context is not free : %u", af_unix_ctx_id );
return RUC_NOK;
}
/*
** reinitilisation of the context
*/
af_unix_ctxInit(p, FALSE);
/*
** remove it for the linked list
*/
af_unix_context_allocated++;
p->free = FALSE;
ruc_objRemove((ruc_obj_desc_t*) p);
return RUC_OK;
}
/**
* Application connect callback
The retcode contains the result of the connect() operation started
in asynchronous mode. That value is one of the following:
- RUC_OK : success
- RUC_NOK : failure(see errnum for details
@param userRef : pointer to a load balancer entry
@param socket_context_ref : index of the socket context
@param retcode : connection status
@param errnum : value of the errno in case of error
@retval none
*/
void north_lbg_connect_cbk (void *userRef,uint32_t socket_context_ref,int retcode,int errnum)
{
north_lbg_entry_ctx_t *entry_p = (north_lbg_entry_ctx_t*)userRef;
uint8_t fake_buf[16];
int len;
int status;
/*
** in case of success starts sending the messages
*/
if (retcode != RUC_OK)
{
// printf("error on connect() %s\n",strerror(errnum));
/*
** restart the delay timer and then retry
*/
north_lbg_entry_start_timer(entry_p,ROZOFS_TMR_GET(TMR_TCP_RECONNECT));
return;
}
/**
* do a fake read to be sure that the socket is up
*/
af_unix_ctx_generic_t *sock_p = af_unix_getObjCtx_p(socket_context_ref);
status = af_unix_recv_stream_sock_recv(sock_p,fake_buf,2,MSG_PEEK,&len);
switch(status)
{
case RUC_OK:
case RUC_WOULDBLOCK:
case RUC_PARTIAL:
north_lbg_entry_stop_timer(entry_p);
sock_p->stats.totalUpDownTransition++;
north_lbg_entry_state_change(entry_p,NORTH_LBG_UP);
/*
** attempt the fill the xmit queue of that entry
*/
north_lbg_poll_xmit_queue((north_lbg_ctx_t*)entry_p->parent,entry_p,NORTH_LBG_MAX_XMIT_ON_UP_TRANSITION+2);
// entry_p->state = NORTH_LBG_UP ;
// entry_p->stats.totalUpDownTransition++;
// printf("Successful reconnection!!!\n");
return;
case RUC_DISC:
default:
/*
** still deconnected-> remove from socket controller and restart the timer
*/
af_unix_disconnect_from_socketCtrl(socket_context_ref);
north_lbg_entry_start_timer(entry_p,ROZOFS_TMR_GET(TMR_TCP_RECONNECT));
}
}
/**
delete a AF_UNIX context
That function is intended to be called when
a Transaction context is deleted. It returns the
Transaction context to the free list. The delete
procedure of the MS automaton and
controller are called by that service.
If the Transaction context is out of limit, and
error is returned.
@param : MS Index
@retval : RUC_OK : context has been deleted
@retval RUC_NOK : out of limit index.
*/
uint32_t af_unix_free_from_idx(uint32_t af_unix_ctx_id) {
af_unix_ctx_generic_t *p;
if (af_unix_ctx_id >= af_unix_context_count) {
/*
** index is out of limits
*/
return RUC_NOK;
}
/*
** get the reference from idx
*/
p = af_unix_getObjCtx_p(af_unix_ctx_id);
/*
** remove the xmit block
*/
// ruc_objRemove((ruc_obj_desc_t *)&p->xmitCtx);
/*
** Take care of the current index of the show
*/
if (next_display_af_unix_ctx == (ruc_obj_desc_t*)p) {
ruc_objGetNext(&af_unix_context_activeListHead.link, &next_display_af_unix_ctx);
}
/*
** remove it from the active list
*/
ruc_objRemove((ruc_obj_desc_t*) p);
/*
** insert it in the free list
*/
af_unix_context_allocated--;
p->free = TRUE;
ruc_objInsertTail((ruc_obj_desc_t*) af_unix_context_freeListHead,
(ruc_obj_desc_t*) p);
return RUC_OK;
}
/**
* API that provide the current state of a load balancing Group
@param lbg_idx : index of the load balancing group
@retval none
*/
void north_lbg_update_available_state(uint32_t lbg_idx)
{
north_lbg_ctx_t *lbg_p;
lbg_p = north_lbg_getObjCtx_p(lbg_idx);
if (lbg_p == NULL)
{
warning("north_lbg_update_available_state: no such instance %d ",lbg_idx);
return;
}
if (lbg_p->userPollingCallBack == NULL) return;
int i;
north_lbg_entry_ctx_t *entry_p;
af_unix_ctx_generic_t *sock_p ;
/* configure each stream connection */
entry_p = lbg_p->entry_tb;
for (i = 0; i < lbg_p->nb_entries_conf ; i++,entry_p++) {
/*
** get the reference from idx
*/
sock_p = af_unix_getObjCtx_p(entry_p->sock_ctx_ref);
if (sock_p == NULL)
{
continue;
}
if (sock_p->cnx_availability_state == AF_UNIX_CNX_AVAILABLE)
{
lbg_p->available_state = 1;
/*
** for the future when we will move the supervision contexts in the load balancer
** we change the state of the global availability of the LBG to STORCLI_LBG_RUNNING
** Up to now all that code is in storcli.
*/
return;
}
}
lbg_p->available_state = 0;
}
/**
* Get the IP@ of active entry of the lbg
@param lbg_idx : reference of the load balancing group
@retval != 0 IP@ of the active export
@retval == 0 no IP@
*/
uint32_t north_lbg_get_remote_ip_address(int lbg_idx)
{
north_lbg_ctx_t *lbg_p;
lbg_p = north_lbg_getObjCtx_p(lbg_idx);
if (lbg_p == NULL)
{
return 0;
}
if (lbg_p->active_standby_mode < 0) return 0;
/*
** When the socket becomes active, stop the polling
*/
if (lbg_p->active_lbg_entry != -1) {
af_unix_ctx_generic_t *sock_p = af_unix_getObjCtx_p(lbg_p->entry_tb[lbg_p->active_lbg_entry].sock_ctx_ref);
if (sock_p == NULL)
{
return 0;
}
return (sock_p->remote_ipaddr_host);
}
return 0;
}
int north_lbg_send_from_shaper(int lbg_idx,void *buf_p)
{
north_lbg_ctx_t *lbg_p;
int entry_idx;
int ret = 0;
/*
** remove the buffer from any pending list
*/
ruc_objRemove((ruc_obj_desc_t*)buf_p);
lbg_p = north_lbg_getObjCtx_p(lbg_idx);
if (lbg_p == NULL)
{
severe("north_lbg_send: no such instance %d ",lbg_idx);
return -1;
}
/*
** OK there is at least one entry that is free, so get the next valid entry
*/
reloop:
entry_idx = north_lbg_get_next_valid_entry(lbg_p);
if (entry_idx < 0)
{
/*
** that situation must not occur since there is at leat one entry that is UP!!!!
*/
// RUC_WARNING(-1);
return -1;
}
/*
** That's fine, get the pointer to the entry in order to get its socket context reference
*/
north_lbg_entry_ctx_t *entry_p = &lbg_p->entry_tb[entry_idx];
NORTH_LBG_START_PROF((&entry_p->stats));
ret = af_unix_generic_send_stream_with_idx(entry_p->sock_ctx_ref,buf_p);
NORTH_LBG_STOP_PROF((&entry_p->stats));
if (ret == 0)
{
/*
** set the timer to supervise the connection (it only affects client connections)
*/
if (lbg_p->userPollingCallBack != NULL)
{
af_unix_ctx_generic_t *this = af_unix_getObjCtx_p(entry_p->sock_ctx_ref);
af_inet_enable_cnx_supervision(this);
af_inet_set_cnx_tmo(this,lbg_p->tmo_supervision_in_sec*10*5);
}
lbg_p->stats.totalXmit++;
entry_p->stats.totalXmit++;
return 0;
}
/*
** retry on a next entry
** we might need to update the retry counter of the buffer ??
*/
lbg_p->stats.totalXmitError++;
goto reloop;
return 0;
}
/**
* create a north load balancing object with AF_INET
@param lbg_idx : reference of the load balancing group
@param buf_p: pointer to the buffer to send
retval 0 : success
retval -1 : error
*/
int north_lbg_send(int lbg_idx,void *buf_p)
{
north_lbg_ctx_t *lbg_p;
int entry_idx;
int ret = 0;
/*
** remove the buffer from any pending list
*/
ruc_objRemove((ruc_obj_desc_t*)buf_p);
lbg_p = north_lbg_getObjCtx_p(lbg_idx);
if (lbg_p == NULL)
{
severe("north_lbg_send: no such instance %d ",lbg_idx);
return -1;
}
if ((lbg_p->state == NORTH_LBG_SHUTTING_DOWN) || (lbg_p->free == TRUE))
{
return -1;
}
reloop:
/*
** we have the context, search for a valid entry
*/
if (lbg_p->state == NORTH_LBG_DOWN)
{
/*
** Insert the buffer in the global pending list of the load balancing froup
*/
ruc_objInsertTail((ruc_obj_desc_t*)&lbg_p->xmitList[0],(ruc_obj_desc_t*)buf_p);
/*
** update statistics
*/
lbg_p->stats.xmitQueuelen++;
return 0;
}
/*
** check if the main queue is empty if the queue is not empty just queue our message
** at the tail of the load balancer main queue
*/
if (!ruc_objIsEmptyList((ruc_obj_desc_t*)&lbg_p->xmitList[0]))
{
/*
** queue the message at the tail
*/
ruc_objInsertTail((ruc_obj_desc_t*)&lbg_p->xmitList[0],(ruc_obj_desc_t*)buf_p);
/*
** update statistics
*/
lbg_p->stats.xmitQueuelen++;
return 0;
}
/*
** Check whether this LBG is in active/standby mode
** in this case send the message on the active connection
*/
if (lbg_p->active_standby_mode == 1) {
/*
** No actibve entry elected. The LBG should nbe down !!!
*/
if (lbg_p->active_lbg_entry < 0) {
/*
** queue the message at the tail
*/
ruc_objInsertTail((ruc_obj_desc_t*)&lbg_p->xmitList[0],(ruc_obj_desc_t*)buf_p);
/*
** update statistics
*/
lbg_p->stats.xmitQueuelen++;
return 0;
}
/*
** Get the active socket to send on
*/
entry_idx = lbg_p->active_lbg_entry;
}
else {
/*
** OK there is at least one entry that is free, so get the next valid entry
*/
entry_idx = north_lbg_get_next_valid_entry(lbg_p);
if (entry_idx < 0)
{
/*
** that situation must not occur since there is at leat one entry that is UP!!!!
*/
// RUC_WARNING(-1);
return -1;
}
}
/*
** That's fine, get the pointer to the entry in order to get its socket context reference
*/
north_lbg_entry_ctx_t *entry_p = &lbg_p->entry_tb[entry_idx];
NORTH_LBG_START_PROF((&entry_p->stats));
ret = af_unix_generic_send_stream_with_idx(entry_p->sock_ctx_ref,buf_p);
NORTH_LBG_STOP_PROF((&entry_p->stats));
if (ret == 0)
{
/*
** set the timer to supervise the connection (it only affects client connections)
*/
if (lbg_p->userPollingCallBack != NULL)
{
af_unix_ctx_generic_t *this = af_unix_getObjCtx_p(entry_p->sock_ctx_ref);
af_inet_enable_cnx_supervision(this);
af_inet_set_cnx_tmo(this,lbg_p->tmo_supervision_in_sec*10*5);
}
lbg_p->stats.totalXmit++;
entry_p->stats.totalXmit++;
return 0;
}
/*
** retry on a next entry
** we might need to update the retry counter of the buffer ??
*/
lbg_p->stats.totalXmitError++;
goto reloop;
return 0;
}
void north_lbg_entries_debug_show(uint32_t tcpRef, void *bufRef) {
char *pChar = uma_dbg_get_buffer();
int siocinq_value;
int siocoutq_value;
{
north_lbg_ctx_t *lbg_p;
ruc_obj_desc_t *pnext;
int i;
pChar += sprintf(pChar, " LBG Name | lbg_id | idx | sock | state | rdy | Queue | Cnx Attpts | Xmit Attpts | Recv count | Recv-Q | Send-Q |\n");
pChar += sprintf(pChar, "--------------------------+--------+------+------+------------+-----+-----------+------------+-------------+-------------+-----------+-----------+\n");
pnext = (ruc_obj_desc_t*) NULL;
while ((lbg_p = (north_lbg_ctx_t*) ruc_objGetNext((ruc_obj_desc_t*) & north_lbg_context_activeListHead,
&pnext))
!= (north_lbg_ctx_t*) NULL) {
if (lbg_p->nb_entries_conf == 0) continue;
north_lbg_entry_ctx_t *entry_p = lbg_p->entry_tb;
af_unix_ctx_generic_t *sock_p;
for (i = 0; i < lbg_p->nb_entries_conf; i++, entry_p++) {
sock_p = af_unix_getObjCtx_p(entry_p->sock_ctx_ref);
pChar += sprintf(pChar, " %-24s |", lbg_p->name);
pChar += sprintf(pChar, " %4d |", lbg_p->index);
if (lbg_p->active_lbg_entry == i) {
pChar += sprintf(pChar, " %2d *|", i);
}
else {
pChar += sprintf(pChar, " %2d |", i);
}
pChar += sprintf(pChar, " %4d |", sock_p->socketRef); /** socket */
pChar += sprintf(pChar, " %s |", lbg_north_state2String(entry_p->state));
while(1)
{
af_unix_ctx_generic_t *sock_p ;
sock_p = af_unix_getObjCtx_p(entry_p->sock_ctx_ref);
if (sock_p == NULL)
{
pChar += sprintf(pChar, " %s |","???");
break;
}
pChar += sprintf(pChar, " %s |",(sock_p->cnx_availability_state == AF_UNIX_CNX_AVAILABLE )?"YES":" NO");
break;
}
pChar += sprintf(pChar, " %s |", ruc_objIsEmptyList((ruc_obj_desc_t*) & entry_p->xmitList) ? " EMPTY" : "NON EMPTY");
pChar += sprintf(pChar, " %10llu |", (unsigned long long int) entry_p->stats.totalConnectAttempts);
pChar += sprintf(pChar, " %10llu |", (unsigned long long int) entry_p->stats.totalXmit);
pChar += sprintf(pChar, " %10llu |", (unsigned long long int) entry_p->stats.totalRecv);
siocinq_value = 0;
siocoutq_value = 0;
if (sock_p->socketRef)
{
ioctl(sock_p->socketRef,SIOCINQ,&siocinq_value);
ioctl(sock_p->socketRef,SIOCOUTQ,&siocoutq_value);
}
pChar += sprintf(pChar, " %8d |", siocinq_value);
pChar += sprintf(pChar, " %8d |\n",siocoutq_value);
}
}
}
uma_dbg_send(tcpRef, bufRef, TRUE, uma_dbg_get_buffer());
}
