void north_lbg_poll_xmit_queue(north_lbg_ctx_t *lbg_p, north_lbg_entry_ctx_t *entry_p,int xmit_credit)
{
ruc_obj_desc_t *pnext;
int ret;
int i;
void *buf_p;
pnext = (ruc_obj_desc_t*)NULL;
for (i = 0; i < xmit_credit; i++)
{
while ((buf_p = (void*) ruc_objGetNext((ruc_obj_desc_t*)&lbg_p->xmitList[0],
&pnext))
!=NULL)
{
ruc_objRemove((ruc_obj_desc_t*)buf_p);
lbg_p->stats.xmitQueuelen--;
ret = af_unix_generic_send_stream_with_idx(entry_p->sock_ctx_ref,buf_p);
if (ret == 0)
{
lbg_p->stats.totalXmit++;
entry_p->stats.totalXmit++;
continue;
}
/*
** there is an error, the interface is down again, so requeue the buffer at the head
** do we need to udpate the retry counter of the buffer ??
*/
lbg_p->stats.totalXmitError++;
lbg_p->stats.xmitQueuelen++;
ruc_objInsert(&lbg_p->xmitList[0],(ruc_obj_desc_t*) buf_p);
return;
}
return;
}
}
/**
Processes a disk response
Called from the socket controller when there is a response from a disk thread
the response is either for a disk read or write
@param msg: pointer to disk response message
@retval :none
*/
void af_unix_disk_response(storio_disk_thread_msg_t *msg)
{
storio_disk_thread_request_e opcode;
rozorpc_srv_ctx_t * rpcCtx;
int ret;
uint64_t tic, toc;
struct timeval tv;
rpcCtx = msg->rpcCtx;
opcode = msg->opcode;
tic = msg->timeStart;
switch (opcode) {
case STORIO_DISK_THREAD_READ:
STOP_PROFILING_IO(read,msg->size);
update_read_detailed_counters(toc - tic);
break;
case STORIO_DISK_THREAD_WRITE:
STOP_PROFILING_IO(write,msg->size);
update_write_detailed_counters(toc - tic);
break;
case STORIO_DISK_THREAD_TRUNCATE:
STOP_PROFILING(truncate);
break;
default:
severe("Unexpected opcode %d", opcode);
}
/*
** send the response towards the storcli process that initiates the disk operation
*/
ret = af_unix_generic_send_stream_with_idx((int)rpcCtx->socketRef,rpcCtx->xmitBuf);
if (ret == 0) {
/**
* success so remove the reference of the xmit buffer since it is up to the called
* function to release it
*/
ROZORPC_SRV_STATS(ROZORPC_SRV_SEND);
rpcCtx->xmitBuf = NULL;
}
else {
ROZORPC_SRV_STATS(ROZORPC_SRV_SEND_ERROR);
}
rozorpc_srv_release_context(rpcCtx);
}
int north_lbg_send_from_shaper(int lbg_idx,void *buf_p)
{
north_lbg_ctx_t *lbg_p;
int entry_idx;
int ret = 0;
/*
** remove the buffer from any pending list
*/
ruc_objRemove((ruc_obj_desc_t*)buf_p);
lbg_p = north_lbg_getObjCtx_p(lbg_idx);
if (lbg_p == NULL)
{
severe("north_lbg_send: no such instance %d ",lbg_idx);
return -1;
}
/*
** OK there is at least one entry that is free, so get the next valid entry
*/
reloop:
entry_idx = north_lbg_get_next_valid_entry(lbg_p);
if (entry_idx < 0)
{
/*
** that situation must not occur since there is at leat one entry that is UP!!!!
*/
// RUC_WARNING(-1);
return -1;
}
/*
** That's fine, get the pointer to the entry in order to get its socket context reference
*/
north_lbg_entry_ctx_t *entry_p = &lbg_p->entry_tb[entry_idx];
NORTH_LBG_START_PROF((&entry_p->stats));
ret = af_unix_generic_send_stream_with_idx(entry_p->sock_ctx_ref,buf_p);
NORTH_LBG_STOP_PROF((&entry_p->stats));
if (ret == 0)
{
/*
** set the timer to supervise the connection (it only affects client connections)
*/
if (lbg_p->userPollingCallBack != NULL)
{
af_unix_ctx_generic_t *this = af_unix_getObjCtx_p(entry_p->sock_ctx_ref);
af_inet_enable_cnx_supervision(this);
af_inet_set_cnx_tmo(this,lbg_p->tmo_supervision_in_sec*10*5);
}
lbg_p->stats.totalXmit++;
entry_p->stats.totalXmit++;
return 0;
}
/*
** retry on a next entry
** we might need to update the retry counter of the buffer ??
*/
lbg_p->stats.totalXmitError++;
goto reloop;
return 0;
}
/**
* create a north load balancing object with AF_INET
@param lbg_idx : reference of the load balancing group
@param buf_p: pointer to the buffer to send
retval 0 : success
retval -1 : error
*/
int north_lbg_send(int lbg_idx,void *buf_p)
{
north_lbg_ctx_t *lbg_p;
int entry_idx;
int ret = 0;
/*
** remove the buffer from any pending list
*/
ruc_objRemove((ruc_obj_desc_t*)buf_p);
lbg_p = north_lbg_getObjCtx_p(lbg_idx);
if (lbg_p == NULL)
{
severe("north_lbg_send: no such instance %d ",lbg_idx);
return -1;
}
if ((lbg_p->state == NORTH_LBG_SHUTTING_DOWN) || (lbg_p->free == TRUE))
{
return -1;
}
reloop:
/*
** we have the context, search for a valid entry
*/
if (lbg_p->state == NORTH_LBG_DOWN)
{
/*
** Insert the buffer in the global pending list of the load balancing froup
*/
ruc_objInsertTail((ruc_obj_desc_t*)&lbg_p->xmitList[0],(ruc_obj_desc_t*)buf_p);
/*
** update statistics
*/
lbg_p->stats.xmitQueuelen++;
return 0;
}
/*
** check if the main queue is empty if the queue is not empty just queue our message
** at the tail of the load balancer main queue
*/
if (!ruc_objIsEmptyList((ruc_obj_desc_t*)&lbg_p->xmitList[0]))
{
/*
** queue the message at the tail
*/
ruc_objInsertTail((ruc_obj_desc_t*)&lbg_p->xmitList[0],(ruc_obj_desc_t*)buf_p);
/*
** update statistics
*/
lbg_p->stats.xmitQueuelen++;
return 0;
}
/*
** Check whether this LBG is in active/standby mode
** in this case send the message on the active connection
*/
if (lbg_p->active_standby_mode == 1) {
/*
** No actibve entry elected. The LBG should nbe down !!!
*/
if (lbg_p->active_lbg_entry < 0) {
/*
** queue the message at the tail
*/
ruc_objInsertTail((ruc_obj_desc_t*)&lbg_p->xmitList[0],(ruc_obj_desc_t*)buf_p);
/*
** update statistics
*/
lbg_p->stats.xmitQueuelen++;
return 0;
}
/*
** Get the active socket to send on
*/
entry_idx = lbg_p->active_lbg_entry;
}
else {
/*
** OK there is at least one entry that is free, so get the next valid entry
*/
entry_idx = north_lbg_get_next_valid_entry(lbg_p);
if (entry_idx < 0)
{
/*
** that situation must not occur since there is at leat one entry that is UP!!!!
*/
// RUC_WARNING(-1);
return -1;
}
}
/*
** That's fine, get the pointer to the entry in order to get its socket context reference
*/
north_lbg_entry_ctx_t *entry_p = &lbg_p->entry_tb[entry_idx];
NORTH_LBG_START_PROF((&entry_p->stats));
ret = af_unix_generic_send_stream_with_idx(entry_p->sock_ctx_ref,buf_p);
NORTH_LBG_STOP_PROF((&entry_p->stats));
if (ret == 0)
{
/*
** set the timer to supervise the connection (it only affects client connections)
*/
if (lbg_p->userPollingCallBack != NULL)
{
af_unix_ctx_generic_t *this = af_unix_getObjCtx_p(entry_p->sock_ctx_ref);
af_inet_enable_cnx_supervision(this);
af_inet_set_cnx_tmo(this,lbg_p->tmo_supervision_in_sec*10*5);
}
lbg_p->stats.totalXmit++;
entry_p->stats.totalXmit++;
return 0;
}
/*
** retry on a next entry
** we might need to update the retry counter of the buffer ??
*/
lbg_p->stats.totalXmitError++;
goto reloop;
return 0;
}
/**
* send a rpc reply: the encoding function MUST be found in xdr_result
of the gateway context
It is assumed that the xmitBuf MUST be found in xmitBuf field
In case of a success it is up to the called function to release the xmit buffer
@param p : pointer to the root transaction context used for the read
@param arg_ret : returned argument to encode
@retval none
*/
void storaged_srv_forward_read_success (rozorpc_srv_ctx_t *p,sp_read_ret_t * arg_ret)
{
int ret;
uint8_t *pbuf; /* pointer to the part that follows the header length */
uint32_t *header_len_p; /* pointer to the array that contains the length of the rpc message*/
XDR xdrs;
int len;
int cur_len;
if (p->xmitBuf == NULL)
{
ROZORPC_SRV_STATS(ROZORPC_SRV_NO_BUFFER_ERROR);
severe("no xmit buffer");
goto error;
}
/*
** create xdr structure on top of the buffer that will be used for sending the response
*/
header_len_p = (uint32_t*)ruc_buf_getPayload(p->xmitBuf);
pbuf = (uint8_t*) (header_len_p+1);
len = (int)ruc_buf_getMaxPayloadLen(p->xmitBuf);
len -= sizeof(uint32_t);
xdrmem_create(&xdrs,(char*)pbuf,len,XDR_ENCODE);
/*
** just encode the header
*/
if (rozofs_encode_rpc_reply(&xdrs,(xdrproc_t)NULL,(caddr_t)NULL,p->src_transaction_id) != TRUE)
{
ROZORPC_SRV_STATS(ROZORPC_SRV_ENCODING_ERROR);
severe("rpc reply encoding error");
goto error;
}
/*
** OK now starts encoding the response
*/
xdr_sp_status_t (&xdrs, &arg_ret->status);
xdr_uint32_t (&xdrs, &arg_ret->sp_read_ret_t_u.rsp.filler);
xdr_uint32_t (&xdrs, &arg_ret->sp_read_ret_t_u.rsp.bins.bins_len);
/*
** skip the bins
*/
cur_len = xdr_getpos(&xdrs) ;
cur_len += arg_ret->sp_read_ret_t_u.rsp.bins.bins_len;
xdr_setpos(&xdrs,cur_len);
/*
** encode the length of the file
*/
xdr_uint64_t (&xdrs, &arg_ret->sp_read_ret_t_u.rsp.file_size);
#if 0 // for future usage with distributed cache
/*
** encode the optim array
*/
xdr_bytes (&xdrs, (char **)&arg_ret->sp_read_ret_t_u.rsp.optim.optim_val,
(u_int *) &arg_ret->sp_read_ret_t_u.rsp.optim.optim_len, ~0);
#endif
/*
** compute the total length of the message for the rpc header and add 4 bytes more bytes for
** the ruc buffer to take care of the header length of the rpc message.
*/
int total_len = xdr_getpos(&xdrs) ;
*header_len_p = htonl(0x80000000 | total_len);
total_len +=sizeof(uint32_t);
ruc_buf_setPayloadLen(p->xmitBuf,total_len);
/*
** Get the callback for sending back the response:
** A callback is needed since the request for read might be local or remote
*/
ret = af_unix_generic_send_stream_with_idx((int)p->socketRef,p->xmitBuf);
if (ret == 0)
{
/**
* success so remove the reference of the xmit buffer since it is up to the called
* function to release it
*/
ROZORPC_SRV_STATS(ROZORPC_SRV_SEND);
p->xmitBuf = NULL;
}
else
{
ROZORPC_SRV_STATS(ROZORPC_SRV_SEND_ERROR);
}
error:
return;
}
