void rozofs_export_lbg_cnx_polling(af_unix_ctx_generic_t *sock_p)
{
void *xmit_buf = NULL;
int ret;
int timeout = (int)ROZOFS_TMR_GET(TMR_RPC_NULL_PROC_TCP);
af_inet_set_cnx_tmo(sock_p,timeout*10*5);
/*
** attempt to poll
*/
xmit_buf = ruc_buf_getBuffer(ROZOFS_TX_SMALL_TX_POOL);
if (xmit_buf == NULL)
{
return ;
}
ret = rozofs_export_poll_tx(sock_p,EXPORT_PROGRAM,EXPORT_VERSION,EP_NULL,
(xdrproc_t) xdr_void, (caddr_t) NULL,
xmit_buf,
0,
timeout, /** TMO in secs */
rozofs_export_poll_cbk,
(void*)NULL);
if (ret < 0)
{
/*
** direct need to free the xmit buffer
*/
ruc_buf_freeBuffer(xmit_buf);
}
}
/**
*
Callback to allocate a buffer for receiving a rpc message (mainly a RPC response
The service might reject the buffer allocation because the pool runs
out of buffer or because there is no pool with a buffer that is large enough
for receiving the message because of a out of range size.
@param userRef : pointer to a user reference: not used here
@param socket_context_ref: socket context reference
@param len : length of the incoming message
@retval <>NULL pointer to a receive buffer
@retval == NULL no buffer
*/
void * storage_lbg_userRcvAllocBufCallBack(void *userRef, uint32_t socket_context_ref, uint32_t len) {
void *buf=NULL;
/*
** check if a small or a large buffer must be allocated
*/
if (len <= rozofs_storcli_south_small_buf_sz) {
buf = ruc_buf_getBuffer(ROZOFS_STORCLI_SOUTH_SMALL_POOL);
if (buf != NULL) return buf;
}
if (len <= rozofs_storcli_south_large_buf_sz) {
return ruc_buf_getBuffer(ROZOFS_STORCLI_SOUTH_LARGE_POOL);
}
fatal("Out of range received size: %d (max %d)",len,rozofs_storcli_south_large_buf_sz);
return NULL;
}
/**
* Encode the dirty message to invalidate some sections for a server
@param ctx_p : pointer to the dirty management context
@param pool_ref : poll ref to get the buffer from
@param srv_rank : Server to build the message for
@retval the buffer containing the encode message
*/
void * exp_cache_encode_invalidate_sections_msg(exp_cache_dirty_ctx_t * ctx_p,
void * pool_ref,
int srv_rank)
{
gw_invalidate_sections_t * msg = NULL;
void * xmit_buf = NULL;
int ret;
exp_invalidate_type_e build_msg_ret;
/*
** allocate an xmit buffer
*/
xmit_buf = ruc_buf_getBuffer(pool_ref);
if (xmit_buf == NULL) {
severe ("Out of buffer");
return NULL;
}
/*
** Build the structure to encode from the cache data
*/
build_msg_ret = exp_cache_build_invalidate_sections_msg(ctx_p,srv_rank);
/*
** Nothing to send
*/
if ((build_msg_ret == exp_invalidate_nothing)||(build_msg_ret == exp_invalidate_error)) {
ruc_buf_freeBuffer(xmit_buf);
return NULL;
}
if (build_msg_ret == exp_invalidate_too_big) {
/*
** Well let's invalidate everything, since we can not encode the invalidate sections
*/
ruc_buf_freeBuffer(xmit_buf);
return exp_cache_encode_invalidate_all_msg(ctx_p,pool_ref,srv_rank);
}
/*
** Encode the message in the buffer
*/
ret = exp_cache_encode_common(GW_INVALIDATE_SECTIONS,(xdrproc_t) xdr_gw_invalidate_sections_t,msg, xmit_buf);
if (ret != 0) {
/*
** Well let's invalidate everything, since we can not encode the invalidate sections
*/
ruc_buf_freeBuffer(xmit_buf);
return exp_cache_encode_invalidate_all_msg(ctx_p,pool_ref,srv_rank);
}
return xmit_buf;
}
/**
*
Callback to allocate a buffer for receiving a rpc message (mainly a RPC response
The service might reject the buffer allocation because the pool runs
out of buffer or because there is no pool with a buffer that is large enough
for receiving the message because of a out of range size.
@param userRef : pointer to a user reference: not used here
@param socket_context_ref: socket context reference
@param len : length of the incoming message
@retval <>NULL pointer to a receive buffer
@retval == NULL no buffer
*/
void * rozofs_storcli_north_RcvAllocBufCallBack(void *userRef,uint32_t socket_context_ref,uint32_t len)
{
/*
** check if a small or a large buffer must be allocated
*/
if (len > rozofs_storcli_read_write_buf_sz)
{
return NULL;
}
return ruc_buf_getBuffer(storcli_north_buffer_pool_p);
}
/**
* Encode the message to invalidate the whole cache of a server
@param ctx_p : pointer to the dirty management context
@param pool_ref : poll ref to get the buffer from
@param srv_rank : Server to build the message for
@retval the buffer containing the encode message
*/
void * exp_cache_encode_invalidate_all_msg(exp_cache_dirty_ctx_t * ctx_p,
void * pool_ref,
int srv_rank)
{
gw_header_t * msg = NULL;
void * xmit_buf = NULL;
int ret;
/*
** allocate an xmit buffer
*/
xmit_buf = ruc_buf_getBuffer(pool_ref);
if (xmit_buf == NULL) {
severe ("Out of buffer");
return NULL;
}
/*
** Build the structure to encode from the cache data
*/
msg = exp_cache_build_invalidate_all_msg(ctx_p,srv_rank);
if (msg == NULL) {
ruc_buf_freeBuffer(xmit_buf);
return NULL;
}
/*
** Encode the message in the buffer
*/
ret = exp_cache_encode_common(GW_INVALIDATE_ALL,(xdrproc_t) xdr_gw_header_t,msg, xmit_buf);
if (ret != 0) {
ruc_buf_freeBuffer(xmit_buf);
return NULL;
}
/*
** Since everything has been invalidated, let's operate a flip flop on the cache
*/
exp_dirty_active_switch(ctx_p, srv_rank);
return xmit_buf;
}
void sp_read_1_svc_disk_thread(void * pt, rozorpc_srv_ctx_t *req_ctx_p) {
static sp_read_ret_t ret;
START_PROFILING(read);
/*
** allocate a buffer for the response
*/
req_ctx_p->xmitBuf = ruc_buf_getBuffer(storage_xmit_buffer_pool_p);
if (req_ctx_p->xmitBuf == NULL)
{
severe("sp_read_1_svc_disk_thread Out of memory STORAGE_NORTH_LARGE_POOL");
errno = ENOMEM;
req_ctx_p->xmitBuf = req_ctx_p->recv_buf;
req_ctx_p->recv_buf = NULL;
goto error;
}
/*
** Set the position where the data have to be written in the xmit buffer
*/
req_ctx_p->position = storage_get_position_of_first_byte2write_from_read_req();
if (storio_disk_thread_intf_send(STORIO_DISK_THREAD_READ, req_ctx_p, tic) == 0) {
return;
}
severe("storio_disk_thread_intf_send %s", strerror(errno));
error:
ret.status = SP_FAILURE;
ret.sp_read_ret_t_u.error = errno;
rozorpc_srv_forward_reply(req_ctx_p,(char*)&ret);
/*
** release the context
*/
rozorpc_srv_release_context(req_ctx_p);
STOP_PROFILING(read);
return ;
}
/**
*
Callback to allocate a buffer for receiving a rpc message (mainly a RPC response
The service might reject the buffer allocation because the pool runs
out of buffer or because there is no pool with a buffer that is large enough
for receiving the message because of a out of range size.
@param userRef : pointer to a user reference: not used here
@param socket_context_ref: socket context reference
@param len : length of the incoming message
@retval <>NULL pointer to a receive buffer
@retval == NULL no buffer
*/
void * rozofs_storcli_north_RcvAllocBufCallBack(void *userRef,uint32_t socket_context_ref,uint32_t len)
{
uint32_t free_count = rozofs_storcli_get_free_transaction_context();
if (free_count < STORCLI_CTX_MIN_CNT) {
storcli_buf_depletion_count++;
return NULL;
}
if (stc_rng_is_full()) {
storcli_rng_full_count++;
return NULL;
}
/*
** check if a small or a large buffer must be allocated
*/
if (len > rozofs_storcli_read_write_buf_sz)
{
return NULL;
}
return ruc_buf_getBuffer(storcli_north_buffer_pool_p);
}
/**
* Internal Read procedure
That procedure is used when it is required to read the last block before
performing the truncate
@param working_ctx_p: pointer to the root transaction
@retval 0 on success
retval < 0 on error (see errno for error details)
*/
int rozofs_storcli_internal_read_before_truncate_req(rozofs_storcli_ctx_t *working_ctx_p)
{
storcli_truncate_arg_t *storcli_truncate_rq_p;
void *xmit_buf = NULL;
storcli_read_arg_t storcli_read_args;
storcli_read_arg_t *request = &storcli_read_args;
struct rpc_msg call_msg;
int bufsize;
uint32_t *header_size_p;
XDR xdrs;
uint8_t *arg_p;
storcli_truncate_rq_p = (storcli_truncate_arg_t*)&working_ctx_p->storcli_truncate_arg;
/*
** allocated a buffer from sending the request
*/
xmit_buf = ruc_buf_getBuffer(ROZOFS_STORCLI_NORTH_SMALL_POOL);
if (xmit_buf == NULL)
{
severe(" out of small buffer on north interface ");
errno = ENOMEM;
goto failure;
}
/*
** build the RPC message
*/
request->sid = 0; /* not significant */
request->layout = storcli_truncate_rq_p->layout;
request->cid = storcli_truncate_rq_p->cid;
request->spare = 0; /* not significant */
memcpy(request->dist_set, storcli_truncate_rq_p->dist_set, ROZOFS_SAFE_MAX*sizeof (uint8_t));
memcpy(request->fid, storcli_truncate_rq_p->fid, sizeof (sp_uuid_t));
request->proj_id = 0; /* not significant */
request->bid = storcli_truncate_rq_p->bid;
request->nb_proj = 1;
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = (int)ruc_buf_getMaxPayloadLen(xmit_buf);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_get_transaction_id();
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)STORCLI_PROGRAM;
call_msg.rm_call.cb_vers = (uint32_t)STORCLI_VERSION;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
errno = EFAULT;
severe(" rpc header encode error ");
goto failure;
}
/*
** insert the procedure number, NULL credential and verifier
*/
uint32_t opcode = STORCLI_READ;
uint32_t null_val = 0;
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if (xdr_storcli_read_arg_t(&xdrs,request) == FALSE)
{
severe(" internal read request encoding error ");
errno = EFAULT;
goto failure;
}
/*
** Now get the current length and fill the header of the message
*/
int position = XDR_GETPOS(&xdrs);
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** Submit the pseudo request
*/
rozofs_storcli_read_req_init(0,xmit_buf,rozofs_storcli_internal_read_before_truncate_rsp_cbk,(void*)working_ctx_p,STORCLI_DO_NOT_QUEUE);
return 0;
failure:
if (xmit_buf != NULL) ruc_buf_freeBuffer(xmit_buf);
return -1;
}
/**
Initial truncate request
@param socket_ctx_p: pointer to the af unix socket
@param socketId: reference of the socket (not used)
@param rozofs_storcli_remote_rsp_cbk: callback for sending out the response
@retval : TRUE-> xmit ready event expected
@retval : FALSE-> xmit ready event not expected
*/
void rozofs_storcli_truncate_req_init(uint32_t socket_ctx_idx, void *recv_buf,rozofs_storcli_resp_pf_t rozofs_storcli_remote_rsp_cbk)
{
rozofs_rpc_call_hdr_with_sz_t *com_hdr_p;
rozofs_storcli_ctx_t *working_ctx_p = NULL;
int i;
uint32_t msg_len; /* length of the rpc messsage including the header length */
storcli_truncate_arg_t *storcli_truncate_rq_p = NULL;
rozofs_rpc_call_hdr_t hdr; /* structure that contains the rpc header in host format */
int len; /* effective length of application message */
uint8_t *pmsg; /* pointer to the first available byte in the application message */
uint32_t header_len;
XDR xdrs;
int errcode = EINVAL;
/*
** allocate a context for the duration of the write
*/
working_ctx_p = rozofs_storcli_alloc_context();
if (working_ctx_p == NULL)
{
/*
** that situation MUST not occur since there the same number of receive buffer and working context!!
*/
severe("out of working read/write saved context");
goto failure;
}
storcli_truncate_rq_p = &working_ctx_p->storcli_truncate_arg;
STORCLI_START_NORTH_PROF(working_ctx_p,truncate,0);
/*
** Get the full length of the message and adjust it the the length of the applicative part (RPC header+application msg)
*/
msg_len = ruc_buf_getPayloadLen(recv_buf);
msg_len -=sizeof(uint32_t);
/*
** save the reference of the received socket since it will be needed for sending back the
** response
*/
working_ctx_p->socketRef = socket_ctx_idx;
working_ctx_p->user_param = NULL;
working_ctx_p->recv_buf = recv_buf;
working_ctx_p->response_cbk = rozofs_storcli_remote_rsp_cbk;
/*
** Get the payload of the receive buffer and set the pointer to the array that describes the write request
*/
com_hdr_p = (rozofs_rpc_call_hdr_with_sz_t*) ruc_buf_getPayload(recv_buf);
memcpy(&hdr,&com_hdr_p->hdr,sizeof(rozofs_rpc_call_hdr_t));
/*
** swap the rpc header
*/
scv_call_hdr_ntoh(&hdr);
pmsg = rozofs_storcli_set_ptr_on_nfs_call_msg((char*)&com_hdr_p->hdr,&header_len);
if (pmsg == NULL)
{
errcode = EFAULT;
goto failure;
}
/*
** map the memory on the first applicative RPC byte available and prepare to decode:
** notice that we will not call XDR_FREE since the application MUST
** provide a pointer for storing the file handle
*/
len = msg_len - header_len;
xdrmem_create(&xdrs,(char*)pmsg,len,XDR_DECODE);
/*
** store the source transaction id needed for the reply
*/
working_ctx_p->src_transaction_id = hdr.hdr.xid;
/*
** decode the RPC message of the truncate request
*/
if (xdr_storcli_truncate_arg_t(&xdrs,storcli_truncate_rq_p) == FALSE)
{
/*
** decoding error
*/
errcode = EFAULT;
severe("rpc trucnate request decoding error");
goto failure;
}
/*
** init of the load balancing group/ projection association table:
** That table is ordered: the first corresponds to the storage associated with projection 0, second with 1, etc..
** When build that table, we MUST consider the value of the base which is associated with the distribution
*/
uint8_t rozofs_safe = rozofs_get_rozofs_safe(storcli_truncate_rq_p->layout);
int lbg_in_distribution = 0;
for (i = 0; i <rozofs_safe ; i ++)
{
/*
** Get the load balancing group associated with the sid
*/
int lbg_id = rozofs_storcli_get_lbg_for_sid(storcli_truncate_rq_p->cid,storcli_truncate_rq_p->dist_set[i]);
if (lbg_id < 0)
{
/*
** there is no associated between the sid and the lbg. It is typically the case
** when a new cluster has been added to the configuration and the client does not
** know yet the configuration change
*/
severe("sid is unknown !! %d\n",storcli_truncate_rq_p->dist_set[i]);
continue;
}
rozofs_storcli_lbg_prj_insert_lbg_and_sid(working_ctx_p->lbg_assoc_tb,lbg_in_distribution,
lbg_id,
storcli_truncate_rq_p->dist_set[i]);
rozofs_storcli_lbg_prj_insert_lbg_state(working_ctx_p->lbg_assoc_tb,
lbg_in_distribution,
NORTH_LBG_GET_STATE(working_ctx_p->lbg_assoc_tb[lbg_in_distribution].lbg_id));
lbg_in_distribution++;
if (lbg_in_distribution == rozofs_safe) break;
}
/*
** allocate a small buffer that will be used for sending the response to the truncate request
*/
working_ctx_p->xmitBuf = ruc_buf_getBuffer(ROZOFS_STORCLI_NORTH_SMALL_POOL);
if (working_ctx_p == NULL)
{
/*
** that situation MUST not occur since there the same number of receive buffer and working context!!
*/
errcode = ENOMEM;
severe("out of small buffer");
goto failure;
}
/*
** allocate a sequence number for the working context (same aas for read)
*/
working_ctx_p->read_seqnum = rozofs_storcli_allocate_read_seqnum();
/*
** set now the working variable specific for handling the truncate
** we re-use the structure used for writing even if nothing is written
*/
uint8_t forward_projection = rozofs_get_rozofs_forward(storcli_truncate_rq_p->layout);
for (i = 0; i < forward_projection; i++)
{
working_ctx_p->prj_ctx[i].prj_state = ROZOFS_PRJ_READ_IDLE;
working_ctx_p->prj_ctx[i].prj_buf = ruc_buf_getBuffer(ROZOFS_STORCLI_SOUTH_LARGE_POOL);
if (working_ctx_p->prj_ctx[i].prj_buf == NULL)
{
/*
** that situation MUST not occur since there the same number of receive buffer and working context!!
*/
errcode = ENOMEM;
severe("out of large buffer");
goto failure;
}
/*
** increment inuse counter on each buffer since we might need to re-use that packet in case
** of retransmission
*/
working_ctx_p->prj_ctx[i].inuse_valid = 1;
ruc_buf_inuse_increment(working_ctx_p->prj_ctx[i].prj_buf);
/*
** set the pointer to the bins
*/
int position = rozofs_storcli_get_position_of_first_byte2write_in_truncate();
uint8_t *pbuf = (uint8_t*)ruc_buf_getPayload(working_ctx_p->prj_ctx[i].prj_buf);
working_ctx_p->prj_ctx[i].bins = (bin_t*)(pbuf+position);
}
/*
** Prepare for request serialization
*/
memcpy(working_ctx_p->fid_key, storcli_truncate_rq_p->fid, sizeof (sp_uuid_t));
working_ctx_p->opcode_key = STORCLI_TRUNCATE;
{
/**
* lock all the file for a truncate
*/
uint64_t nb_blocks = 0;
nb_blocks--;
int ret;
ret = stc_rng_insert((void*)working_ctx_p,
STORCLI_READ,working_ctx_p->fid_key,
0,nb_blocks,
&working_ctx_p->sched_idx);
if (ret == 0)
{
/*
** there is a current request that is processed with the same fid and there is a collision
*/
return;
}
/*
** no request pending with that fid, so we can process it right away
*/
return rozofs_storcli_truncate_req_processing(working_ctx_p);
}
/*
**_____________________________________________
** Exception cases
**_____________________________________________
*/
/*
** there was a failure while attempting to allocate a memory ressource.
*/
failure:
/*
** send back the response with the appropriated error code.
** note: The received buffer (rev_buf) is
** intended to be released by this service in case of error or the TCP transmitter
** once it has been passed to the TCP stack.
*/
rozofs_storcli_reply_error_with_recv_buf(socket_ctx_idx,recv_buf,NULL,rozofs_storcli_remote_rsp_cbk,errcode);
/*
** check if the root context was allocated. Free it if is exist
*/
if (working_ctx_p != NULL)
{
/*
** remove the reference to the recvbuf to avoid releasing it twice
*/
STORCLI_STOP_NORTH_PROF(working_ctx_p,truncate,0);
working_ctx_p->recv_buf = NULL;
rozofs_storcli_release_context(working_ctx_p);
}
return;
}
int rozofs_storcli_send_common(exportclt_t * clt,uint32_t timeout_sec,uint32_t prog,uint32_t vers,
int opcode,xdrproc_t encode_fct,void *msg2encode_p,
sys_recv_pf_t recv_cbk,void *fuse_ctx_p,
int storcli_idx,fid_t fid)
{
DEBUG_FUNCTION;
uint8_t *arg_p;
uint32_t *header_size_p;
rozofs_tx_ctx_t *rozofs_tx_ctx_p = NULL;
void *xmit_buf = NULL;
int bufsize;
int ret;
int position;
XDR xdrs;
struct rpc_msg call_msg;
uint32_t null_val = 0;
int lbg_id;
/*
** allocate a transaction context
*/
rozofs_tx_ctx_p = rozofs_tx_alloc();
if (rozofs_tx_ctx_p == NULL)
{
/*
** out of context
** --> put a pending list for the future to avoid repluing ENOMEM
*/
TX_STATS(ROZOFS_TX_NO_CTX_ERROR);
errno = ENOMEM;
goto error;
}
/*
** insert the storcli load balancing context in the stclbg_hash_table hash table.
** the context is embedded in the transaction context
*/
stclbg_hash_table_insert_ctx(&rozofs_tx_ctx_p->rw_lbg,fid,storcli_idx);
/*
** Get the load balancing group reference associated with the storcli
*/
lbg_id = storcli_lbg_get_load_balancing_reference(storcli_idx);
/*
** allocate an xmit buffer
*/
xmit_buf = ruc_buf_getBuffer(ROZOFS_TX_LARGE_TX_POOL);
if (xmit_buf == NULL)
{
/*
** something rotten here, we exit we an error
** without activating the FSM
*/
TX_STATS(ROZOFS_TX_NO_BUFFER_ERROR);
errno = ENOMEM;
goto error;
}
/*
** store the reference of the xmit buffer in the transaction context: might be useful
** in case we want to remove it from a transmit list of the underlying network stacks
*/
rozofs_tx_save_xmitBuf(rozofs_tx_ctx_p,xmit_buf);
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = ruc_buf_getMaxPayloadLen(xmit_buf)-sizeof(uint32_t);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_alloc_xid(rozofs_tx_ctx_p);
/*
** check the case of the READ since, we must set the value of the xid
** at the top of the buffer
*/
#if 1
if ((opcode == STORCLI_READ)||(opcode == STORCLI_WRITE))
{
uint32_t *share_p;
void *shared_buf_ref;
RESTORE_FUSE_PARAM(fuse_ctx_p,shared_buf_ref);
if (shared_buf_ref != NULL)
{
share_p = (uint32_t*)ruc_buf_getPayload(shared_buf_ref);
*share_p = (uint32_t)call_msg.rm_xid;
/**
* copy the buffer for the case of the write
*/
if (opcode == STORCLI_WRITE)
{
storcli_write_arg_t *wr_args = (storcli_write_arg_t*)msg2encode_p;
/*
** get the length to copy from the sshared memory
*/
int len = share_p[1];
/*
** Compute and write data offset considering 128bits alignment
*/
int alignment = wr_args->off%16;
share_p[2] = alignment;
/*
** Set pointer to the buffer start and adjust with alignment
*/
uint8_t * buf_start = (uint8_t *)&share_p[4];
buf_start += alignment;
memcpy(buf_start,wr_args->data.data_val,len);
}
}
}
#endif
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)prog;
call_msg.rm_call.cb_vers = (uint32_t)vers;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** insert the procedure number, NULL credential and verifier
*/
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if ((*encode_fct)(&xdrs,msg2encode_p) == FALSE)
{
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** Now get the current length and fill the header of the message
*/
position = XDR_GETPOS(&xdrs);
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** store the receive call back and its associated parameter
*/
rozofs_tx_ctx_p->recv_cbk = recv_cbk;
rozofs_tx_ctx_p->user_param = fuse_ctx_p;
/*
** now send the message
*/
// int lbg_id = storcli_lbg_get_load_balancing_reference();
/*
** increment the number of pending request towards the storcli
*/
rozofs_storcli_pending_req_count++;
ret = north_lbg_send(lbg_id,xmit_buf);
if (ret < 0)
{
if (rozofs_storcli_pending_req_count > 0) rozofs_storcli_pending_req_count--;
TX_STATS(ROZOFS_TX_SEND_ERROR);
errno = EFAULT;
goto error;
}
TX_STATS(ROZOFS_TX_SEND);
/*
** OK, so now finish by starting the guard timer
*/
rozofs_tx_start_timer(rozofs_tx_ctx_p,timeout_sec);
// if (*tx_ptr != NULL) *tx_ptr = rozofs_tx_ctx_p;
return 0;
error:
if (rozofs_tx_ctx_p != NULL) rozofs_tx_free_from_ptr(rozofs_tx_ctx_p);
// if (xmit_buf != NULL) ruc_buf_freeBuffer(xmit_buf);
return -1;
}
/**
* API for creation a transaction towards an exportd
The reference of the north load balancing is extracted for the client structure
fuse_ctx_p:
That API needs the pointer to the current fuse context. That nformation will be
saved in the transaction context as userParam. It is intended to be used later when
the client gets the response from the server
encoding function;
For making that API generic, the caller is intended to provide the function that
will encode the message in XDR format. The source message that is encoded is
supposed to be pointed by msg2encode_p.
Since the service is non-blocking, the caller MUST provide the callback function
that will be used for decoding the message
@param eid : export id
@param fid : unique file id (directory, regular file, etc...)
@param prog : program
@param vers : program version
@param opcode : metadata opcode
@param encode_fct : encoding function
@msg2encode_p : pointer to the message to encode
@param recv_cbk : receive callback function
@param fuse_buffer_ctx_p : pointer to the fuse context
@retval 0 on success;
@retval -1 on error,, errno contains the cause
*/
int rozofs_expgateway_send_routing_common(uint32_t eid,fid_t fid,uint32_t prog,uint32_t vers,
int opcode,xdrproc_t encode_fct,void *msg2encode_p,
sys_recv_pf_t recv_cbk,void *fuse_buffer_ctx_p)
{
DEBUG_FUNCTION;
uint8_t *arg_p;
uint32_t *header_size_p;
rozofs_tx_ctx_t *rozofs_tx_ctx_p = NULL;
void *xmit_buf = NULL;
int bufsize;
int ret;
int position;
XDR xdrs;
struct rpc_msg call_msg;
uint32_t null_val = 0;
int lbg_id;
expgw_tx_routing_ctx_t local_routing_ctx;
expgw_tx_routing_ctx_t *routing_ctx_p;
rozofs_fuse_save_ctx_t *fuse_ctx_p=NULL;
/*
** Retrieve fuse context when a buffer is given
*/
fuse_ctx_p = NULL;
if (fuse_buffer_ctx_p != NULL) {
if (ruc_buf_checkBuffer(fuse_buffer_ctx_p)) {
GET_FUSE_CTX_P(fuse_ctx_p,fuse_buffer_ctx_p);
}
}
if (fuse_ctx_p != NULL) {
routing_ctx_p = &fuse_ctx_p->expgw_routing_ctx ;
}
else {
routing_ctx_p = &local_routing_ctx;
}
/*
** get the available load balancing group(s) for routing the request
*/
ret = expgw_get_export_routing_lbg_info(eid,fid,routing_ctx_p);
if (ret < 0)
{
/*
** no load balancing group available
*/
errno = EPROTO;
goto error;
}
/*
** allocate a transaction context
*/
rozofs_tx_ctx_p = rozofs_tx_alloc();
if (rozofs_tx_ctx_p == NULL)
{
/*
** out of context
** --> put a pending list for the future to avoid repluing ENOMEM
*/
TX_STATS(ROZOFS_TX_NO_CTX_ERROR);
errno = ENOMEM;
goto error;
}
/*
** allocate an xmit buffer
*/
xmit_buf = ruc_buf_getBuffer(ROZOFS_TX_SMALL_TX_POOL);
if (xmit_buf == NULL)
{
/*
** something rotten here, we exit we an error
** without activating the FSM
*/
TX_STATS(ROZOFS_TX_NO_BUFFER_ERROR);
errno = ENOMEM;
goto error;
}
/*
** The system attempts first to forward the message toward load balancing group
** of an export gateway and then to the master export if the load balancing group
** of the export gateway is not available
*/
lbg_id = expgw_routing_get_next(routing_ctx_p,xmit_buf);
/*
** store the reference of the xmit buffer in the transaction context: might be useful
** in case we want to remove it from a transmit list of the underlying network stacks
*/
rozofs_tx_save_xmitBuf(rozofs_tx_ctx_p,xmit_buf);
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = rozofs_tx_get_small_buffer_size()-sizeof(uint32_t);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_alloc_xid(rozofs_tx_ctx_p);
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)prog;
call_msg.rm_call.cb_vers = (uint32_t)vers;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** insert the procedure number, NULL credential and verifier
*/
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if ((*encode_fct)(&xdrs,msg2encode_p) == FALSE)
{
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** Now get the current length and fill the header of the message
*/
position = XDR_GETPOS(&xdrs);
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** store the receive call back and its associated parameter
*/
rozofs_tx_ctx_p->recv_cbk = recv_cbk;
rozofs_tx_ctx_p->user_param = fuse_buffer_ctx_p;
/*
** now send the message
*/
reloop:
ret = north_lbg_send(lbg_id,xmit_buf);
if (ret < 0)
{
TX_STATS(ROZOFS_TX_SEND_ERROR);
/*
** attempt to get the next available load balancing group
*/
lbg_id = expgw_routing_get_next(routing_ctx_p,xmit_buf);
if (lbg_id >= 0) goto reloop;
errno = EFAULT;
goto error;
}
TX_STATS(ROZOFS_TX_SEND);
/*
** OK, so now finish by starting the guard timer
*/
if (opcode == EP_STATFS) {
/* df must give a response (even negative) in less than 2 seconds !!! */
rozofs_tx_start_timer(rozofs_tx_ctx_p, 2);
}
else {
rozofs_tx_start_timer(rozofs_tx_ctx_p, ROZOFS_TMR_GET(TMR_EXPORT_PROGRAM));
}
return 0;
error:
if (rozofs_tx_ctx_p != NULL) rozofs_tx_free_from_ptr(rozofs_tx_ctx_p);
return -1;
}
int storcli_lbg_cnx_sup_is_selectable(int lbg_id)
{
uint64_t current_date;
storcli_lbg_cnx_supervision_t *p;
void *xmit_buf = NULL;
int ret;
if (lbg_id >=STORCLI_MAX_LBG) return 0;
p = &storcli_lbg_cnx_supervision_tab[lbg_id];
if (p->state == STORCLI_LBG_RUNNING) return 1;
current_date = timer_get_ticker();
// if (current_date > p->expiration_date) return 1;
/*
** check if poll is active
*/
if (p->poll_state == STORCLI_POLL_IN_PRG) return 0;
/*
** check the period
*/
if (current_date > p->next_poll_date)
{
/*
** attempt to poll
*/
p->poll_counter++;
xmit_buf = ruc_buf_getBuffer(ROZOFS_STORCLI_SOUTH_LARGE_POOL);
if (xmit_buf == NULL)
{
return 0;
}
p->poll_state = STORCLI_POLL_IN_PRG;
/*
** increment the inuse to avoid a release of the xmit buffer by rozofs_sorcli_send_rq_common()
*/
ruc_buf_inuse_increment(xmit_buf);
ret = rozofs_sorcli_send_rq_common(lbg_id,ROZOFS_TMR_GET(TMR_RPC_NULL_PROC_LBG),STORAGE_PROGRAM,STORAGE_VERSION,SP_NULL,
(xdrproc_t) xdr_void, (caddr_t) NULL,
xmit_buf,
lbg_id,
0,
0,
rozofs_storcli_sp_null_processing_cbk,
(void*)NULL);
ruc_buf_inuse_decrement(xmit_buf);
if (ret < 0)
{
/*
** direct need to free the xmit buffer
*/
ruc_buf_freeBuffer(xmit_buf);
return 0;
}
/*
** Check if there is direct response from tx module
*/
if (p->poll_state == STORCLI_POLL_ERR)
{
/*
** set the next expiration date
*/
p->next_poll_date = current_date+STORCLI_LBG_SP_NULL_INTERVAL;
/*
** release the xmit buffer since there was a direct reply from the lbg while attempting to send the buffer
*/
ruc_buf_freeBuffer(xmit_buf);
return 0;
}
}
return 0;
}
/**
Server callback for GW_PROGRAM protocol:
GW_INVALIDATE_SECTIONS
GW_INVALIDATE_ALL
GW_CONFIGURATION
GW_POLL
That callback is called upon receiving a GW_PROGRAM message
from the master exportd
@param socket_ctx_p: pointer to the af unix socket
@param socketId: reference of the socket (not used)
@retval : TRUE-> xmit ready event expected
@retval : FALSE-> xmit ready event not expected
*/
void storio_req_rcv_cbk(void *userRef,uint32_t socket_ctx_idx, void *recv_buf)
{
uint32_t *com_hdr_p;
rozofs_rpc_call_hdr_t hdr;
sp_status_ret_t arg_err;
char * arguments;
int size = 0;
rozorpc_srv_ctx_t *rozorpc_srv_ctx_p = NULL;
com_hdr_p = (uint32_t*) ruc_buf_getPayload(recv_buf);
com_hdr_p +=1; /* skip the size of the rpc message */
memcpy(&hdr,com_hdr_p,sizeof(rozofs_rpc_call_hdr_t));
scv_call_hdr_ntoh(&hdr);
/*
** allocate a context for the duration of the transaction since it might be possible
** that the gateway needs to interrogate the exportd and thus needs to save the current
** request until receiving the response from the exportd
*/
rozorpc_srv_ctx_p = rozorpc_srv_alloc_context();
if (rozorpc_srv_ctx_p == NULL)
{
fatal(" Out of rpc context");
}
/*
** save the initial transaction id, received buffer and reference of the connection
*/
rozorpc_srv_ctx_p->src_transaction_id = hdr.hdr.xid;
rozorpc_srv_ctx_p->recv_buf = recv_buf;
rozorpc_srv_ctx_p->socketRef = socket_ctx_idx;
/*
** Allocate buffer for decoded aeguments
*/
rozorpc_srv_ctx_p->decoded_arg = ruc_buf_getBuffer(decoded_rpc_buffer_pool);
if (rozorpc_srv_ctx_p->decoded_arg == NULL) {
rozorpc_srv_ctx_p->xmitBuf = rozorpc_srv_ctx_p->recv_buf;
rozorpc_srv_ctx_p->recv_buf = NULL;
rozorpc_srv_ctx_p->xdr_result =(xdrproc_t) xdr_sp_status_ret_t;
arg_err.status = SP_FAILURE;
arg_err.sp_status_ret_t_u.error = ENOMEM;
rozorpc_srv_forward_reply(rozorpc_srv_ctx_p,(char*)&arg_err);
rozorpc_srv_release_context(rozorpc_srv_ctx_p);
return;
}
arguments = ruc_buf_getPayload(rozorpc_srv_ctx_p->decoded_arg);
void (*local)(void *, rozorpc_srv_ctx_t *);
switch (hdr.proc) {
case SP_NULL:
rozorpc_srv_ctx_p->arg_decoder = (xdrproc_t) xdr_void;
rozorpc_srv_ctx_p->xdr_result = (xdrproc_t) xdr_void;
local = sp_null_1_svc_nb;
break;
case SP_WRITE:
rozorpc_srv_ctx_p->arg_decoder = (xdrproc_t) xdr_sp_write_arg_no_bins_t;
rozorpc_srv_ctx_p->xdr_result = (xdrproc_t) xdr_sp_write_ret_t;
// local = sp_write_1_svc_nb;
local = sp_write_1_svc_disk_thread;
size = sizeof (sp_write_arg_no_bins_t);
break;
case SP_READ:
rozorpc_srv_ctx_p->arg_decoder = (xdrproc_t) xdr_sp_read_arg_t;
rozorpc_srv_ctx_p->xdr_result = (xdrproc_t) xdr_sp_read_ret_t;
// local = sp_read_1_svc_nb;
local = sp_read_1_svc_disk_thread;
size = sizeof (sp_read_arg_t);
break;
case SP_TRUNCATE:
rozorpc_srv_ctx_p->arg_decoder = (xdrproc_t) xdr_sp_truncate_arg_t;
rozorpc_srv_ctx_p->xdr_result = (xdrproc_t) xdr_sp_status_ret_t;
// local = sp_truncate_1_svc_nb;
local = sp_truncate_1_svc_disk_thread;
size = sizeof (sp_truncate_arg_t);
break;
default:
rozorpc_srv_ctx_p->xmitBuf = rozorpc_srv_ctx_p->recv_buf;
rozorpc_srv_ctx_p->recv_buf = NULL;
rozorpc_srv_ctx_p->xdr_result =(xdrproc_t) xdr_sp_status_ret_t;
arg_err.status = SP_FAILURE;
arg_err.sp_status_ret_t_u.error = EPROTO;
rozorpc_srv_forward_reply(rozorpc_srv_ctx_p,(char*)&arg_err);
rozorpc_srv_release_context(rozorpc_srv_ctx_p);
return;
}
memset(arguments,0, size);
ruc_buf_setPayloadLen(rozorpc_srv_ctx_p->decoded_arg,size); // for debug
/*
** decode the payload of the rpc message
*/
if (!rozorpc_srv_getargs_with_position (recv_buf, (xdrproc_t) rozorpc_srv_ctx_p->arg_decoder,
(caddr_t) arguments, &rozorpc_srv_ctx_p->position))
{
rozorpc_srv_ctx_p->xmitBuf = rozorpc_srv_ctx_p->recv_buf;
rozorpc_srv_ctx_p->recv_buf = NULL;
rozorpc_srv_ctx_p->xdr_result = (xdrproc_t)xdr_sp_status_ret_t;
arg_err.status = SP_FAILURE;
arg_err.sp_status_ret_t_u.error = errno;
rozorpc_srv_forward_reply(rozorpc_srv_ctx_p,(char*)&arg_err);
/*
** release the context
*/
rozorpc_srv_release_context(rozorpc_srv_ctx_p);
return;
}
/*
** call the user call-back
*/
(*local)(arguments, rozorpc_srv_ctx_p);
}
/**
* allocate a receive_buffer from the default AF_UNIX pool
*
@param none
@retval <> NULL address of the xmit buffer
@retval == NULL out of buffer
*/
void *af_unix_alloc_recv_buf() {
return ruc_buf_getBuffer(af_unix_buffer_pool_tb[1]);
}
/**
* ROZOFS Generic RPC Request transaction in non-blocking mode
That service initiates RPC call towards the destination referenced by its associated load balancing group
WHen the transaction is started, the application will received the response thanks the provided callback
The first parameter is a user dependent reference and the second pointer is the pointer to the decoded
area.
In case of decoding error, transmission error, the second pointer is NULL and errno is asserted with the
error.
The array provided for decoding the response might be a static variable within the user context or
can be an allocated array. If that array has be allocated by the application it is up to the application
to release it
@param lbg_id : reference of the load balancing group of the exportd
@param prog : program
@param vers : program version
@param opcode : metadata opcode
@param encode_fct : encoding function
@param msg2encode_p : pointer to the message to encode
@param decode_fct : xdr function for message decoding
@param ret: pointer to the array that is used for message decoding
@parem ret_len : length of the array used for decoding
@param recv_cbk : receive callback function (for interpretation of the rpc result
@param ctx_p : pointer to the user context
@retval 0 on success;
@retval -1 on error,, errno contains the cause
*/
int rozofs_rpc_non_blocking_req_send (int lbg_id,uint32_t prog,uint32_t vers,
int opcode,xdrproc_t encode_fct,void *msg2encode_p,
xdrproc_t decode_fct,
sys_recv_pf_t recv_cbk,void *ctx_p,
void *ret,int ret_len)
{
DEBUG_FUNCTION;
uint8_t *arg_p;
uint32_t *header_size_p;
rozofs_tx_ctx_t *rozofs_tx_ctx_p = NULL;
void *xmit_buf = NULL;
int bufsize;
int ret;
int position;
XDR xdrs;
struct rpc_msg call_msg;
uint32_t null_val = 0;
rozofs_tx_rpc_ctx_t *rpc_ctx_p = NULL;
/*
** allocate a rpc context
*/
rozofs_rpc_p = rozofs_rpc_req_alloc();
if (rozofs_rpc_p == NULL)
{
/*
** out of context
*/
errno = ENOMEM;
goto error;
}
/*
** save the rpc parameter of the caller
*/
rpc_ctx_p = &rozofs_tx_ctx_p->rpc_ctx;
rpc_ctx_p->user_ref = ctx_p; /* save the user reference of the caller */
rpc_ctx_p->xdr_result = decode_fct; /* save the decoding procedure */
rpc_ctx_p->response_cbk = recv_cbk ;
rpc_ctx_p->ret_len = ret_len;
rpc_ctx_p->ret_p = ret;
START_RPC_REQ_PROFILING_START(rpc_ctx_p);
/*
** allocate a transaction context
*/
rozofs_tx_ctx_p = rozofs_tx_alloc();
if (rozofs_tx_ctx_p == NULL)
{
/*
** out of context
** --> put a pending list for the future to avoid repluing ENOMEM
*/
TX_STATS(ROZOFS_TX_NO_CTX_ERROR);
errno = ENOMEM;
goto error;
}
/*
** allocate an xmit buffer
*/
xmit_buf = ruc_buf_getBuffer(ROZOFS_TX_LARGE_TX_POOL);
if (xmit_buf == NULL)
{
/*
** something rotten here, we exit we an error
** without activating the FSM
*/
TX_STATS(ROZOFS_TX_NO_BUFFER_ERROR);
errno = ENOMEM;
goto error;
}
/*
** store the reference of the xmit buffer in the transaction context: might be useful
** in case we want to remove it from a transmit list of the underlying network stacks
*/
rozofs_tx_save_xmitBuf(rozofs_tx_ctx_p,xmit_buf);
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = ruc_buf_getMaxPayloadLen(xmit_buf);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_alloc_xid(rozofs_tx_ctx_p);
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)prog;
call_msg.rm_call.cb_vers = (uint32_t)vers;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** insert the procedure number, NULL credential and verifier
*/
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if ((*encode_fct)(&xdrs,msg2encode_p) == FALSE)
{
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** Now get the current length and fill the header of the message
*/
position = XDR_GETPOS(&xdrs);
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** store the receive call back and its associated parameter
*/
rozofs_tx_ctx_p->recv_cbk = rozofs_rpc_generic_reply_cbk;
rozofs_tx_ctx_p->user_param = rpc_ctx_p;
/*
** now send the message
*/
ret = north_lbg_send(lbg_id,xmit_buf);
if (ret < 0)
{
TX_STATS(ROZOFS_TX_SEND_ERROR);
errno = EFAULT;
goto error;
}
TX_STATS(ROZOFS_TX_SEND);
/*
** OK, so now finish by starting the guard timer
*/
rozofs_tx_start_timer(rozofs_tx_ctx_p, 25);
return 0;
error:
if (rozofs_tx_ctx_p != NULL) rozofs_tx_free_from_ptr(rozofs_tx_ctx_p);
if (rpc_ctx_p != NULL) rozofs_rpc_req_free(rpc_ctx_p);
return -1;
}
/**
* test function for allocatiing a buffer in the client space
The service might reject the buffer allocation because the pool runs
out of buffer or because there is no pool with a buffer that is large enough
for receiving the message because of a out of range size.
@param userRef : pointer to a user reference: not used here
@param socket_context_ref: socket context reference
@param len : length of the incoming message
@retval <>NULL pointer to a receive buffer
@retval == NULL no buffer
*/
void * af_unix_disk_userRcvAllocBufCallBack(void *userRef,uint32_t socket_context_ref,uint32_t len) {
return ruc_buf_getBuffer(af_unix_disk_pool_recv);
}
void sp_read_1_svc_nb(void * pt, rozorpc_srv_ctx_t *req_ctx_p) {
sp_read_arg_t * args = (sp_read_arg_t *) pt;
static sp_read_ret_t ret;
storage_t *st = 0;
START_PROFILING_IO(read, args->nb_proj * rozofs_get_max_psize(args->layout)
* sizeof (bin_t));
ret.status = SP_FAILURE;
/*
** allocate a buffer for the response
*/
req_ctx_p->xmitBuf = ruc_buf_getBuffer(storage_xmit_buffer_pool_p);
if (req_ctx_p->xmitBuf == NULL)
{
severe("Out of memory STORAGE_NORTH_LARGE_POOL");
ret.sp_read_ret_t_u.error = ENOMEM;
req_ctx_p->xmitBuf = req_ctx_p->recv_buf;
req_ctx_p->recv_buf = NULL;
goto error;
}
// Get the storage for the couple (cid;sid)
if ((st = storaged_lookup(args->cid, args->sid)) == 0) {
ret.sp_read_ret_t_u.error = errno;
goto error;
}
/*
** set the pointer to the bins
*/
int position = storage_get_position_of_first_byte2write_from_read_req();
uint8_t *pbuf = (uint8_t*)ruc_buf_getPayload(req_ctx_p->xmitBuf);
/*
** clear the length of the bins and set the pointer where data must be returned
*/
ret.sp_read_ret_t_u.rsp.bins.bins_val =(char *)(pbuf+position); ;
ret.sp_read_ret_t_u.rsp.bins.bins_len = 0;
#if 0 // for future usage with distributed cache
/*
** clear the optimization array
*/
ret.sp_read_ret_t_u.rsp.optim.optim_val = (char*)sp_optim;
ret.sp_read_ret_t_u.rsp.optim.optim_len = 0;
#endif
// Read projections
if (storage_read(st, args->layout, (sid_t *) args->dist_set, args->spare,
(unsigned char *) args->fid, args->bid, args->nb_proj,
(bin_t *) ret.sp_read_ret_t_u.rsp.bins.bins_val,
(size_t *) & ret.sp_read_ret_t_u.rsp.bins.bins_len,
&ret.sp_read_ret_t_u.rsp.file_size) != 0) {
ret.sp_read_ret_t_u.error = errno;
goto error;
}
ret.status = SP_SUCCESS;
storaged_srv_forward_read_success(req_ctx_p,&ret);
/*
** check the case of the readahead
*/
storage_check_readahead();
goto out;
error:
rozorpc_srv_forward_reply(req_ctx_p,(char*)&ret);
/*
** release the context
*/
out:
rozorpc_srv_release_context(req_ctx_p);
STOP_PROFILING(read);
return ;
}
/**
Initial write repair request
Here it is assumed that storclo is working with the context that has been allocated
@param working_ctx_p: pointer to the working context of a read transaction
@retval : TRUE-> xmit ready event expected
@retval : FALSE-> xmit ready event not expected
*/
void rozofs_storcli_repair_req_init(rozofs_storcli_ctx_t *working_ctx_p)
{
int i;
storcli_read_arg_t *storcli_read_rq_p = (storcli_read_arg_t*)&working_ctx_p->storcli_read_arg;
STORCLI_START_NORTH_PROF(working_ctx_p,repair,0);
/*
** set the pointer to to first available data (decoded data)
*/
working_ctx_p->data_write_p = working_ctx_p->data_read_p;
/*
** set now the working variable specific for handling the write
** We need one large buffer per projection that will be written on storage
** we keep the buffer that have been allocated for the read.
*/
uint8_t forward_projection = rozofs_get_rozofs_forward(storcli_read_rq_p->layout);
for (i = 0; i < forward_projection; i++)
{
working_ctx_p->prj_ctx[i].prj_state = ROZOFS_PRJ_WR_IDLE;
if (working_ctx_p->prj_ctx[i].prj_buf == NULL)
{
working_ctx_p->prj_ctx[i].prj_buf = ruc_buf_getBuffer(ROZOFS_STORCLI_SOUTH_LARGE_POOL);
if (working_ctx_p->prj_ctx[i].prj_buf == NULL)
{
/*
** that situation MUST not occur since there the same number of receive buffer and working context!!
*/
severe("out of large buffer");
goto failure;
}
}
/*
** set the pointer to the bins
*/
int position;
// For compatibility between new clients and old storages
if (storcli_storage_supports_repair2) {
position = rozofs_storcli_repair2_get_position_of_first_byte2write();
}
else {
position = rozofs_storcli_repair_get_position_of_first_byte2write();
}
uint8_t *pbuf = (uint8_t*)ruc_buf_getPayload(working_ctx_p->prj_ctx[i].prj_buf);
working_ctx_p->prj_ctx[i].bins = (bin_t*)(pbuf+position);
}
/*
** now regenerate the projections that were in error
*/
rozofs_storcli_transform_forward_repair(working_ctx_p,
storcli_read_rq_p->layout,
storcli_read_rq_p->nb_proj,
(char *)working_ctx_p->data_write_p);
/*
** starts the sending of the repaired projections
*/
rozofs_storcli_write_repair_req_processing(working_ctx_p);
return;
failure:
/*
** send back the response of the read request towards rozofsmount
*/
rozofs_storcli_read_reply_success(working_ctx_p);
/*
** release the root transaction context
*/
STORCLI_STOP_NORTH_PROF(working_ctx_p,repair,0);
rozofs_storcli_release_context(working_ctx_p);
}
/**
* allocate a xmit_buffer from the default AF_UNIX pool
*
@param none
@retval <> NULL address of the xmit buffer
@retval == NULL out of buffer
*/
void *af_unix_alloc_xmit_buf() {
return ruc_buf_getBuffer(af_unix_buffer_pool_tb[0]);
}
