/**
* send a write success reply
That API fill up the common header with the SP_READ_RSP opcode
insert the transaction_id associated with the inittial request transaction id
insert a status OK
insert the length of the data payload
In case of a success it is up to the called function to release the xmit buffer
@param p : pointer to the root transaction context used for the read
@retval none
*/
void rozofs_storcli_write_reply_success(rozofs_storcli_ctx_t *p)
{
int ret;
uint8_t *pbuf; /* pointer to the part that follows the header length */
uint32_t *header_len_p; /* pointer to the array that contains the length of the rpc message*/
XDR xdrs;
int len;
storcli_status_ret_t status;
storcli_write_arg_no_data_t *storcli_write_rq_p = NULL;
status.status = STORCLI_SUCCESS;
status.storcli_status_ret_t_u.error = 0; /* NS */
storcli_write_rq_p = (storcli_write_arg_no_data_t*)&p->storcli_write_arg;
STORCLI_STOP_NORTH_PROF(p,write,storcli_write_rq_p->len);
/*
** create xdr structure on top of the buffer that will be used for sending the response
*/
header_len_p = (uint32_t*)ruc_buf_getPayload(p->xmitBuf);
pbuf = (uint8_t*) (header_len_p+1);
len = (int)ruc_buf_getMaxPayloadLen(p->xmitBuf);
len -= sizeof(uint32_t);
xdrmem_create(&xdrs,(char*)pbuf,len,XDR_ENCODE);
if (rozofs_encode_rpc_reply(&xdrs,(xdrproc_t)xdr_sp_status_ret_t,(caddr_t)&status,p->src_transaction_id) != TRUE)
{
severe("rpc reply encoding error");
goto error;
}
/*
** compute the total length of the message for the rpc header and add 4 bytes more bytes for
** the ruc buffer to take care of the header length of the rpc message.
*/
int total_len = xdr_getpos(&xdrs) ;
*header_len_p = htonl(0x80000000 | total_len);
total_len +=sizeof(uint32_t);
ruc_buf_setPayloadLen(p->xmitBuf,total_len);
/*
** Clear the reference of the seqnum to prevent any late response to be processed
** by setting seqnum to 0 any late response is ignored and the associated ressources
** will released (buffer associated with the response). This typically permits to
** avoid sending again the response while this has already been done
*/
p->read_seqnum = 0;
/*
** Get the callback for sending back the response:
** A callback is needed since the request for read might be local or remote
*/
ret = (*p->response_cbk)(p->xmitBuf,p->socketRef,p->user_param);
if (ret == 0)
{
/**
* success so remove the reference of the xmit buffer since it is up to the called
* function to release it
*/
p->xmitBuf = NULL;
}
error:
return;
}
/**
* ROZOFS Generic RPC Request transaction in non-blocking mode
That service initiates RPC call towards the destination referenced by its associated load balancing group
WHen the transaction is started, the application will received the response thanks the provided callback
The first parameter is a user dependent reference and the second pointer is the pointer to the decoded
area.
In case of decoding error, transmission error, the second pointer is NULL and errno is asserted with the
error.
The array provided for decoding the response might be a static variable within the user context or
can be an allocated array. If that array has be allocated by the application it is up to the application
to release it
@param lbg_id : reference of the load balancing group of the exportd
@param prog : program
@param vers : program version
@param opcode : metadata opcode
@param encode_fct : encoding function
@param msg2encode_p : pointer to the message to encode
@param decode_fct : xdr function for message decoding
@param ret: pointer to the array that is used for message decoding
@parem ret_len : length of the array used for decoding
@param recv_cbk : receive callback function (for interpretation of the rpc result
@param ctx_p : pointer to the user context
@retval 0 on success;
@retval -1 on error,, errno contains the cause
*/
int rozofs_rpc_non_blocking_req_send (int lbg_id,uint32_t prog,uint32_t vers,
int opcode,xdrproc_t encode_fct,void *msg2encode_p,
xdrproc_t decode_fct,
sys_recv_pf_t recv_cbk,void *ctx_p,
void *ret,int ret_len)
{
DEBUG_FUNCTION;
uint8_t *arg_p;
uint32_t *header_size_p;
rozofs_tx_ctx_t *rozofs_tx_ctx_p = NULL;
void *xmit_buf = NULL;
int bufsize;
int ret;
int position;
XDR xdrs;
struct rpc_msg call_msg;
uint32_t null_val = 0;
rozofs_tx_rpc_ctx_t *rpc_ctx_p = NULL;
/*
** allocate a rpc context
*/
rozofs_rpc_p = rozofs_rpc_req_alloc();
if (rozofs_rpc_p == NULL)
{
/*
** out of context
*/
errno = ENOMEM;
goto error;
}
/*
** save the rpc parameter of the caller
*/
rpc_ctx_p = &rozofs_tx_ctx_p->rpc_ctx;
rpc_ctx_p->user_ref = ctx_p; /* save the user reference of the caller */
rpc_ctx_p->xdr_result = decode_fct; /* save the decoding procedure */
rpc_ctx_p->response_cbk = recv_cbk ;
rpc_ctx_p->ret_len = ret_len;
rpc_ctx_p->ret_p = ret;
START_RPC_REQ_PROFILING_START(rpc_ctx_p);
/*
** allocate a transaction context
*/
rozofs_tx_ctx_p = rozofs_tx_alloc();
if (rozofs_tx_ctx_p == NULL)
{
/*
** out of context
** --> put a pending list for the future to avoid repluing ENOMEM
*/
TX_STATS(ROZOFS_TX_NO_CTX_ERROR);
errno = ENOMEM;
goto error;
}
/*
** allocate an xmit buffer
*/
xmit_buf = ruc_buf_getBuffer(ROZOFS_TX_LARGE_TX_POOL);
if (xmit_buf == NULL)
{
/*
** something rotten here, we exit we an error
** without activating the FSM
*/
TX_STATS(ROZOFS_TX_NO_BUFFER_ERROR);
errno = ENOMEM;
goto error;
}
/*
** store the reference of the xmit buffer in the transaction context: might be useful
** in case we want to remove it from a transmit list of the underlying network stacks
*/
rozofs_tx_save_xmitBuf(rozofs_tx_ctx_p,xmit_buf);
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = ruc_buf_getMaxPayloadLen(xmit_buf);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_alloc_xid(rozofs_tx_ctx_p);
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)prog;
call_msg.rm_call.cb_vers = (uint32_t)vers;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** insert the procedure number, NULL credential and verifier
*/
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if ((*encode_fct)(&xdrs,msg2encode_p) == FALSE)
{
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** Now get the current length and fill the header of the message
*/
position = XDR_GETPOS(&xdrs);
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** store the receive call back and its associated parameter
*/
rozofs_tx_ctx_p->recv_cbk = rozofs_rpc_generic_reply_cbk;
rozofs_tx_ctx_p->user_param = rpc_ctx_p;
/*
** now send the message
*/
ret = north_lbg_send(lbg_id,xmit_buf);
if (ret < 0)
{
TX_STATS(ROZOFS_TX_SEND_ERROR);
errno = EFAULT;
goto error;
}
TX_STATS(ROZOFS_TX_SEND);
/*
** OK, so now finish by starting the guard timer
*/
rozofs_tx_start_timer(rozofs_tx_ctx_p, 25);
return 0;
error:
if (rozofs_tx_ctx_p != NULL) rozofs_tx_free_from_ptr(rozofs_tx_ctx_p);
if (rpc_ctx_p != NULL) rozofs_rpc_req_free(rpc_ctx_p);
return -1;
}
/**
* Internal Read procedure
That procedure is used when it is required to read the last block before
performing the truncate
@param working_ctx_p: pointer to the root transaction
@retval 0 on success
retval < 0 on error (see errno for error details)
*/
int rozofs_storcli_internal_read_before_truncate_req(rozofs_storcli_ctx_t *working_ctx_p)
{
storcli_truncate_arg_t *storcli_truncate_rq_p;
void *xmit_buf = NULL;
storcli_read_arg_t storcli_read_args;
storcli_read_arg_t *request = &storcli_read_args;
struct rpc_msg call_msg;
int bufsize;
uint32_t *header_size_p;
XDR xdrs;
uint8_t *arg_p;
storcli_truncate_rq_p = (storcli_truncate_arg_t*)&working_ctx_p->storcli_truncate_arg;
/*
** allocated a buffer from sending the request
*/
xmit_buf = ruc_buf_getBuffer(ROZOFS_STORCLI_NORTH_SMALL_POOL);
if (xmit_buf == NULL)
{
severe(" out of small buffer on north interface ");
errno = ENOMEM;
goto failure;
}
/*
** build the RPC message
*/
request->sid = 0; /* not significant */
request->layout = storcli_truncate_rq_p->layout;
request->cid = storcli_truncate_rq_p->cid;
request->spare = 0; /* not significant */
memcpy(request->dist_set, storcli_truncate_rq_p->dist_set, ROZOFS_SAFE_MAX*sizeof (uint8_t));
memcpy(request->fid, storcli_truncate_rq_p->fid, sizeof (sp_uuid_t));
request->proj_id = 0; /* not significant */
request->bid = storcli_truncate_rq_p->bid;
request->nb_proj = 1;
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = (int)ruc_buf_getMaxPayloadLen(xmit_buf);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_get_transaction_id();
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)STORCLI_PROGRAM;
call_msg.rm_call.cb_vers = (uint32_t)STORCLI_VERSION;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
errno = EFAULT;
severe(" rpc header encode error ");
goto failure;
}
/*
** insert the procedure number, NULL credential and verifier
*/
uint32_t opcode = STORCLI_READ;
uint32_t null_val = 0;
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if (xdr_storcli_read_arg_t(&xdrs,request) == FALSE)
{
severe(" internal read request encoding error ");
errno = EFAULT;
goto failure;
}
/*
** Now get the current length and fill the header of the message
*/
int position = XDR_GETPOS(&xdrs);
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** Submit the pseudo request
*/
rozofs_storcli_read_req_init(0,xmit_buf,rozofs_storcli_internal_read_before_truncate_rsp_cbk,(void*)working_ctx_p,STORCLI_DO_NOT_QUEUE);
return 0;
failure:
if (xmit_buf != NULL) ruc_buf_freeBuffer(xmit_buf);
return -1;
}
int rozofs_storcli_send_common(exportclt_t * clt,uint32_t timeout_sec,uint32_t prog,uint32_t vers,
int opcode,xdrproc_t encode_fct,void *msg2encode_p,
sys_recv_pf_t recv_cbk,void *fuse_ctx_p,
int storcli_idx,fid_t fid)
{
DEBUG_FUNCTION;
uint8_t *arg_p;
uint32_t *header_size_p;
rozofs_tx_ctx_t *rozofs_tx_ctx_p = NULL;
void *xmit_buf = NULL;
int bufsize;
int ret;
int position;
XDR xdrs;
struct rpc_msg call_msg;
uint32_t null_val = 0;
int lbg_id;
/*
** allocate a transaction context
*/
rozofs_tx_ctx_p = rozofs_tx_alloc();
if (rozofs_tx_ctx_p == NULL)
{
/*
** out of context
** --> put a pending list for the future to avoid repluing ENOMEM
*/
TX_STATS(ROZOFS_TX_NO_CTX_ERROR);
errno = ENOMEM;
goto error;
}
/*
** insert the storcli load balancing context in the stclbg_hash_table hash table.
** the context is embedded in the transaction context
*/
stclbg_hash_table_insert_ctx(&rozofs_tx_ctx_p->rw_lbg,fid,storcli_idx);
/*
** Get the load balancing group reference associated with the storcli
*/
lbg_id = storcli_lbg_get_load_balancing_reference(storcli_idx);
/*
** allocate an xmit buffer
*/
xmit_buf = ruc_buf_getBuffer(ROZOFS_TX_LARGE_TX_POOL);
if (xmit_buf == NULL)
{
/*
** something rotten here, we exit we an error
** without activating the FSM
*/
TX_STATS(ROZOFS_TX_NO_BUFFER_ERROR);
errno = ENOMEM;
goto error;
}
/*
** store the reference of the xmit buffer in the transaction context: might be useful
** in case we want to remove it from a transmit list of the underlying network stacks
*/
rozofs_tx_save_xmitBuf(rozofs_tx_ctx_p,xmit_buf);
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = ruc_buf_getMaxPayloadLen(xmit_buf)-sizeof(uint32_t);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_alloc_xid(rozofs_tx_ctx_p);
/*
** check the case of the READ since, we must set the value of the xid
** at the top of the buffer
*/
#if 1
if ((opcode == STORCLI_READ)||(opcode == STORCLI_WRITE))
{
uint32_t *share_p;
void *shared_buf_ref;
RESTORE_FUSE_PARAM(fuse_ctx_p,shared_buf_ref);
if (shared_buf_ref != NULL)
{
share_p = (uint32_t*)ruc_buf_getPayload(shared_buf_ref);
*share_p = (uint32_t)call_msg.rm_xid;
/**
* copy the buffer for the case of the write
*/
if (opcode == STORCLI_WRITE)
{
storcli_write_arg_t *wr_args = (storcli_write_arg_t*)msg2encode_p;
/*
** get the length to copy from the sshared memory
*/
int len = share_p[1];
/*
** Compute and write data offset considering 128bits alignment
*/
int alignment = wr_args->off%16;
share_p[2] = alignment;
/*
** Set pointer to the buffer start and adjust with alignment
*/
uint8_t * buf_start = (uint8_t *)&share_p[4];
buf_start += alignment;
memcpy(buf_start,wr_args->data.data_val,len);
}
}
}
#endif
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)prog;
call_msg.rm_call.cb_vers = (uint32_t)vers;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** insert the procedure number, NULL credential and verifier
*/
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if ((*encode_fct)(&xdrs,msg2encode_p) == FALSE)
{
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** Now get the current length and fill the header of the message
*/
position = XDR_GETPOS(&xdrs);
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** store the receive call back and its associated parameter
*/
rozofs_tx_ctx_p->recv_cbk = recv_cbk;
rozofs_tx_ctx_p->user_param = fuse_ctx_p;
/*
** now send the message
*/
// int lbg_id = storcli_lbg_get_load_balancing_reference();
/*
** increment the number of pending request towards the storcli
*/
rozofs_storcli_pending_req_count++;
ret = north_lbg_send(lbg_id,xmit_buf);
if (ret < 0)
{
if (rozofs_storcli_pending_req_count > 0) rozofs_storcli_pending_req_count--;
TX_STATS(ROZOFS_TX_SEND_ERROR);
errno = EFAULT;
goto error;
}
TX_STATS(ROZOFS_TX_SEND);
/*
** OK, so now finish by starting the guard timer
*/
rozofs_tx_start_timer(rozofs_tx_ctx_p,timeout_sec);
// if (*tx_ptr != NULL) *tx_ptr = rozofs_tx_ctx_p;
return 0;
error:
if (rozofs_tx_ctx_p != NULL) rozofs_tx_free_from_ptr(rozofs_tx_ctx_p);
// if (xmit_buf != NULL) ruc_buf_freeBuffer(xmit_buf);
return -1;
}
int rozofs_export_poll_tx(af_unix_ctx_generic_t *sock_p,
uint32_t prog,uint32_t vers,
int opcode,xdrproc_t encode_fct,void *msg2encode_p,
void *xmit_buf,
int extra_len,
int applicative_tmo_sec,
sys_recv_pf_t recv_cbk,void *user_ctx_p)
{
DEBUG_FUNCTION;
uint8_t *arg_p;
uint32_t *header_size_p;
rozofs_tx_ctx_t *rozofs_tx_ctx_p = NULL;
int bufsize;
int ret;
int position;
XDR xdrs;
struct rpc_msg call_msg;
uint32_t null_val = 0;
uint32_t sock_idx_in_lbg = -1;
/*
** get the entry within the load balancing group
*/
{
north_lbg_ctx_t *lbg_p;
int lbg_id = sock_p->availability_param;
int start_idx;
/*
** Get the pointer to the lbg
*/
lbg_p = north_lbg_getObjCtx_p(lbg_id);
if (lbg_p == NULL)
{
severe("rozofs_export_poll_tx: no such instance %d ",lbg_id);
return -1;
}
for (start_idx = 0; start_idx < lbg_p->nb_entries_conf; start_idx++)
{
if (lbg_p->entry_tb[start_idx].sock_ctx_ref == sock_p->index) break;
}
if (lbg_p->nb_entries_conf == start_idx)
{
severe("rozofs_export_poll_tx: no such instance %d ",sock_p->index);
return -1;
}
/**
** start_idx is the index within the load balancing
*/
sock_idx_in_lbg = start_idx;
//info("JPM poll lbg %d %s entry %d",lbg_id,lbg_p->name,sock_idx_in_lbg);
}
/*
** allocate a transaction context
*/
rozofs_tx_ctx_p = rozofs_tx_alloc();
if (rozofs_tx_ctx_p == NULL)
{
/*
** out of context
** --> put a pending list for the future to avoid repluing ENOMEM
*/
TX_STATS(ROZOFS_TX_NO_CTX_ERROR);
errno = ENOMEM;
goto error;
}
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = (int)ruc_buf_getMaxPayloadLen(xmit_buf);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_alloc_xid(rozofs_tx_ctx_p);
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)prog;
call_msg.rm_call.cb_vers = (uint32_t)vers;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** insert the procedure number, NULL credential and verifier
*/
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if ((*encode_fct)(&xdrs,msg2encode_p) == FALSE)
{
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** Now get the current length and fill the header of the message
*/
position = XDR_GETPOS(&xdrs);
/*
** add the extra_len if any
*/
position +=extra_len;
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** store the receive call back and its associated parameter
*/
rozofs_tx_ctx_p->recv_cbk = recv_cbk;
rozofs_tx_ctx_p->user_param = user_ctx_p;
rozofs_tx_write_opaque_data(rozofs_tx_ctx_p,1,1); /* lock */
rozofs_tx_write_opaque_data(rozofs_tx_ctx_p,2,sock_p->availability_param); /* lock */
rozofs_tx_write_opaque_data(rozofs_tx_ctx_p,3,sock_idx_in_lbg); /* lock */
/*
** now send the message
*/
ret = af_unix_generic_stream_send(sock_p,xmit_buf);
if (ret < 0)
{
TX_STATS(ROZOFS_TX_SEND_ERROR);
errno = EFAULT;
goto error;
}
TX_STATS(ROZOFS_TX_SEND);
/*
** just iunlock the context and don't care about the end of transaction
** the transaction might end because of a direct error sending (tcp
** disconnection)
**
** By not releasing the tx context the end of the transaction ends upon receiving
** the tx timer expiration
*/
rozofs_tx_write_opaque_data(rozofs_tx_ctx_p,1,0); /* unlock */
/*
** OK, so now finish by starting the guard timer
*/
rozofs_tx_start_timer(rozofs_tx_ctx_p,applicative_tmo_sec);
return 0;
error:
if (rozofs_tx_ctx_p != NULL) rozofs_tx_free_from_ptr(rozofs_tx_ctx_p);
return -1;
}
/**
Encode a cache gateway message in a buffer
@param opcode code of the message
@param encode_fct encoding function
@param msg arguments to encode
@param xmit_buf buffer where to encode the message in
*/
int exp_cache_encode_common(int opcode,xdrproc_t encode_fct,void *msg, void * xmit_buf)
{
uint8_t *arg_p;
uint32_t *header_size_p;
int bufsize;
int position;
XDR xdrs;
struct rpc_msg call_msg;
uint32_t null_val = 0;
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = ruc_buf_getMaxPayloadLen(xmit_buf);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = exp_cache_xid++;
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
call_msg.rm_call.cb_prog = (uint32_t)GW_PROGRAM;
call_msg.rm_call.cb_vers = (uint32_t)GW_VERSION;
if (! xdr_callhdr(&xdrs, &call_msg))
{
return -1;
}
/*
** insert the procedure number, NULL credential and verifier
*/
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if ((*encode_fct)(&xdrs,msg) == FALSE)
{
return -1;
}
/*
** Now get the current length and fill the header of the message
*/
position = XDR_GETPOS(&xdrs);
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
return 0;
}
int rozofs_sorcli_poll_tx(af_unix_ctx_generic_t *sock_p,
uint32_t prog,uint32_t vers,
int opcode,xdrproc_t encode_fct,void *msg2encode_p,
void *xmit_buf,
int extra_len,
int applicative_tmo_sec,
sys_recv_pf_t recv_cbk,void *user_ctx_p)
{
DEBUG_FUNCTION;
uint8_t *arg_p;
uint32_t *header_size_p;
rozofs_tx_ctx_t *rozofs_tx_ctx_p = NULL;
int bufsize;
int ret;
int position;
XDR xdrs;
struct rpc_msg call_msg;
uint32_t null_val = 0;
/*
** allocate a transaction context
*/
rozofs_tx_ctx_p = rozofs_tx_alloc();
if (rozofs_tx_ctx_p == NULL)
{
/*
** out of context
** --> put a pending list for the future to avoid repluing ENOMEM
*/
TX_STATS(ROZOFS_TX_NO_CTX_ERROR);
errno = ENOMEM;
goto error;
}
/*
** store the reference of the xmit buffer in the transaction context: might be useful
** in case we want to remove it from a transmit list of the underlying network stacks
*/
rozofs_tx_save_xmitBuf(rozofs_tx_ctx_p,xmit_buf);
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = (int)ruc_buf_getMaxPayloadLen(xmit_buf);
bufsize -= sizeof(uint32_t); /* skip length*/
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_alloc_xid(rozofs_tx_ctx_p);
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)prog;
call_msg.rm_call.cb_vers = (uint32_t)vers;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** insert the procedure number, NULL credential and verifier
*/
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if ((*encode_fct)(&xdrs,msg2encode_p) == FALSE)
{
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** Now get the current length and fill the header of the message
*/
position = XDR_GETPOS(&xdrs);
/*
** add the extra_len if any
*/
position +=extra_len;
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** store the receive call back and its associated parameter
*/
rozofs_tx_ctx_p->recv_cbk = recv_cbk;
rozofs_tx_ctx_p->user_param = user_ctx_p;
rozofs_tx_write_opaque_data(rozofs_tx_ctx_p,1,1); /* lock */
/*
** now send the message
*/
ret = af_unix_generic_stream_send(sock_p,xmit_buf);
if (ret < 0)
{
TX_STATS(ROZOFS_TX_SEND_ERROR);
errno = EFAULT;
goto error;
}
TX_STATS(ROZOFS_TX_SEND);
/*
** just iunlock the context and don't care about the end of transaction
** the transaction might end because of a direct error sending (tcp
** disconnection)
**
** By not releasing the tx context the end of the transaction ends upon receiving
** the tx timer expiration
*/
rozofs_tx_write_opaque_data(rozofs_tx_ctx_p,1,0); /* unlock */
/*
** OK, so now finish by starting the guard timer
*/
rozofs_tx_start_timer(rozofs_tx_ctx_p,applicative_tmo_sec);
return 0;
error:
if (rozofs_tx_ctx_p != NULL) rozofs_tx_free_from_ptr(rozofs_tx_ctx_p);
return -1;
}
/**
* send a rpc reply: the encoding function MUST be found in xdr_result
of the gateway context
It is assumed that the xmitBuf MUST be found in xmitBuf field
In case of a success it is up to the called function to release the xmit buffer
@param p : pointer to the root transaction context used for the read
@param arg_ret : returned argument to encode
@retval none
*/
void storaged_srv_forward_read_success (rozorpc_srv_ctx_t *p,sp_read_ret_t * arg_ret)
{
int ret;
uint8_t *pbuf; /* pointer to the part that follows the header length */
uint32_t *header_len_p; /* pointer to the array that contains the length of the rpc message*/
XDR xdrs;
int len;
int cur_len;
if (p->xmitBuf == NULL)
{
ROZORPC_SRV_STATS(ROZORPC_SRV_NO_BUFFER_ERROR);
severe("no xmit buffer");
goto error;
}
/*
** create xdr structure on top of the buffer that will be used for sending the response
*/
header_len_p = (uint32_t*)ruc_buf_getPayload(p->xmitBuf);
pbuf = (uint8_t*) (header_len_p+1);
len = (int)ruc_buf_getMaxPayloadLen(p->xmitBuf);
len -= sizeof(uint32_t);
xdrmem_create(&xdrs,(char*)pbuf,len,XDR_ENCODE);
/*
** just encode the header
*/
if (rozofs_encode_rpc_reply(&xdrs,(xdrproc_t)NULL,(caddr_t)NULL,p->src_transaction_id) != TRUE)
{
ROZORPC_SRV_STATS(ROZORPC_SRV_ENCODING_ERROR);
severe("rpc reply encoding error");
goto error;
}
/*
** OK now starts encoding the response
*/
xdr_sp_status_t (&xdrs, &arg_ret->status);
xdr_uint32_t (&xdrs, &arg_ret->sp_read_ret_t_u.rsp.filler);
xdr_uint32_t (&xdrs, &arg_ret->sp_read_ret_t_u.rsp.bins.bins_len);
/*
** skip the bins
*/
cur_len = xdr_getpos(&xdrs) ;
cur_len += arg_ret->sp_read_ret_t_u.rsp.bins.bins_len;
xdr_setpos(&xdrs,cur_len);
/*
** encode the length of the file
*/
xdr_uint64_t (&xdrs, &arg_ret->sp_read_ret_t_u.rsp.file_size);
#if 0 // for future usage with distributed cache
/*
** encode the optim array
*/
xdr_bytes (&xdrs, (char **)&arg_ret->sp_read_ret_t_u.rsp.optim.optim_val,
(u_int *) &arg_ret->sp_read_ret_t_u.rsp.optim.optim_len, ~0);
#endif
/*
** compute the total length of the message for the rpc header and add 4 bytes more bytes for
** the ruc buffer to take care of the header length of the rpc message.
*/
int total_len = xdr_getpos(&xdrs) ;
*header_len_p = htonl(0x80000000 | total_len);
total_len +=sizeof(uint32_t);
ruc_buf_setPayloadLen(p->xmitBuf,total_len);
/*
** Get the callback for sending back the response:
** A callback is needed since the request for read might be local or remote
*/
ret = af_unix_generic_send_stream_with_idx((int)p->socketRef,p->xmitBuf);
if (ret == 0)
{
/**
* success so remove the reference of the xmit buffer since it is up to the called
* function to release it
*/
ROZORPC_SRV_STATS(ROZORPC_SRV_SEND);
p->xmitBuf = NULL;
}
else
{
ROZORPC_SRV_STATS(ROZORPC_SRV_SEND_ERROR);
}
error:
return;
}
int rozofs_sorcli_send_rq_common(uint32_t lbg_id,uint32_t timeout_sec, uint32_t prog,uint32_t vers,
int opcode,xdrproc_t encode_fct,void *msg2encode_p,
void *xmit_buf,
uint32_t seqnum,
uint32_t opaque_value_idx1,
int extra_len,
sys_recv_pf_t recv_cbk,void *user_ctx_p)
{
DEBUG_FUNCTION;
uint8_t *arg_p;
uint32_t *header_size_p;
rozofs_tx_ctx_t *rozofs_tx_ctx_p = NULL;
int bufsize;
int ret;
int position;
XDR xdrs;
struct rpc_msg call_msg;
uint32_t null_val = 0;
/*
** allocate a transaction context
*/
rozofs_tx_ctx_p = rozofs_tx_alloc();
if (rozofs_tx_ctx_p == NULL)
{
/*
** out of context
** --> put a pending list for the future to avoid repluing ENOMEM
*/
TX_STATS(ROZOFS_TX_NO_CTX_ERROR);
errno = ENOMEM;
goto error;
}
/*
** store the reference of the xmit buffer in the transaction context: might be useful
** in case we want to remove it from a transmit list of the underlying network stacks
*/
rozofs_tx_save_xmitBuf(rozofs_tx_ctx_p,xmit_buf);
/*
** get the pointer to the payload of the buffer
*/
header_size_p = (uint32_t*) ruc_buf_getPayload(xmit_buf);
arg_p = (uint8_t*)(header_size_p+1);
/*
** create the xdr_mem structure for encoding the message
*/
bufsize = (int)ruc_buf_getMaxPayloadLen(xmit_buf);
xdrmem_create(&xdrs,(char*)arg_p,bufsize,XDR_ENCODE);
/*
** fill in the rpc header
*/
call_msg.rm_direction = CALL;
/*
** allocate a xid for the transaction
*/
call_msg.rm_xid = rozofs_tx_alloc_xid(rozofs_tx_ctx_p);
call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
/* XXX: prog and vers have been long historically :-( */
call_msg.rm_call.cb_prog = (uint32_t)prog;
call_msg.rm_call.cb_vers = (uint32_t)vers;
if (! xdr_callhdr(&xdrs, &call_msg))
{
/*
** THIS MUST NOT HAPPEN
*/
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** insert the procedure number, NULL credential and verifier
*/
XDR_PUTINT32(&xdrs, (int32_t *)&opcode);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
XDR_PUTINT32(&xdrs, (int32_t *)&null_val);
/*
** ok now call the procedure to encode the message
*/
if ((*encode_fct)(&xdrs,msg2encode_p) == FALSE)
{
TX_STATS(ROZOFS_TX_ENCODING_ERROR);
errno = EPROTO;
goto error;
}
/*
** Now get the current length and fill the header of the message
*/
position = XDR_GETPOS(&xdrs);
/*
** add the extra_len if any
*/
position +=extra_len;
/*
** update the length of the message : must be in network order
*/
*header_size_p = htonl(0x80000000 | position);
/*
** set the payload length in the xmit buffer
*/
int total_len = sizeof(*header_size_p)+ position;
ruc_buf_setPayloadLen(xmit_buf,total_len);
/*
** store the receive call back and its associated parameter
*/
rozofs_tx_ctx_p->recv_cbk = recv_cbk;
rozofs_tx_ctx_p->user_param = user_ctx_p;
/*
** store the sequence number in one of the opaque user data array of the transaction
*/
rozofs_tx_write_opaque_data( rozofs_tx_ctx_p,0,seqnum);
rozofs_tx_write_opaque_data( rozofs_tx_ctx_p,1,opaque_value_idx1);
rozofs_tx_write_opaque_data( rozofs_tx_ctx_p,2,lbg_id);
/*
** now send the message
*/
// STORCLI_STOP_NORTH_PROF_SRV((rozofs_storcli_ctx_t*)user_ctx_p,read_req,0); // FDL
#ifndef TEST_STORCLI_TEST
ret = north_lbg_send(lbg_id,xmit_buf);
#else
ret = test_north_lbg_send(lbg_id,xmit_buf);
#endif
if (ret < 0)
{
TX_STATS(ROZOFS_TX_SEND_ERROR);
errno = EFAULT;
goto error;
}
TX_STATS(ROZOFS_TX_SEND);
/*
** OK, so now finish by starting the guard timer
*/
rozofs_tx_start_timer(rozofs_tx_ctx_p,timeout_sec);
return 0;
error:
if (rozofs_tx_ctx_p != NULL) rozofs_tx_free_from_ptr(rozofs_tx_ctx_p);
return -1;
}
/**
* send a error read reply by using the receiver buffer
@param socket_ctx_idx: index of the TCP connection
@param recv_buf: pointer to the ruc_buffer that contains the message
@param rozofs_storcli_remote_rsp_cbk: callback for sending out the response
@param user_param : pointer to a user opaque parameter (non significant for a remote access)
@param error : error code
@retval none
*/
void rozofs_storcli_reply_error_with_recv_buf(uint32_t socket_ctx_idx,
void *recv_buf,
void *user_param,
rozofs_storcli_resp_pf_t rozofs_storcli_remote_rsp_cbk,
int error)
{
rozofs_rpc_call_hdr_with_sz_t *com_hdr_p;
uint32_t msg_len; /* length of the rpc messsage including the header length */
rozofs_rpc_call_hdr_t hdr; /* structure that contains the rpc header in host format */
int ret;
uint8_t *pbuf; /* pointer to the part that follows the header length */
uint32_t *header_len_p; /* pointer to the array that contains the length of the rpc message*/
XDR xdrs;
int len;
storcli_status_ret_t status;
status.status = STORCLI_FAILURE;
status.storcli_status_ret_t_u.error = error;
/*
** Get the full length of the message and adjust it the the length of the applicative part (RPC header+application msg)
*/
msg_len = ruc_buf_getPayloadLen(recv_buf);
msg_len -=sizeof(uint32_t);
/*
** Get the payload of the receive buffer and set the pointer to array that describes the read request
*/
com_hdr_p = (rozofs_rpc_call_hdr_with_sz_t*) ruc_buf_getPayload(recv_buf);
memcpy(&hdr,&com_hdr_p->hdr,sizeof(rozofs_rpc_call_hdr_t));
/*
** swap the rpc header
*/
scv_call_hdr_ntoh(&hdr);
/*
** create xdr structure on top of the buffer that will be used for sending the response
*/
header_len_p = (uint32_t*)ruc_buf_getPayload(recv_buf);
pbuf = (uint8_t*) (header_len_p+1);
len = (int)ruc_buf_getMaxPayloadLen(recv_buf);
len -= sizeof(uint32_t);
xdrmem_create(&xdrs,(char*)pbuf,len,XDR_ENCODE);
if (rozofs_encode_rpc_reply(&xdrs,(xdrproc_t)xdr_sp_status_ret_t,(caddr_t)&status,hdr.hdr.xid) != TRUE)
{
severe("rpc reply encoding error");
goto error;
}
/*
** compute the total length of the message for the rpc header and add 4 bytes more bytes for
** the ruc buffer to take care of the header length of the rpc message.
*/
int total_len = xdr_getpos(&xdrs) ;
*header_len_p = htonl(0x80000000 | total_len);
total_len +=sizeof(uint32_t);
ruc_buf_setPayloadLen(recv_buf,total_len);
/*
** Get the callback for sending back the response:
** A callback is needed since the request for read might be local or remote
*/
ret = (*rozofs_storcli_remote_rsp_cbk)(recv_buf,socket_ctx_idx,user_param);
if (ret == 0)
{
return;
}
error:
ruc_buf_freeBuffer(recv_buf);
return;
}
void rozofs_storcli_read_reply_success(rozofs_storcli_ctx_t *p)
{
int ret;
uint8_t *pbuf; /* pointer to the part that follows the header length */
uint32_t *header_len_p; /* pointer to the array that contains the length of the rpc message*/
XDR xdrs;
int len;
storcli_status_t status = STORCLI_SUCCESS;
int data_len;
uint8_t eof_flag = 0;
/*
** create xdr structure on top of the buffer that will be used for sending the response
*/
header_len_p = (uint32_t*)ruc_buf_getPayload(p->xmitBuf);
pbuf = (uint8_t*) (header_len_p+1);
len = (int)ruc_buf_getMaxPayloadLen(p->xmitBuf);
len -= sizeof(uint32_t);
xdrmem_create(&xdrs,(char*)pbuf,len,XDR_ENCODE);
if (rozofs_encode_rpc_reply(&xdrs,(xdrproc_t)xdr_storcli_status_t,(caddr_t)&status,p->src_transaction_id) != TRUE)
{
severe("rpc reply encoding error");
goto error;
}
/*
** Encode the length of returned data
*/
data_len = rozofs_storcli_transform_get_read_len_in_bytes(p->block_ctx_table,
p->effective_number_of_blocks,
&eof_flag);
STORCLI_STOP_NORTH_PROF(p,read,data_len);
/*
** skip the alignment
*/
int position;
position = xdr_getpos(&xdrs);
position += sizeof(uint32_t);
xdr_setpos(&xdrs,position);
XDR_PUTINT32(&xdrs, (int32_t *)&data_len);
/*
** round up data_len to 4 bytes alignment
*/
if ((data_len%4)!= 0) data_len = (data_len &(~0x3))+4;
/*
** compute the total length of the message for the rpc header and add 4 bytes more bytes for
** the ruc buffer to take care of the header length of the rpc message.
*/
int total_len = xdr_getpos(&xdrs)+data_len ;
*header_len_p = htonl(0x80000000 | total_len);
total_len +=sizeof(uint32_t);
ruc_buf_setPayloadLen(p->xmitBuf,total_len);
/*
** Clear the reference of the seqnum to prevent any late response to be processed
** by setting seqnum to 0 any late response is ignored and the associated ressources
** will released (buffer associated with the response). This typically permits to
** avoid sending again the response while this has already been done
*/
p->read_seqnum = 0;
/*
** Get the callback for sending back the response:
** A callback is needed since the request for read might be local or remote
*/
ret = (*p->response_cbk)(p->xmitBuf,p->socketRef,p->user_param);
if (ret == 0)
{
/**
* success so remove the reference of the xmit buffer since it is up to the called
* function to release it
*/
p->xmitBuf = NULL;
}
error:
// #warning need to consider the case of a local read triggers by a write request. Without a guard time the write working can be lost!!
return;
}
