int rozofs_storcli_north_interface_init(uint32_t eid,uint16_t rozofsmount_instance,uint32_t instance,
int read_write_buf_count,int read_write_buf_sz)
{
int ret = 0;
char sunpath[AF_UNIX_SOCKET_NAME_SIZE];
rozofs_storcli_read_write_buf_count = read_write_buf_count;
rozofs_storcli_read_write_buf_sz = read_write_buf_sz ;
while(1)
{
storcli_north_buffer_pool_p = ruc_buf_poolCreate(rozofs_storcli_read_write_buf_count,rozofs_storcli_read_write_buf_sz);
if (storcli_north_buffer_pool_p == NULL)
{
ret = -1;
severe( "ruc_buf_poolCreate(%d,%d)", rozofs_storcli_read_write_buf_count, rozofs_storcli_read_write_buf_sz );
break;
}
/*
** create the listening af unix socket on the north interface
*/
af_unix_test_family.rpc_recv_max_sz = rozofs_storcli_read_write_buf_sz;
sprintf(sunpath,"%s%d.%d_lbg%d_inst_1",ROZOFS_SOCK_FAMILY_STORCLI_NORTH_SUNPATH,eid,rozofsmount_instance,instance);
// sprintf(sunpath,"%s%d.%d_inst_%d",ROZOFS_SOCK_FAMILY_STORCLI_NORTH_SUNPATH,eid,rozofsmount_instance,instance);
ret = af_unix_sock_listening_create("STORCLI_NORTH",
sunpath,
&af_unix_test_family
);
break;
}
return ret;
}
/*__________________________________________________________________________
* Initialize the disk thread interface
*
* @param hostname storio hostname (for tests)
* @param nb_threads Number of threads that can process the disk requests
* @param nb_buffer Number of buffer for sending and number of receiving buffer
*
* @retval 0 on success -1 in case of error
*/
int storio_disk_thread_intf_create(char * hostname, int instance_id, int nb_threads, int nb_buffer) {
af_unix_disk_thread_count = nb_threads;
af_unix_disk_pool_send = ruc_buf_poolCreate(nb_buffer,sizeof(storio_disk_thread_msg_t));
if (af_unix_disk_pool_send == NULL) {
fatal("storio_disk_thread_intf_create af_unix_disk_pool_send (%d,%d)", nb_buffer, (int)sizeof(storio_disk_thread_msg_t));
return -1;
}
ruc_buffer_debug_register_pool("diskSendPool",af_unix_disk_pool_send);
af_unix_disk_pool_recv = ruc_buf_poolCreate(1,sizeof(storio_disk_thread_msg_t));
if (af_unix_disk_pool_recv == NULL) {
fatal("storio_disk_thread_intf_create af_unix_disk_pool_recv (1,%d)", (int)sizeof(storio_disk_thread_msg_t));
return -1;
}
ruc_buffer_debug_register_pool("diskRecvPool",af_unix_disk_pool_recv);
/*
** init of the AF_UNIX sockaddr associated with the south socket (socket used for disk response receive)
*/
storio_set_socket_name_with_hostname(&storio_south_socket_name,ROZOFS_SOCK_FAMILY_DISK_SOUTH,hostname, instance_id);
/*
** hostname is required for the case when several storaged run on the same server
** as is the case of test on one server only
*/
af_unix_disk_south_socket_ref = af_unix_disk_response_socket_create(storio_south_socket_name.sun_path);
if (af_unix_disk_south_socket_ref < 0) {
fatal("storio_create_disk_thread_intf af_unix_sock_create(%s) %s",storio_south_socket_name.sun_path, strerror(errno));
return -1;
}
/*
** init of the AF_UNIX sockaddr associated with the north socket (socket used for disk request receive)
*/
storio_set_socket_name_with_hostname(&storio_north_socket_name,ROZOFS_SOCK_FAMILY_DISK_NORTH,hostname,instance_id);
uma_dbg_addTopic_option("diskThreads", disk_thread_debug,UMA_DBG_OPTION_RESET);
/*
** attach the callback on socket controller
*/
ruc_sockCtrl_attach_applicative_poller(af_unix_disk_scheduler_entry_point);
return storio_disk_thread_create(hostname, nb_threads, instance_id);
}
/**
* Init of the pseudo fuse thread
@param ch : initial channel
@param se : initial session
@param rozofs_fuse_buffer_count : number of request buffers (corresponds to the number of fuse save context)
@retval 0 on success
@retval -1 on error
*/
int rozofs_fuse_init(struct fuse_chan *ch,struct fuse_session *se,int rozofs_fuse_buffer_count)
{
int status = 0;
// return 0;
int fileflags;
rozofs_fuse_ctx_p = malloc(sizeof (rozofs_fuse_ctx_t));
if (rozofs_fuse_ctx_p == NULL)
{
/*
** cannot allocate memory for fuse rozofs context
*/
return -1;
}
/*
** clear read/write merge stats table
*/
memset(rozofs_write_merge_stats_tab,0,sizeof(uint64_t)*RZ_FUSE_WRITE_MAX);
memset (rozofs_write_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset (rozofs_read_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset(&rozofs_fuse_read_write_stats_buf,0,sizeof(rozofs_fuse_read_write_stats));
/*
** init of the context
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = NULL;
rozofs_fuse_ctx_p->ch = NULL;
rozofs_fuse_ctx_p->se = se;
rozofs_fuse_ctx_p->bufsize = 0;
rozofs_fuse_ctx_p->buf_fuse_req_p = NULL;
rozofs_fuse_ctx_p->initBufCount = rozofs_fuse_buffer_count;
while (1)
{
/*
** get the receive buffer size for former channel in order to create the request distributor
*/
int bufsize = fuse_chan_bufsize(ch);
rozofs_fuse_ctx_p->bufsize = bufsize;
/*
** create the pool
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = ruc_buf_poolCreate(rozofs_fuse_buffer_count,sizeof(rozofs_fuse_save_ctx_t));
if (rozofs_fuse_ctx_p->fuseReqPoolRef == NULL)
{
ERRLOG "rozofs_fuse_init buffer pool creation error(%d,%d)", (int)rozofs_fuse_buffer_count, (int)sizeof(rozofs_fuse_save_ctx_t) ENDERRLOG ;
status = -1;
break;
}
/*
** allocate a buffer for receiving the fuse request
*/
rozofs_fuse_ctx_p->buf_fuse_req_p = malloc(bufsize);
if (rozofs_fuse_ctx_p == NULL)
{
ERRLOG "rozofs_fuse_init out of memory %d", bufsize ENDERRLOG ;
status = -1;
break;
}
/*
** get the fd of the channel
*/
rozofs_fuse_ctx_p->fd = fuse_chan_fd(ch);
/*
** create a new channel with the specific operation for rozofs (non-blocking)
*/
rozofs_fuse_ctx_p->ch = fuse_chan_new(&rozofs_fuse_ch_ops,fuse_chan_fd(ch),fuse_chan_bufsize(ch),rozofs_fuse_ctx_p);
if (rozofs_fuse_ctx_p->ch == NULL)
{
ERRLOG "rozofs_fuse_init fuse_chan_new error" ENDERRLOG ;
status = -1;
break;
}
/*
** remove the association between the initial session and channel
*/
fuse_session_remove_chan(ch);
/*
** OK, now add the new channel
*/
fuse_session_add_chan(se,rozofs_fuse_ctx_p->ch );
/*
** set the channel in non blocking mode
*/
if((fileflags=fcntl(rozofs_fuse_ctx_p->fd,F_GETFL,0))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
if((fcntl(rozofs_fuse_ctx_p->fd,F_SETFL,fileflags|O_NDELAY))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
/*
** perform the connection with the socket controller
*/
/*
** OK, we are almost done, just need to connect with the socket controller
*/
rozofs_fuse_ctx_p->connectionId = ruc_sockctl_connect(rozofs_fuse_ctx_p->fd, // Reference of the socket
"rozofs_fuse", // name of the socket
3, // Priority within the socket controller
(void*)rozofs_fuse_ctx_p, // user param for socketcontroller callback
&rozofs_fuse_callBack_sock); // Default callbacks
if (rozofs_fuse_ctx_p->connectionId == NULL)
{
/*
** Fail to connect with the socket controller
*/
RUC_WARNING(-1);
status = -1;
break;
}
status = 0;
break;
}
uma_dbg_addTopic("fuse", rozofs_fuse_show);
return status;
}
/**
* Init of the pseudo fuse thread
@param ch : initial channel
@param se : initial session
@param rozofs_fuse_buffer_count : number of request buffers (corresponds to the number of fuse save context)
@retval 0 on success
@retval -1 on error
*/
int rozofs_fuse_init(struct fuse_chan *ch,struct fuse_session *se,int rozofs_fuse_buffer_count)
{
int status = 0;
int i;
// return 0;
fuse_sharemem_init_done = 0;
int fileflags;
rozofs_fuse_ctx_p = malloc(sizeof (rozofs_fuse_ctx_t));
if (rozofs_fuse_ctx_p == NULL)
{
/*
** cannot allocate memory for fuse rozofs context
*/
return -1;
}
memset(rozofs_fuse_ctx_p,0,sizeof (rozofs_fuse_ctx_t));
/*
** clear read/write merge stats table
*/
memset(rozofs_write_merge_stats_tab,0,sizeof(uint64_t)*RZ_FUSE_WRITE_MAX);
memset (rozofs_write_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset (rozofs_read_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset(&rozofs_fuse_read_write_stats_buf,0,sizeof(rozofs_fuse_read_write_stats));
/*
** init of the context
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = NULL;
rozofs_fuse_ctx_p->ch = NULL;
rozofs_fuse_ctx_p->se = se;
rozofs_fuse_ctx_p->bufsize = 0;
rozofs_fuse_ctx_p->buf_fuse_req_p = NULL;
rozofs_fuse_ctx_p->dir_attr_invalidate = 0;
rozofs_fuse_ctx_p->initBufCount = rozofs_fuse_buffer_count;
while (1)
{
/*
** get the receive buffer size for former channel in order to create the request distributor:
** note: by default the fuse buffer is 4K+128K: for RozoFS the payload can reach 512K (x4)
*/
int bufsize = fuse_chan_bufsize(ch)*4;
rozofs_fuse_ctx_p->bufsize = bufsize;
/*
** create the distributor fro receiving data from fuse kernel
*/
status = rozofs_fuse_init_rcv_buffer_pool(ROZOFS_FUSE_RECV_BUF_COUNT,bufsize);
if (status < 0)
{
severe( "rozofs_fuse_init fuse buffer pool creation error(%d,%d)", (int)ROZOFS_FUSE_RECV_BUF_COUNT, (int)bufsize ) ;
status = -1;
break;
}
/*
** create the pool
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = ruc_buf_poolCreate(rozofs_fuse_buffer_count,sizeof(rozofs_fuse_save_ctx_t));
if (rozofs_fuse_ctx_p->fuseReqPoolRef == NULL)
{
severe( "rozofs_fuse_init buffer pool creation error(%d,%d)", (int)rozofs_fuse_buffer_count, (int)sizeof(rozofs_fuse_save_ctx_t) ) ;
status = -1;
break;
}
ruc_buffer_debug_register_pool("fuseCtx", rozofs_fuse_ctx_p->fuseReqPoolRef);
/*
** Allocate a head of list for queueing pending lookup requests
*/
for (i=0; i< ROZOFS_MAX_LKUP_QUEUE ; i++)
{
ruc_listHdrInit(&rozofs_lookup_queue[i]);
}
/*
** allocate a buffer for receiving the fuse request
*/
rozofs_fuse_ctx_p->buf_fuse_req_p = malloc(bufsize);
if (rozofs_fuse_ctx_p == NULL)
{
severe( "rozofs_fuse_init out of memory %d", bufsize ) ;
status = -1;
break;
}
/*
** get the fd of the channel
*/
rozofs_fuse_ctx_p->fd = fuse_chan_fd(ch);
/*
** wait the end of the share memroy init prior providing it to fuse
*/
while (rozofs_shared_mem_init_done == 0) sleep(1);
/*
** create a new channel with the specific operation for rozofs (non-blocking)
*/
rozofs_fuse_ctx_p->ch = fuse_chan_new(&rozofs_fuse_ch_ops,fuse_chan_fd(ch),bufsize,rozofs_fuse_ctx_p);
if (rozofs_fuse_ctx_p->ch == NULL)
{
severe( "rozofs_fuse_init fuse_chan_new error" ) ;
status = -1;
break;
}
/*
** remove the association between the initial session and channel
*/
fuse_session_remove_chan(ch);
/*
** OK, now add the new channel
*/
fuse_session_add_chan(se,rozofs_fuse_ctx_p->ch );
/*
** set the channel in non blocking mode
*/
if((fileflags=fcntl(rozofs_fuse_ctx_p->fd,F_GETFL,0))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
if((fcntl(rozofs_fuse_ctx_p->fd,F_SETFL,fileflags|O_NDELAY))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
/*
** send XON to the fuse channel
*/
{
int ret;
rozofs_fuse_ctx_p->ioctl_supported = 1;
rozofs_fuse_ctx_p->data_xon = 1;
while(1)
{
ret = ioctl(rozofs_fuse_ctx_p->fd,1,NULL);
if (ret < 0)
{
warning("ioctl error %s",strerror(errno));
rozofs_fuse_ctx_p->ioctl_supported = 0;
break;
}
if (rozofs_fuse_ctx_p->dir_attr_invalidate == 0)
{
ret = ioctl(rozofs_fuse_ctx_p->fd,3,NULL);
if (ret < 0)
{
warning("ioctl error %s",strerror(errno));
// rozofs_fuse_ctx_p->ioctl_supported = 0;
rozofs_fuse_ctx_p->dir_attr_invalidate = 1;
break;
}
}
break;
}
}
/*
** perform the connection with the socket controller
*/
/*
** OK, we are almost done, just need to connect with the socket controller
*/
rozofs_fuse_ctx_p->connectionId = ruc_sockctl_connect(rozofs_fuse_ctx_p->fd, // Reference of the socket
"rozofs_fuse", // name of the socket
3, // Priority within the socket controller
(void*)rozofs_fuse_ctx_p, // user param for socketcontroller callback
&rozofs_fuse_callBack_sock); // Default callbacks
if (rozofs_fuse_ctx_p->connectionId == NULL)
{
/*
** Fail to connect with the socket controller
*/
RUC_WARNING(-1);
status = -1;
break;
}
rozofs_fuse_get_ticker();
status = 0;
break;
}
/*
** attach the callback on socket controller
*/
//#warning no poller
ruc_sockCtrl_attach_applicative_poller(rozofs_fuse_scheduler_entry_point);
for(i = 0; i < 3;i++) fuse_profile[i] = 0;
uma_dbg_addTopic("fuse", rozofs_fuse_show);
return status;
}
/**
af_unix_module_init
create the Transaction context pool
@param : af_unix_ctx_count : number of Transaction context
@param : max_xmit_buf_count : number of xmit buffers
@param : max_xmit_buf_size : xmit buffer size
@param : max_recv_buf_count : number of receive buffers
@param : max_recv_buf_count : receive buffer size
@retval : RUC_OK : done
@retval RUC_NOK : out of memory
*/
uint32_t af_unix_module_init(uint32_t af_unix_ctx_count,
int max_xmit_buf_count, int max_xmit_buf_size,
int max_recv_buf_count, int max_recv_buf_size
) {
af_unix_ctx_generic_t *p;
uint32_t idxCur;
ruc_obj_desc_t *pnext;
uint32_t ret = RUC_OK;
af_unix_xmit_buf_count = max_xmit_buf_count;
af_unix_xmit_buf_size = max_xmit_buf_size;
af_unix_recv_buf_count = max_recv_buf_count;
af_unix_recv_buf_size = max_recv_buf_size;
af_unix_context_allocated = 0;
af_unix_context_count = af_unix_ctx_count;
while (1) {
af_unix_buffer_pool_tb[0] = ruc_buf_poolCreate(af_unix_xmit_buf_count, af_unix_xmit_buf_size);
if (af_unix_buffer_pool_tb[0] == NULL) {
ret = RUC_NOK;
severe( "xmit ruc_buf_poolCreate(%d,%d)", af_unix_xmit_buf_count, af_unix_xmit_buf_size );
break;
}
af_unix_buffer_pool_tb[1] = ruc_buf_poolCreate(af_unix_recv_buf_count, af_unix_recv_buf_size);
if (af_unix_buffer_pool_tb[1] == NULL) {
ret = RUC_NOK;
severe( "rcv ruc_buf_poolCreate(%d,%d)", af_unix_recv_buf_count, af_unix_recv_buf_size );
break;
}
af_unix_context_freeListHead = (af_unix_ctx_generic_t*) NULL;
/*
** create the active list
*/
ruc_listHdrInit((ruc_obj_desc_t*) & af_unix_context_activeListHead);
/*
** create the af unix context pool
*/
af_unix_context_freeListHead = (af_unix_ctx_generic_t*) ruc_listCreate(af_unix_ctx_count, sizeof (af_unix_ctx_generic_t));
if (af_unix_context_freeListHead == (af_unix_ctx_generic_t*) NULL) {
/*
** out of memory
*/
RUC_WARNING(af_unix_ctx_count * sizeof (af_unix_ctx_generic_t));
return RUC_NOK;
}
/*
** store the pointer to the first context
*/
af_unix_context_pfirst = af_unix_context_freeListHead;
/*
** initialize each entry of the free list
*/
idxCur = 0;
pnext = (ruc_obj_desc_t*) NULL;
while ((p = (af_unix_ctx_generic_t*) ruc_objGetNext((ruc_obj_desc_t*) af_unix_context_freeListHead,
&pnext))
!= (af_unix_ctx_generic_t*) NULL) {
p->index = idxCur;
p->free = TRUE;
af_unix_ctxInit(p, TRUE);
idxCur++;
}
af_unix_debug_init();
break;
}
return ret;
}
