/*__________________________________________________________________________
* Initialize the disk thread interface
*
* @param hostname storio hostname (for tests)
* @param nb_threads Number of threads that can process the disk requests
* @param nb_buffer Number of buffer for sending and number of receiving buffer
*
* @retval 0 on success -1 in case of error
*/
int storio_disk_thread_intf_create(char * hostname, int instance_id, int nb_threads, int nb_buffer) {
af_unix_disk_thread_count = nb_threads;
af_unix_disk_pool_send = ruc_buf_poolCreate(nb_buffer,sizeof(storio_disk_thread_msg_t));
if (af_unix_disk_pool_send == NULL) {
fatal("storio_disk_thread_intf_create af_unix_disk_pool_send (%d,%d)", nb_buffer, (int)sizeof(storio_disk_thread_msg_t));
return -1;
}
ruc_buffer_debug_register_pool("diskSendPool",af_unix_disk_pool_send);
af_unix_disk_pool_recv = ruc_buf_poolCreate(1,sizeof(storio_disk_thread_msg_t));
if (af_unix_disk_pool_recv == NULL) {
fatal("storio_disk_thread_intf_create af_unix_disk_pool_recv (1,%d)", (int)sizeof(storio_disk_thread_msg_t));
return -1;
}
ruc_buffer_debug_register_pool("diskRecvPool",af_unix_disk_pool_recv);
/*
** init of the AF_UNIX sockaddr associated with the south socket (socket used for disk response receive)
*/
storio_set_socket_name_with_hostname(&storio_south_socket_name,ROZOFS_SOCK_FAMILY_DISK_SOUTH,hostname, instance_id);
/*
** hostname is required for the case when several storaged run on the same server
** as is the case of test on one server only
*/
af_unix_disk_south_socket_ref = af_unix_disk_response_socket_create(storio_south_socket_name.sun_path);
if (af_unix_disk_south_socket_ref < 0) {
fatal("storio_create_disk_thread_intf af_unix_sock_create(%s) %s",storio_south_socket_name.sun_path, strerror(errno));
return -1;
}
/*
** init of the AF_UNIX sockaddr associated with the north socket (socket used for disk request receive)
*/
storio_set_socket_name_with_hostname(&storio_north_socket_name,ROZOFS_SOCK_FAMILY_DISK_NORTH,hostname,instance_id);
uma_dbg_addTopic_option("diskThreads", disk_thread_debug,UMA_DBG_OPTION_RESET);
/*
** attach the callback on socket controller
*/
ruc_sockCtrl_attach_applicative_poller(af_unix_disk_scheduler_entry_point);
return storio_disk_thread_create(hostname, nb_threads, instance_id);
}
int rozofs_create_shared_memory(int key_instance,int pool_idx,uint32_t buf_nb, uint32_t buf_sz)
{
key_t key = 0x524F5A00 | key_instance;
rozofs_storcli_shared_mem[pool_idx].key = key;
rozofs_storcli_shared_mem[pool_idx].buf_sz = buf_sz;
rozofs_storcli_shared_mem[pool_idx].buf_count = buf_nb;
rozofs_storcli_shared_mem[pool_idx].pool_p = ruc_buf_poolCreate_shared(buf_nb,buf_sz,key);
if (rozofs_storcli_shared_mem[pool_idx].pool_p == NULL) return -1;
switch(pool_idx) {
case SHAREMEM_IDX_READ: ruc_buffer_debug_register_pool("read_pool_shared", rozofs_storcli_shared_mem[pool_idx].pool_p); break;
case SHAREMEM_IDX_WRITE: ruc_buffer_debug_register_pool("write_pool_shared", rozofs_storcli_shared_mem[pool_idx].pool_p); break;
default:
break;
}
rozofs_storcli_shared_mem[pool_idx].data_p = ruc_buf_get_pool_base_data(rozofs_storcli_shared_mem[pool_idx].pool_p);
return 0;
}
/**
* Init of the pseudo fuse thread
@param ch : initial channel
@param se : initial session
@param rozofs_fuse_buffer_count : number of request buffers (corresponds to the number of fuse save context)
@retval 0 on success
@retval -1 on error
*/
int rozofs_fuse_init(struct fuse_chan *ch,struct fuse_session *se,int rozofs_fuse_buffer_count)
{
int status = 0;
// return 0;
int fileflags;
rozofs_fuse_ctx_p = malloc(sizeof (rozofs_fuse_ctx_t));
if (rozofs_fuse_ctx_p == NULL)
{
/*
** cannot allocate memory for fuse rozofs context
*/
return -1;
}
/*
** clear read/write merge stats table
*/
memset(rozofs_write_merge_stats_tab,0,sizeof(uint64_t)*RZ_FUSE_WRITE_MAX);
memset (rozofs_write_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset (rozofs_read_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset(&rozofs_fuse_read_write_stats_buf,0,sizeof(rozofs_fuse_read_write_stats));
/*
** init of the context
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = NULL;
rozofs_fuse_ctx_p->ch = NULL;
rozofs_fuse_ctx_p->se = se;
rozofs_fuse_ctx_p->bufsize = 0;
rozofs_fuse_ctx_p->buf_fuse_req_p = NULL;
rozofs_fuse_ctx_p->initBufCount = rozofs_fuse_buffer_count;
while (1)
{
/*
** get the receive buffer size for former channel in order to create the request distributor
*/
int bufsize = fuse_chan_bufsize(ch);
rozofs_fuse_ctx_p->bufsize = bufsize;
/*
** create the pool
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = ruc_buf_poolCreate(rozofs_fuse_buffer_count,sizeof(rozofs_fuse_save_ctx_t));
if (rozofs_fuse_ctx_p->fuseReqPoolRef == NULL)
{
severe( "rozofs_fuse_init buffer pool creation error(%d,%d)", (int)rozofs_fuse_buffer_count, (int)sizeof(rozofs_fuse_save_ctx_t) ) ;
status = -1;
break;
}
ruc_buffer_debug_register_pool("fuseCtx", rozofs_fuse_ctx_p->fuseReqPoolRef);
/*
** allocate a buffer for receiving the fuse request
*/
rozofs_fuse_ctx_p->buf_fuse_req_p = malloc(bufsize);
if (rozofs_fuse_ctx_p == NULL)
{
severe( "rozofs_fuse_init out of memory %d", bufsize ) ;
status = -1;
break;
}
/*
** get the fd of the channel
*/
rozofs_fuse_ctx_p->fd = fuse_chan_fd(ch);
/*
** create a new channel with the specific operation for rozofs (non-blocking)
*/
rozofs_fuse_ctx_p->ch = fuse_chan_new(&rozofs_fuse_ch_ops,fuse_chan_fd(ch),fuse_chan_bufsize(ch),rozofs_fuse_ctx_p);
if (rozofs_fuse_ctx_p->ch == NULL)
{
severe( "rozofs_fuse_init fuse_chan_new error" ) ;
status = -1;
break;
}
/*
** remove the association between the initial session and channel
*/
fuse_session_remove_chan(ch);
/*
** OK, now add the new channel
*/
fuse_session_add_chan(se,rozofs_fuse_ctx_p->ch );
/*
** set the channel in non blocking mode
*/
if((fileflags=fcntl(rozofs_fuse_ctx_p->fd,F_GETFL,0))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
if((fcntl(rozofs_fuse_ctx_p->fd,F_SETFL,fileflags|O_NDELAY))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
/*
** perform the connection with the socket controller
*/
/*
** OK, we are almost done, just need to connect with the socket controller
*/
rozofs_fuse_ctx_p->connectionId = ruc_sockctl_connect(rozofs_fuse_ctx_p->fd, // Reference of the socket
"rozofs_fuse", // name of the socket
3, // Priority within the socket controller
(void*)rozofs_fuse_ctx_p, // user param for socketcontroller callback
&rozofs_fuse_callBack_sock); // Default callbacks
if (rozofs_fuse_ctx_p->connectionId == NULL)
{
/*
** Fail to connect with the socket controller
*/
RUC_WARNING(-1);
status = -1;
break;
}
status = 0;
break;
}
uma_dbg_addTopic("fuse", rozofs_fuse_show);
return status;
}
/**
* Init of the pseudo fuse thread
@param ch : initial channel
@param se : initial session
@param rozofs_fuse_buffer_count : number of request buffers (corresponds to the number of fuse save context)
@retval 0 on success
@retval -1 on error
*/
int rozofs_fuse_init(struct fuse_chan *ch,struct fuse_session *se,int rozofs_fuse_buffer_count)
{
int status = 0;
int i;
// return 0;
fuse_sharemem_init_done = 0;
int fileflags;
rozofs_fuse_ctx_p = malloc(sizeof (rozofs_fuse_ctx_t));
if (rozofs_fuse_ctx_p == NULL)
{
/*
** cannot allocate memory for fuse rozofs context
*/
return -1;
}
memset(rozofs_fuse_ctx_p,0,sizeof (rozofs_fuse_ctx_t));
/*
** clear read/write merge stats table
*/
memset(rozofs_write_merge_stats_tab,0,sizeof(uint64_t)*RZ_FUSE_WRITE_MAX);
memset (rozofs_write_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset (rozofs_read_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset(&rozofs_fuse_read_write_stats_buf,0,sizeof(rozofs_fuse_read_write_stats));
/*
** init of the context
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = NULL;
rozofs_fuse_ctx_p->ch = NULL;
rozofs_fuse_ctx_p->se = se;
rozofs_fuse_ctx_p->bufsize = 0;
rozofs_fuse_ctx_p->buf_fuse_req_p = NULL;
rozofs_fuse_ctx_p->dir_attr_invalidate = 0;
rozofs_fuse_ctx_p->initBufCount = rozofs_fuse_buffer_count;
while (1)
{
/*
** get the receive buffer size for former channel in order to create the request distributor:
** note: by default the fuse buffer is 4K+128K: for RozoFS the payload can reach 512K (x4)
*/
int bufsize = fuse_chan_bufsize(ch)*4;
rozofs_fuse_ctx_p->bufsize = bufsize;
/*
** create the distributor fro receiving data from fuse kernel
*/
status = rozofs_fuse_init_rcv_buffer_pool(ROZOFS_FUSE_RECV_BUF_COUNT,bufsize);
if (status < 0)
{
severe( "rozofs_fuse_init fuse buffer pool creation error(%d,%d)", (int)ROZOFS_FUSE_RECV_BUF_COUNT, (int)bufsize ) ;
status = -1;
break;
}
/*
** create the pool
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = ruc_buf_poolCreate(rozofs_fuse_buffer_count,sizeof(rozofs_fuse_save_ctx_t));
if (rozofs_fuse_ctx_p->fuseReqPoolRef == NULL)
{
severe( "rozofs_fuse_init buffer pool creation error(%d,%d)", (int)rozofs_fuse_buffer_count, (int)sizeof(rozofs_fuse_save_ctx_t) ) ;
status = -1;
break;
}
ruc_buffer_debug_register_pool("fuseCtx", rozofs_fuse_ctx_p->fuseReqPoolRef);
/*
** Allocate a head of list for queueing pending lookup requests
*/
for (i=0; i< ROZOFS_MAX_LKUP_QUEUE ; i++)
{
ruc_listHdrInit(&rozofs_lookup_queue[i]);
}
/*
** allocate a buffer for receiving the fuse request
*/
rozofs_fuse_ctx_p->buf_fuse_req_p = malloc(bufsize);
if (rozofs_fuse_ctx_p == NULL)
{
severe( "rozofs_fuse_init out of memory %d", bufsize ) ;
status = -1;
break;
}
/*
** get the fd of the channel
*/
rozofs_fuse_ctx_p->fd = fuse_chan_fd(ch);
/*
** wait the end of the share memroy init prior providing it to fuse
*/
while (rozofs_shared_mem_init_done == 0) sleep(1);
/*
** create a new channel with the specific operation for rozofs (non-blocking)
*/
rozofs_fuse_ctx_p->ch = fuse_chan_new(&rozofs_fuse_ch_ops,fuse_chan_fd(ch),bufsize,rozofs_fuse_ctx_p);
if (rozofs_fuse_ctx_p->ch == NULL)
{
severe( "rozofs_fuse_init fuse_chan_new error" ) ;
status = -1;
break;
}
/*
** remove the association between the initial session and channel
*/
fuse_session_remove_chan(ch);
/*
** OK, now add the new channel
*/
fuse_session_add_chan(se,rozofs_fuse_ctx_p->ch );
/*
** set the channel in non blocking mode
*/
if((fileflags=fcntl(rozofs_fuse_ctx_p->fd,F_GETFL,0))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
if((fcntl(rozofs_fuse_ctx_p->fd,F_SETFL,fileflags|O_NDELAY))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
/*
** send XON to the fuse channel
*/
{
int ret;
rozofs_fuse_ctx_p->ioctl_supported = 1;
rozofs_fuse_ctx_p->data_xon = 1;
while(1)
{
ret = ioctl(rozofs_fuse_ctx_p->fd,1,NULL);
if (ret < 0)
{
warning("ioctl error %s",strerror(errno));
rozofs_fuse_ctx_p->ioctl_supported = 0;
break;
}
if (rozofs_fuse_ctx_p->dir_attr_invalidate == 0)
{
ret = ioctl(rozofs_fuse_ctx_p->fd,3,NULL);
if (ret < 0)
{
warning("ioctl error %s",strerror(errno));
// rozofs_fuse_ctx_p->ioctl_supported = 0;
rozofs_fuse_ctx_p->dir_attr_invalidate = 1;
break;
}
}
break;
}
}
/*
** perform the connection with the socket controller
*/
/*
** OK, we are almost done, just need to connect with the socket controller
*/
rozofs_fuse_ctx_p->connectionId = ruc_sockctl_connect(rozofs_fuse_ctx_p->fd, // Reference of the socket
"rozofs_fuse", // name of the socket
3, // Priority within the socket controller
(void*)rozofs_fuse_ctx_p, // user param for socketcontroller callback
&rozofs_fuse_callBack_sock); // Default callbacks
if (rozofs_fuse_ctx_p->connectionId == NULL)
{
/*
** Fail to connect with the socket controller
*/
RUC_WARNING(-1);
status = -1;
break;
}
rozofs_fuse_get_ticker();
status = 0;
break;
}
/*
** attach the callback on socket controller
*/
//#warning no poller
ruc_sockCtrl_attach_applicative_poller(rozofs_fuse_scheduler_entry_point);
for(i = 0; i < 3;i++) fuse_profile[i] = 0;
uma_dbg_addTopic("fuse", rozofs_fuse_show);
return status;
}