/**
* init of the expt_thread
@param none
*/
int expt_thread_init()
{
if (expt_thread_init_done == 1) return 0;
/*
** clear the pointers to the contexts
*/
memset(expt_thread_ctx_p,0,sizeof(expt_ctx_t*[EXPT_MAX_TRACK_CTX]));
expt_thread_max_context = EXPT_MAX_TRACK_CTX ;
expt_thread_cur_context = 0;
/**
* create the writeback thread
*/
if ((errno = pthread_create(&exptctx_thread, NULL,
expt_thread, NULL)) != 0) {
severe("can't create inode tracking thread %s", strerror(errno));
return -1;
}
uma_dbg_addTopic("inode_trck",show_expt_thread);
expt_thread_init_done = 1;
return 0;
}
/*_______________________________________________________________________
* Initialize the thoughput measurement service
*
*/
void rozofs_throughput_counter_init(void) {
/*
** allocate memory for bandwidth computation
*/
rozofs_thr_counter[ROZOFS_READ_THR_E]= rozofs_thr_cnts_allocate(NULL, "Read");
rozofs_thr_counter[ROZOFS_WRITE_THR_E]= rozofs_thr_cnts_allocate(NULL, "Write");
/*
** Register the diagnostic function
*/
uma_dbg_addTopic("throughput", display_throughput);
}
/**
* This function is the entry point for setting rozofs in non-blocking mode
@param args->ch: reference of the fuse channnel
@param args->se: reference of the fuse session
@param args->max_transactions: max number of transactions that can be handled in parallel
@retval -1 on error
@retval : no retval -> only on fatal error
*/
int expgwc_start_nb_blocking_th(void *args) {
int ret;
//sem_t semForEver; /* semaphore for blocking the main thread doing nothing */
exportd_start_conf_param_t *args_p = (exportd_start_conf_param_t*)args;
ret = expgwc_non_blocking_init(args_p->debug_port, args_p->instance);
if (ret != RUC_OK) {
/*
** fatal error
*/
fatal("can't initialize non blocking thread");
return -1;
}
/*
** add profiler subject (exportd statistics)
*/
uma_dbg_addTopic("profiler", show_profiler);
// uma_dbg_addTopic("profiler_conf", show_profiler_conf);
uma_dbg_addTopic("profiler_short", show_profiler_short);
uma_dbg_addTopic("vfstat", show_vfstat);
uma_dbg_addTopic("vfstat_stor",show_vfstat_stor);
uma_dbg_addTopic("vfstat_vol",show_vfstat_vol);
uma_dbg_addTopic("vfstat_exp",show_vfstat_eid);
uma_dbg_addTopic("lv2_cache",show_lv2_attribute_cache);
uma_dbg_addTopic("flock", show_flock);
expgwc_non_blocking_thread_started = 1;
info("exportd non-blocking thread started (instance: %d, port: %d).",
args_p->instance, args_p->debug_port);
/*
** main loop
*/
while (1) {
ruc_sockCtrl_selectWait();
}
}
/**
* Init of the pseudo fuse thread
@param ch : initial channel
@param se : initial session
@param rozofs_fuse_buffer_count : number of request buffers (corresponds to the number of fuse save context)
@retval 0 on success
@retval -1 on error
*/
int rozofs_fuse_init(struct fuse_chan *ch,struct fuse_session *se,int rozofs_fuse_buffer_count)
{
int status = 0;
// return 0;
int fileflags;
rozofs_fuse_ctx_p = malloc(sizeof (rozofs_fuse_ctx_t));
if (rozofs_fuse_ctx_p == NULL)
{
/*
** cannot allocate memory for fuse rozofs context
*/
return -1;
}
/*
** clear read/write merge stats table
*/
memset(rozofs_write_merge_stats_tab,0,sizeof(uint64_t)*RZ_FUSE_WRITE_MAX);
memset (rozofs_write_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset (rozofs_read_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset(&rozofs_fuse_read_write_stats_buf,0,sizeof(rozofs_fuse_read_write_stats));
/*
** init of the context
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = NULL;
rozofs_fuse_ctx_p->ch = NULL;
rozofs_fuse_ctx_p->se = se;
rozofs_fuse_ctx_p->bufsize = 0;
rozofs_fuse_ctx_p->buf_fuse_req_p = NULL;
rozofs_fuse_ctx_p->initBufCount = rozofs_fuse_buffer_count;
while (1)
{
/*
** get the receive buffer size for former channel in order to create the request distributor
*/
int bufsize = fuse_chan_bufsize(ch);
rozofs_fuse_ctx_p->bufsize = bufsize;
/*
** create the pool
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = ruc_buf_poolCreate(rozofs_fuse_buffer_count,sizeof(rozofs_fuse_save_ctx_t));
if (rozofs_fuse_ctx_p->fuseReqPoolRef == NULL)
{
ERRLOG "rozofs_fuse_init buffer pool creation error(%d,%d)", (int)rozofs_fuse_buffer_count, (int)sizeof(rozofs_fuse_save_ctx_t) ENDERRLOG ;
status = -1;
break;
}
/*
** allocate a buffer for receiving the fuse request
*/
rozofs_fuse_ctx_p->buf_fuse_req_p = malloc(bufsize);
if (rozofs_fuse_ctx_p == NULL)
{
ERRLOG "rozofs_fuse_init out of memory %d", bufsize ENDERRLOG ;
status = -1;
break;
}
/*
** get the fd of the channel
*/
rozofs_fuse_ctx_p->fd = fuse_chan_fd(ch);
/*
** create a new channel with the specific operation for rozofs (non-blocking)
*/
rozofs_fuse_ctx_p->ch = fuse_chan_new(&rozofs_fuse_ch_ops,fuse_chan_fd(ch),fuse_chan_bufsize(ch),rozofs_fuse_ctx_p);
if (rozofs_fuse_ctx_p->ch == NULL)
{
ERRLOG "rozofs_fuse_init fuse_chan_new error" ENDERRLOG ;
status = -1;
break;
}
/*
** remove the association between the initial session and channel
*/
fuse_session_remove_chan(ch);
/*
** OK, now add the new channel
*/
fuse_session_add_chan(se,rozofs_fuse_ctx_p->ch );
/*
** set the channel in non blocking mode
*/
if((fileflags=fcntl(rozofs_fuse_ctx_p->fd,F_GETFL,0))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
if((fcntl(rozofs_fuse_ctx_p->fd,F_SETFL,fileflags|O_NDELAY))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
/*
** perform the connection with the socket controller
*/
/*
** OK, we are almost done, just need to connect with the socket controller
*/
rozofs_fuse_ctx_p->connectionId = ruc_sockctl_connect(rozofs_fuse_ctx_p->fd, // Reference of the socket
"rozofs_fuse", // name of the socket
3, // Priority within the socket controller
(void*)rozofs_fuse_ctx_p, // user param for socketcontroller callback
&rozofs_fuse_callBack_sock); // Default callbacks
if (rozofs_fuse_ctx_p->connectionId == NULL)
{
/*
** Fail to connect with the socket controller
*/
RUC_WARNING(-1);
status = -1;
break;
}
status = 0;
break;
}
uma_dbg_addTopic("fuse", rozofs_fuse_show);
return status;
}
/*__________________________________________________________________________
*/
void rozofs_timer_conf_dbg_init()
{
uma_dbg_addTopic("tmr_show", dbg_show_tmr);
uma_dbg_addTopic("tmr_set", dbg_set_tmr);
uma_dbg_addTopic("tmr_default", dbg_set_tmr_default);
}
/**
* This function is the entry point for setting rozofs in non-blocking mode
@param args->ch: reference of the fuse channnel
@param args->se: reference of the fuse session
@param args->max_transactions: max number of transactions that can be handled in parallel
@retval -1 on error
@retval : no retval -> only on fatal error
*/
int rozofs_stat_start(void *args) {
uma_dbg_thread_add_self("Main");
int ret;
//sem_t semForEver; /* semaphore for blocking the main thread doing nothing */
args_p = args;
exportclt_t *exportclt_p = (exportclt_t*)args_p->exportclt;
uint16_t debug_port = args_p->debug_port;
uint16_t export_listening_port = (uint16_t)exportclt_p->listen_port;
info("exportd listening port %d",export_listening_port);
/*
** allocate memory for bandwidth computation
*/
rozofs_throughput_counter_init();
ret = ruc_init(FALSE, debug_port,export_listening_port);
if (ret != RUC_OK) {
/*
** fatal error
*/
return -1;
}
{
char name[32];
sprintf(name, "rozofsmount %d", args_p->instance);
uma_dbg_set_name(name);
}
/*
** Send the file lock reset request to remove old locks
** (NB This is not actually a low level fuse API....)
*/
rozofs_ll_clear_client_file_lock(exportclt.eid,rozofs_client_hash);
/*
** init of the fuse part
*/
ret = rozofs_fuse_init(args_p->ch, args_p->se, args_p->max_transactions);
if (ret != RUC_OK) {
/*
** fatal error
*/
return -1;
}
#if 0
NO MORE LBG MASTER. USE ONLY LBG EXPORTD.
/*
** Perform the init with exportd--> setup of the TCP connection associated with the load balancing group
*/
uint16_t export_nb_port = rozofs_get_service_port_export_master_eproto();
if (export_lbg_initialize((exportclt_t*) args_p->exportclt, EXPORT_PROGRAM, EXPORT_VERSION, export_nb_port,
(af_stream_poll_CBK_t) rozofs_export_lbg_cnx_polling) != 0) {
severe("Cannot setup the load balancing group towards Exportd");
}
#endif
//#warning storcli instances are hardcoded
if (storcli_lbg_initialize((exportclt_t*) args_p->exportclt,"rozofsmount", args_p->instance, 1, 2) != 0) {
severe("Cannot setup the load balancing group towards StorCli");
}
rozofs_signals_declare("rozofsmount", common_config.nb_core_file);
/*
** change the priority of the main thread
*/
#if 1
{
struct sched_param my_priority;
int policy=-1;
int ret= 0;
pthread_getschedparam(pthread_self(),&policy,&my_priority);
info("storio main thread Scheduling policy = %s\n",
(policy == SCHED_OTHER) ? "SCHED_OTHER" :
(policy == SCHED_FIFO) ? "SCHED_FIFO" :
(policy == SCHED_RR) ? "SCHED_RR" :
"???");
#if 1
my_priority.sched_priority= 98;
policy = SCHED_RR;
ret = pthread_setschedparam(pthread_self(),policy,&my_priority);
if (ret < 0)
{
severe("error on sched_setscheduler: %s",strerror(errno));
}
pthread_getschedparam(pthread_self(),&policy,&my_priority);
DEBUG("RozoFS thread Scheduling policy (prio %d) = %s\n",my_priority.sched_priority,
(policy == SCHED_OTHER) ? "SCHED_OTHER" :
(policy == SCHED_FIFO) ? "SCHED_FIFO" :
(policy == SCHED_RR) ? "SCHED_RR" :
"???");
#endif
}
#endif
/*
** start the file KPI service
*/
rzkpi_file_service_init();
/*
** create the fuse threads
*/
info("FDL RozoFs Instance %d",args_p->instance);
ret = rozofs_fuse_thread_intf_create("localhost",args_p->instance,3);
if (ret < 0)
{
fatal("Cannot create fuse threads");
}
fuse_reply_thread_init();
uma_dbg_addTopic("fuse_reply_thread", show_fuse_reply_thread);
/*
** main loop
*/
while (1) {
ruc_sockCtrl_selectWait();
}
fatal( "main() code is rotten" );
}
/*__________________________________________________________________________
Register to the debug SWBB
==========================================================================
PARAMETERS:
-
RETURN: none
==========================================================================*/
void north_lbg_debug_init() {
uma_dbg_addTopic(NORTH_LBG_DEBUG_TOPIC, north_lbg_debug);
uma_dbg_addTopic("lbg_entries", north_lbg_entries_debug);
}
/*__________________________________________________________________________
Register to the debug SWBB
==========================================================================
PARAMETERS:
-
RETURN: none
==========================================================================*/
void rozofs_tx_debug_init() {
uma_dbg_addTopic(ROZOFS_TX_DEBUG_TOPIC, rozofs_tx_debug);
}
/**
* Init of the pseudo fuse thread
@param ch : initial channel
@param se : initial session
@param rozofs_fuse_buffer_count : number of request buffers (corresponds to the number of fuse save context)
@retval 0 on success
@retval -1 on error
*/
int rozofs_fuse_init(struct fuse_chan *ch,struct fuse_session *se,int rozofs_fuse_buffer_count)
{
int status = 0;
int i;
// return 0;
fuse_sharemem_init_done = 0;
int fileflags;
rozofs_fuse_ctx_p = malloc(sizeof (rozofs_fuse_ctx_t));
if (rozofs_fuse_ctx_p == NULL)
{
/*
** cannot allocate memory for fuse rozofs context
*/
return -1;
}
memset(rozofs_fuse_ctx_p,0,sizeof (rozofs_fuse_ctx_t));
/*
** clear read/write merge stats table
*/
memset(rozofs_write_merge_stats_tab,0,sizeof(uint64_t)*RZ_FUSE_WRITE_MAX);
memset (rozofs_write_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset (rozofs_read_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX);
memset(&rozofs_fuse_read_write_stats_buf,0,sizeof(rozofs_fuse_read_write_stats));
/*
** init of the context
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = NULL;
rozofs_fuse_ctx_p->ch = NULL;
rozofs_fuse_ctx_p->se = se;
rozofs_fuse_ctx_p->bufsize = 0;
rozofs_fuse_ctx_p->buf_fuse_req_p = NULL;
rozofs_fuse_ctx_p->dir_attr_invalidate = 0;
rozofs_fuse_ctx_p->initBufCount = rozofs_fuse_buffer_count;
while (1)
{
/*
** get the receive buffer size for former channel in order to create the request distributor:
** note: by default the fuse buffer is 4K+128K: for RozoFS the payload can reach 512K (x4)
*/
int bufsize = fuse_chan_bufsize(ch)*4;
rozofs_fuse_ctx_p->bufsize = bufsize;
/*
** create the distributor fro receiving data from fuse kernel
*/
status = rozofs_fuse_init_rcv_buffer_pool(ROZOFS_FUSE_RECV_BUF_COUNT,bufsize);
if (status < 0)
{
severe( "rozofs_fuse_init fuse buffer pool creation error(%d,%d)", (int)ROZOFS_FUSE_RECV_BUF_COUNT, (int)bufsize ) ;
status = -1;
break;
}
/*
** create the pool
*/
rozofs_fuse_ctx_p->fuseReqPoolRef = ruc_buf_poolCreate(rozofs_fuse_buffer_count,sizeof(rozofs_fuse_save_ctx_t));
if (rozofs_fuse_ctx_p->fuseReqPoolRef == NULL)
{
severe( "rozofs_fuse_init buffer pool creation error(%d,%d)", (int)rozofs_fuse_buffer_count, (int)sizeof(rozofs_fuse_save_ctx_t) ) ;
status = -1;
break;
}
ruc_buffer_debug_register_pool("fuseCtx", rozofs_fuse_ctx_p->fuseReqPoolRef);
/*
** Allocate a head of list for queueing pending lookup requests
*/
for (i=0; i< ROZOFS_MAX_LKUP_QUEUE ; i++)
{
ruc_listHdrInit(&rozofs_lookup_queue[i]);
}
/*
** allocate a buffer for receiving the fuse request
*/
rozofs_fuse_ctx_p->buf_fuse_req_p = malloc(bufsize);
if (rozofs_fuse_ctx_p == NULL)
{
severe( "rozofs_fuse_init out of memory %d", bufsize ) ;
status = -1;
break;
}
/*
** get the fd of the channel
*/
rozofs_fuse_ctx_p->fd = fuse_chan_fd(ch);
/*
** wait the end of the share memroy init prior providing it to fuse
*/
while (rozofs_shared_mem_init_done == 0) sleep(1);
/*
** create a new channel with the specific operation for rozofs (non-blocking)
*/
rozofs_fuse_ctx_p->ch = fuse_chan_new(&rozofs_fuse_ch_ops,fuse_chan_fd(ch),bufsize,rozofs_fuse_ctx_p);
if (rozofs_fuse_ctx_p->ch == NULL)
{
severe( "rozofs_fuse_init fuse_chan_new error" ) ;
status = -1;
break;
}
/*
** remove the association between the initial session and channel
*/
fuse_session_remove_chan(ch);
/*
** OK, now add the new channel
*/
fuse_session_add_chan(se,rozofs_fuse_ctx_p->ch );
/*
** set the channel in non blocking mode
*/
if((fileflags=fcntl(rozofs_fuse_ctx_p->fd,F_GETFL,0))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
if((fcntl(rozofs_fuse_ctx_p->fd,F_SETFL,fileflags|O_NDELAY))==-1)
{
RUC_WARNING(errno);
status = -1;
break;
}
/*
** send XON to the fuse channel
*/
{
int ret;
rozofs_fuse_ctx_p->ioctl_supported = 1;
rozofs_fuse_ctx_p->data_xon = 1;
while(1)
{
ret = ioctl(rozofs_fuse_ctx_p->fd,1,NULL);
if (ret < 0)
{
warning("ioctl error %s",strerror(errno));
rozofs_fuse_ctx_p->ioctl_supported = 0;
break;
}
if (rozofs_fuse_ctx_p->dir_attr_invalidate == 0)
{
ret = ioctl(rozofs_fuse_ctx_p->fd,3,NULL);
if (ret < 0)
{
warning("ioctl error %s",strerror(errno));
// rozofs_fuse_ctx_p->ioctl_supported = 0;
rozofs_fuse_ctx_p->dir_attr_invalidate = 1;
break;
}
}
break;
}
}
/*
** perform the connection with the socket controller
*/
/*
** OK, we are almost done, just need to connect with the socket controller
*/
rozofs_fuse_ctx_p->connectionId = ruc_sockctl_connect(rozofs_fuse_ctx_p->fd, // Reference of the socket
"rozofs_fuse", // name of the socket
3, // Priority within the socket controller
(void*)rozofs_fuse_ctx_p, // user param for socketcontroller callback
&rozofs_fuse_callBack_sock); // Default callbacks
if (rozofs_fuse_ctx_p->connectionId == NULL)
{
/*
** Fail to connect with the socket controller
*/
RUC_WARNING(-1);
status = -1;
break;
}
rozofs_fuse_get_ticker();
status = 0;
break;
}
/*
** attach the callback on socket controller
*/
//#warning no poller
ruc_sockCtrl_attach_applicative_poller(rozofs_fuse_scheduler_entry_point);
for(i = 0; i < 3;i++) fuse_profile[i] = 0;
uma_dbg_addTopic("fuse", rozofs_fuse_show);
return status;
}
/*__________________________________________________________________________
Register to the debug SWBB
==========================================================================
PARAMETERS:
-
RETURN: none
==========================================================================*/
void geo_proc_debug_init() {
uma_dbg_addTopic(GEO_CTX_DEBUG_TOPIC, geo_proc_debug);
}