int rozofs_storcli_north_interface_init(uint32_t eid,uint16_t rozofsmount_instance,uint32_t instance, int read_write_buf_count,int read_write_buf_sz) { int ret = 0; char sunpath[AF_UNIX_SOCKET_NAME_SIZE]; rozofs_storcli_read_write_buf_count = read_write_buf_count; rozofs_storcli_read_write_buf_sz = read_write_buf_sz ; while(1) { storcli_north_buffer_pool_p = ruc_buf_poolCreate(rozofs_storcli_read_write_buf_count,rozofs_storcli_read_write_buf_sz); if (storcli_north_buffer_pool_p == NULL) { ret = -1; severe( "ruc_buf_poolCreate(%d,%d)", rozofs_storcli_read_write_buf_count, rozofs_storcli_read_write_buf_sz ); break; } /* ** create the listening af unix socket on the north interface */ af_unix_test_family.rpc_recv_max_sz = rozofs_storcli_read_write_buf_sz; sprintf(sunpath,"%s%d.%d_lbg%d_inst_1",ROZOFS_SOCK_FAMILY_STORCLI_NORTH_SUNPATH,eid,rozofsmount_instance,instance); // sprintf(sunpath,"%s%d.%d_inst_%d",ROZOFS_SOCK_FAMILY_STORCLI_NORTH_SUNPATH,eid,rozofsmount_instance,instance); ret = af_unix_sock_listening_create("STORCLI_NORTH", sunpath, &af_unix_test_family ); break; } return ret; }
/*__________________________________________________________________________ * Initialize the disk thread interface * * @param hostname storio hostname (for tests) * @param nb_threads Number of threads that can process the disk requests * @param nb_buffer Number of buffer for sending and number of receiving buffer * * @retval 0 on success -1 in case of error */ int storio_disk_thread_intf_create(char * hostname, int instance_id, int nb_threads, int nb_buffer) { af_unix_disk_thread_count = nb_threads; af_unix_disk_pool_send = ruc_buf_poolCreate(nb_buffer,sizeof(storio_disk_thread_msg_t)); if (af_unix_disk_pool_send == NULL) { fatal("storio_disk_thread_intf_create af_unix_disk_pool_send (%d,%d)", nb_buffer, (int)sizeof(storio_disk_thread_msg_t)); return -1; } ruc_buffer_debug_register_pool("diskSendPool",af_unix_disk_pool_send); af_unix_disk_pool_recv = ruc_buf_poolCreate(1,sizeof(storio_disk_thread_msg_t)); if (af_unix_disk_pool_recv == NULL) { fatal("storio_disk_thread_intf_create af_unix_disk_pool_recv (1,%d)", (int)sizeof(storio_disk_thread_msg_t)); return -1; } ruc_buffer_debug_register_pool("diskRecvPool",af_unix_disk_pool_recv); /* ** init of the AF_UNIX sockaddr associated with the south socket (socket used for disk response receive) */ storio_set_socket_name_with_hostname(&storio_south_socket_name,ROZOFS_SOCK_FAMILY_DISK_SOUTH,hostname, instance_id); /* ** hostname is required for the case when several storaged run on the same server ** as is the case of test on one server only */ af_unix_disk_south_socket_ref = af_unix_disk_response_socket_create(storio_south_socket_name.sun_path); if (af_unix_disk_south_socket_ref < 0) { fatal("storio_create_disk_thread_intf af_unix_sock_create(%s) %s",storio_south_socket_name.sun_path, strerror(errno)); return -1; } /* ** init of the AF_UNIX sockaddr associated with the north socket (socket used for disk request receive) */ storio_set_socket_name_with_hostname(&storio_north_socket_name,ROZOFS_SOCK_FAMILY_DISK_NORTH,hostname,instance_id); uma_dbg_addTopic_option("diskThreads", disk_thread_debug,UMA_DBG_OPTION_RESET); /* ** attach the callback on socket controller */ ruc_sockCtrl_attach_applicative_poller(af_unix_disk_scheduler_entry_point); return storio_disk_thread_create(hostname, nb_threads, instance_id); }
/** * Init of the pseudo fuse thread @param ch : initial channel @param se : initial session @param rozofs_fuse_buffer_count : number of request buffers (corresponds to the number of fuse save context) @retval 0 on success @retval -1 on error */ int rozofs_fuse_init(struct fuse_chan *ch,struct fuse_session *se,int rozofs_fuse_buffer_count) { int status = 0; // return 0; int fileflags; rozofs_fuse_ctx_p = malloc(sizeof (rozofs_fuse_ctx_t)); if (rozofs_fuse_ctx_p == NULL) { /* ** cannot allocate memory for fuse rozofs context */ return -1; } /* ** clear read/write merge stats table */ memset(rozofs_write_merge_stats_tab,0,sizeof(uint64_t)*RZ_FUSE_WRITE_MAX); memset (rozofs_write_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX); memset (rozofs_read_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX); memset(&rozofs_fuse_read_write_stats_buf,0,sizeof(rozofs_fuse_read_write_stats)); /* ** init of the context */ rozofs_fuse_ctx_p->fuseReqPoolRef = NULL; rozofs_fuse_ctx_p->ch = NULL; rozofs_fuse_ctx_p->se = se; rozofs_fuse_ctx_p->bufsize = 0; rozofs_fuse_ctx_p->buf_fuse_req_p = NULL; rozofs_fuse_ctx_p->initBufCount = rozofs_fuse_buffer_count; while (1) { /* ** get the receive buffer size for former channel in order to create the request distributor */ int bufsize = fuse_chan_bufsize(ch); rozofs_fuse_ctx_p->bufsize = bufsize; /* ** create the pool */ rozofs_fuse_ctx_p->fuseReqPoolRef = ruc_buf_poolCreate(rozofs_fuse_buffer_count,sizeof(rozofs_fuse_save_ctx_t)); if (rozofs_fuse_ctx_p->fuseReqPoolRef == NULL) { ERRLOG "rozofs_fuse_init buffer pool creation error(%d,%d)", (int)rozofs_fuse_buffer_count, (int)sizeof(rozofs_fuse_save_ctx_t) ENDERRLOG ; status = -1; break; } /* ** allocate a buffer for receiving the fuse request */ rozofs_fuse_ctx_p->buf_fuse_req_p = malloc(bufsize); if (rozofs_fuse_ctx_p == NULL) { ERRLOG "rozofs_fuse_init out of memory %d", bufsize ENDERRLOG ; status = -1; break; } /* ** get the fd of the channel */ rozofs_fuse_ctx_p->fd = fuse_chan_fd(ch); /* ** create a new channel with the specific operation for rozofs (non-blocking) */ rozofs_fuse_ctx_p->ch = fuse_chan_new(&rozofs_fuse_ch_ops,fuse_chan_fd(ch),fuse_chan_bufsize(ch),rozofs_fuse_ctx_p); if (rozofs_fuse_ctx_p->ch == NULL) { ERRLOG "rozofs_fuse_init fuse_chan_new error" ENDERRLOG ; status = -1; break; } /* ** remove the association between the initial session and channel */ fuse_session_remove_chan(ch); /* ** OK, now add the new channel */ fuse_session_add_chan(se,rozofs_fuse_ctx_p->ch ); /* ** set the channel in non blocking mode */ if((fileflags=fcntl(rozofs_fuse_ctx_p->fd,F_GETFL,0))==-1) { RUC_WARNING(errno); status = -1; break; } if((fcntl(rozofs_fuse_ctx_p->fd,F_SETFL,fileflags|O_NDELAY))==-1) { RUC_WARNING(errno); status = -1; break; } /* ** perform the connection with the socket controller */ /* ** OK, we are almost done, just need to connect with the socket controller */ rozofs_fuse_ctx_p->connectionId = ruc_sockctl_connect(rozofs_fuse_ctx_p->fd, // Reference of the socket "rozofs_fuse", // name of the socket 3, // Priority within the socket controller (void*)rozofs_fuse_ctx_p, // user param for socketcontroller callback &rozofs_fuse_callBack_sock); // Default callbacks if (rozofs_fuse_ctx_p->connectionId == NULL) { /* ** Fail to connect with the socket controller */ RUC_WARNING(-1); status = -1; break; } status = 0; break; } uma_dbg_addTopic("fuse", rozofs_fuse_show); return status; }
/** * Init of the pseudo fuse thread @param ch : initial channel @param se : initial session @param rozofs_fuse_buffer_count : number of request buffers (corresponds to the number of fuse save context) @retval 0 on success @retval -1 on error */ int rozofs_fuse_init(struct fuse_chan *ch,struct fuse_session *se,int rozofs_fuse_buffer_count) { int status = 0; int i; // return 0; fuse_sharemem_init_done = 0; int fileflags; rozofs_fuse_ctx_p = malloc(sizeof (rozofs_fuse_ctx_t)); if (rozofs_fuse_ctx_p == NULL) { /* ** cannot allocate memory for fuse rozofs context */ return -1; } memset(rozofs_fuse_ctx_p,0,sizeof (rozofs_fuse_ctx_t)); /* ** clear read/write merge stats table */ memset(rozofs_write_merge_stats_tab,0,sizeof(uint64_t)*RZ_FUSE_WRITE_MAX); memset (rozofs_write_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX); memset (rozofs_read_buf_section_table,0,sizeof(uint64_t)*ROZOFS_FUSE_NB_OF_BUSIZE_SECTION_MAX); memset(&rozofs_fuse_read_write_stats_buf,0,sizeof(rozofs_fuse_read_write_stats)); /* ** init of the context */ rozofs_fuse_ctx_p->fuseReqPoolRef = NULL; rozofs_fuse_ctx_p->ch = NULL; rozofs_fuse_ctx_p->se = se; rozofs_fuse_ctx_p->bufsize = 0; rozofs_fuse_ctx_p->buf_fuse_req_p = NULL; rozofs_fuse_ctx_p->dir_attr_invalidate = 0; rozofs_fuse_ctx_p->initBufCount = rozofs_fuse_buffer_count; while (1) { /* ** get the receive buffer size for former channel in order to create the request distributor: ** note: by default the fuse buffer is 4K+128K: for RozoFS the payload can reach 512K (x4) */ int bufsize = fuse_chan_bufsize(ch)*4; rozofs_fuse_ctx_p->bufsize = bufsize; /* ** create the distributor fro receiving data from fuse kernel */ status = rozofs_fuse_init_rcv_buffer_pool(ROZOFS_FUSE_RECV_BUF_COUNT,bufsize); if (status < 0) { severe( "rozofs_fuse_init fuse buffer pool creation error(%d,%d)", (int)ROZOFS_FUSE_RECV_BUF_COUNT, (int)bufsize ) ; status = -1; break; } /* ** create the pool */ rozofs_fuse_ctx_p->fuseReqPoolRef = ruc_buf_poolCreate(rozofs_fuse_buffer_count,sizeof(rozofs_fuse_save_ctx_t)); if (rozofs_fuse_ctx_p->fuseReqPoolRef == NULL) { severe( "rozofs_fuse_init buffer pool creation error(%d,%d)", (int)rozofs_fuse_buffer_count, (int)sizeof(rozofs_fuse_save_ctx_t) ) ; status = -1; break; } ruc_buffer_debug_register_pool("fuseCtx", rozofs_fuse_ctx_p->fuseReqPoolRef); /* ** Allocate a head of list for queueing pending lookup requests */ for (i=0; i< ROZOFS_MAX_LKUP_QUEUE ; i++) { ruc_listHdrInit(&rozofs_lookup_queue[i]); } /* ** allocate a buffer for receiving the fuse request */ rozofs_fuse_ctx_p->buf_fuse_req_p = malloc(bufsize); if (rozofs_fuse_ctx_p == NULL) { severe( "rozofs_fuse_init out of memory %d", bufsize ) ; status = -1; break; } /* ** get the fd of the channel */ rozofs_fuse_ctx_p->fd = fuse_chan_fd(ch); /* ** wait the end of the share memroy init prior providing it to fuse */ while (rozofs_shared_mem_init_done == 0) sleep(1); /* ** create a new channel with the specific operation for rozofs (non-blocking) */ rozofs_fuse_ctx_p->ch = fuse_chan_new(&rozofs_fuse_ch_ops,fuse_chan_fd(ch),bufsize,rozofs_fuse_ctx_p); if (rozofs_fuse_ctx_p->ch == NULL) { severe( "rozofs_fuse_init fuse_chan_new error" ) ; status = -1; break; } /* ** remove the association between the initial session and channel */ fuse_session_remove_chan(ch); /* ** OK, now add the new channel */ fuse_session_add_chan(se,rozofs_fuse_ctx_p->ch ); /* ** set the channel in non blocking mode */ if((fileflags=fcntl(rozofs_fuse_ctx_p->fd,F_GETFL,0))==-1) { RUC_WARNING(errno); status = -1; break; } if((fcntl(rozofs_fuse_ctx_p->fd,F_SETFL,fileflags|O_NDELAY))==-1) { RUC_WARNING(errno); status = -1; break; } /* ** send XON to the fuse channel */ { int ret; rozofs_fuse_ctx_p->ioctl_supported = 1; rozofs_fuse_ctx_p->data_xon = 1; while(1) { ret = ioctl(rozofs_fuse_ctx_p->fd,1,NULL); if (ret < 0) { warning("ioctl error %s",strerror(errno)); rozofs_fuse_ctx_p->ioctl_supported = 0; break; } if (rozofs_fuse_ctx_p->dir_attr_invalidate == 0) { ret = ioctl(rozofs_fuse_ctx_p->fd,3,NULL); if (ret < 0) { warning("ioctl error %s",strerror(errno)); // rozofs_fuse_ctx_p->ioctl_supported = 0; rozofs_fuse_ctx_p->dir_attr_invalidate = 1; break; } } break; } } /* ** perform the connection with the socket controller */ /* ** OK, we are almost done, just need to connect with the socket controller */ rozofs_fuse_ctx_p->connectionId = ruc_sockctl_connect(rozofs_fuse_ctx_p->fd, // Reference of the socket "rozofs_fuse", // name of the socket 3, // Priority within the socket controller (void*)rozofs_fuse_ctx_p, // user param for socketcontroller callback &rozofs_fuse_callBack_sock); // Default callbacks if (rozofs_fuse_ctx_p->connectionId == NULL) { /* ** Fail to connect with the socket controller */ RUC_WARNING(-1); status = -1; break; } rozofs_fuse_get_ticker(); status = 0; break; } /* ** attach the callback on socket controller */ //#warning no poller ruc_sockCtrl_attach_applicative_poller(rozofs_fuse_scheduler_entry_point); for(i = 0; i < 3;i++) fuse_profile[i] = 0; uma_dbg_addTopic("fuse", rozofs_fuse_show); return status; }
/** af_unix_module_init create the Transaction context pool @param : af_unix_ctx_count : number of Transaction context @param : max_xmit_buf_count : number of xmit buffers @param : max_xmit_buf_size : xmit buffer size @param : max_recv_buf_count : number of receive buffers @param : max_recv_buf_count : receive buffer size @retval : RUC_OK : done @retval RUC_NOK : out of memory */ uint32_t af_unix_module_init(uint32_t af_unix_ctx_count, int max_xmit_buf_count, int max_xmit_buf_size, int max_recv_buf_count, int max_recv_buf_size ) { af_unix_ctx_generic_t *p; uint32_t idxCur; ruc_obj_desc_t *pnext; uint32_t ret = RUC_OK; af_unix_xmit_buf_count = max_xmit_buf_count; af_unix_xmit_buf_size = max_xmit_buf_size; af_unix_recv_buf_count = max_recv_buf_count; af_unix_recv_buf_size = max_recv_buf_size; af_unix_context_allocated = 0; af_unix_context_count = af_unix_ctx_count; while (1) { af_unix_buffer_pool_tb[0] = ruc_buf_poolCreate(af_unix_xmit_buf_count, af_unix_xmit_buf_size); if (af_unix_buffer_pool_tb[0] == NULL) { ret = RUC_NOK; severe( "xmit ruc_buf_poolCreate(%d,%d)", af_unix_xmit_buf_count, af_unix_xmit_buf_size ); break; } af_unix_buffer_pool_tb[1] = ruc_buf_poolCreate(af_unix_recv_buf_count, af_unix_recv_buf_size); if (af_unix_buffer_pool_tb[1] == NULL) { ret = RUC_NOK; severe( "rcv ruc_buf_poolCreate(%d,%d)", af_unix_recv_buf_count, af_unix_recv_buf_size ); break; } af_unix_context_freeListHead = (af_unix_ctx_generic_t*) NULL; /* ** create the active list */ ruc_listHdrInit((ruc_obj_desc_t*) & af_unix_context_activeListHead); /* ** create the af unix context pool */ af_unix_context_freeListHead = (af_unix_ctx_generic_t*) ruc_listCreate(af_unix_ctx_count, sizeof (af_unix_ctx_generic_t)); if (af_unix_context_freeListHead == (af_unix_ctx_generic_t*) NULL) { /* ** out of memory */ RUC_WARNING(af_unix_ctx_count * sizeof (af_unix_ctx_generic_t)); return RUC_NOK; } /* ** store the pointer to the first context */ af_unix_context_pfirst = af_unix_context_freeListHead; /* ** initialize each entry of the free list */ idxCur = 0; pnext = (ruc_obj_desc_t*) NULL; while ((p = (af_unix_ctx_generic_t*) ruc_objGetNext((ruc_obj_desc_t*) af_unix_context_freeListHead, &pnext)) != (af_unix_ctx_generic_t*) NULL) { p->index = idxCur; p->free = TRUE; af_unix_ctxInit(p, TRUE); idxCur++; } af_unix_debug_init(); break; } return ret; }